diff --git a/apps/backend/internal/backendapp/orchestrator.go b/apps/backend/internal/backendapp/orchestrator.go index 718cac1d18..833d35da51 100644 --- a/apps/backend/internal/backendapp/orchestrator.go +++ b/apps/backend/internal/backendapp/orchestrator.go @@ -109,13 +109,14 @@ func provideOrchestrator( if err != nil { return nil, nil, fmt.Errorf("init message queue repo: %w", err) } - queueSettings := resolveQueueSettingsWithStore(settingsStore, pool, log, queueConfiguration(cfg)).Effective + queueResolution := resolveQueueSettingsWithStore(settingsStore, pool, log, queueConfiguration(cfg)) + queueSettings := queueResolution.Effective maxPerSession := queueSettings.MaxPerSession mergeEnabled := queueSettings.MergeEnabled autoMergeEnabled := queueSettings.AutoMergeEnabled msgQueue := messagequeue.NewService(queueRepo, maxPerSession, log) msgQueue.SetMergeEnabled(mergeEnabled) - msgQueue.SetAutoMergeEnabled(autoMergeEnabled) + msgQueue.SetAutoMergePolicy(autoMergeEnabled, queueResolution.Settings.AutoMergeRevision) log.Info("Message queue initialized", zap.Int("max_per_session", maxPerSession), zap.Bool("merge_enabled", mergeEnabled), diff --git a/apps/backend/internal/gateway/websocket/task_notifications.go b/apps/backend/internal/gateway/websocket/task_notifications.go index 798adaea17..f4b8250ec9 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications.go +++ b/apps/backend/internal/gateway/websocket/task_notifications.go @@ -11,6 +11,11 @@ import ( "go.uber.org/zap" ) +const ( + queueStatusScopeKey = "queue_status_scope" + queueStatusScopeTask = "task" +) + type TaskEventBroadcaster struct { hub *Hub subscriptions []bus.Subscription @@ -289,6 +294,9 @@ func (b *TaskEventBroadcaster) routeBroadcast( b.hub.BroadcastToSession(sessionID, msg) return nil } + if extractStringField(data, queueStatusScopeKey) == queueStatusScopeTask { + return nil + } case ws.ActionExecutorPrepareProgress, ws.ActionExecutorPrepareCompleted: // Broadcast to the owning workspace's clients so prepare // progress/warnings are available when the user navigates to the diff --git a/apps/backend/internal/gateway/websocket/task_notifications_test.go b/apps/backend/internal/gateway/websocket/task_notifications_test.go index 4361feca89..7dde94c519 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications_test.go +++ b/apps/backend/internal/gateway/websocket/task_notifications_test.go @@ -246,6 +246,28 @@ func TestTaskEventBroadcaster_CancellationIsSessionScoped(t *testing.T) { } } +func TestTaskEventBroadcaster_DropsTaskScopedQueueStatusWithoutSession(t *testing.T) { + hub := newTestHub(t) + hub.setAuthPolicy(AuthPolicy{Enforced: func() bool { return true }}) + broadcaster := &TaskEventBroadcaster{hub: hub, logger: testLogger()} + payload := map[string]any{ + "task_id": "task-without-session", + "queue_status_scope": "task", + } + + require.NoError(t, broadcaster.broadcastEvent( + context.Background(), + bus.NewEvent(events.MessageQueueStatusChanged, "test", payload), + ws.ActionMessageQueueStatusChanged, + )) + + select { + case leaked := <-hub.broadcast: + t.Fatalf("task-scoped queue status was globally broadcast: %s", leaked.Action) + default: + } +} + func TestTaskEventBroadcaster_DropsUnscopedGitHubCIOptionsWhenAuthIsEnforced(t *testing.T) { hub := newTestHub(t) hub.setAuthPolicy(AuthPolicy{Enforced: func() bool { return true }}) diff --git a/apps/backend/internal/mcp/handlers/config_handlers_test.go b/apps/backend/internal/mcp/handlers/config_handlers_test.go index c4d7f00eaa..915ee3a45b 100644 --- a/apps/backend/internal/mcp/handlers/config_handlers_test.go +++ b/apps/backend/internal/mcp/handlers/config_handlers_test.go @@ -3,6 +3,7 @@ package handlers import ( "context" "encoding/json" + "errors" "fmt" "strings" "testing" @@ -398,6 +399,9 @@ func (r *recordingMessageQueuer) QueueMessageWithMetadata(_ context.Context, ses QueuedBy: userID, Metadata: metadata, } + if msg.ID == "" { + msg.ID = fmt.Sprintf("queued-%d", len(r.calls)+1) + } r.calls = append(r.calls, msg) return &msg, nil } @@ -408,21 +412,132 @@ type pendingMoveRecordingQueuer struct { pendingMoves []messagequeue.PendingMove } -func (r *pendingMoveRecordingQueuer) SetPendingMove(_ context.Context, sessionID string, move *messagequeue.PendingMove) { +func (r *pendingMoveRecordingQueuer) SetPendingMove(_ context.Context, sessionID string, move *messagequeue.PendingMove) error { r.pendingSessionID = sessionID if move != nil { r.pendingMoves = append(r.pendingMoves, *move) } + return nil } -func (r *recordingMessageQueuer) SetPendingMove(_ context.Context, _ string, _ *messagequeue.PendingMove) { +func (r *recordingMessageQueuer) SetPendingMove(_ context.Context, _ string, _ *messagequeue.PendingMove) error { + return nil } -// TakeQueued is a no-op stub — the unit tests below don't exercise rollback, -// they just exercise QueueMessage. Returning (nil, false) is consistent with -// "nothing to take", which is what the rollback path checks before logging. -func (r *recordingMessageQueuer) TakeQueued(_ context.Context, _ string) (*messagequeue.QueuedMessage, bool) { - return nil, false +func (r *recordingMessageQueuer) RemoveEntryForSession( + _ context.Context, + identity messagequeue.QueueSessionIdentity, + entryID string, +) (*messagequeue.QueueRemovalResult, error) { + for index := range r.calls { + entry := r.calls[index] + if entry.ID == entryID && entry.SessionID == identity.SessionID && entry.TaskID == identity.TaskID { + r.calls = append(r.calls[:index], r.calls[index+1:]...) + return &messagequeue.QueueRemovalResult{Removed: []messagequeue.QueuedMessage{entry}}, nil + } + } + return nil, messagequeue.ErrEntryNotFound +} + +func TestApplyMoveTaskImmediate_RollsBackExactHandoffAfterMoveFailure(t *testing.T) { + svc, repo := newTestTaskService(t) + ctx := context.Background() + now := time.Now().UTC() + require.NoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ + ID: "ws-rollback", Name: "Rollback", CreatedAt: now, UpdatedAt: now, + })) + require.NoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ + ID: "wf-rollback", WorkspaceID: "ws-rollback", Name: "Board", CreatedAt: now, UpdatedAt: now, + })) + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: "task-rollback", WorkspaceID: "ws-rollback", WorkflowID: "wf-rollback", + Title: "Rollback", State: v1.TaskStateTODO, CreatedAt: now, UpdatedAt: now, + })) + session := &models.TaskSession{ + ID: "session-rollback", TaskID: "task-rollback", State: models.TaskSessionStateIdle, + IsPrimary: true, StartedAt: now, UpdatedAt: now, + } + require.NoError(t, repo.CreateTaskSession(ctx, session)) + + queue := &recordingMessageQueuer{} + queue.calls = append(queue.calls, messagequeue.QueuedMessage{ + ID: "preexisting", SessionID: session.ID, TaskID: session.TaskID, Content: "keep me", + }) + h := &Handlers{taskSvc: svc, messageQueue: queue, logger: testLogger(t).WithFields()} + msg := makeWSMessage(t, ws.ActionMCPMoveTask, map[string]interface{}{}) + response, err := h.applyMoveTaskImmediate(ctx, msg, struct { + TaskID string `json:"task_id"` + WorkflowID string `json:"workflow_id"` + WorkflowStepID string `json:"workflow_step_id"` + Position int `json:"position"` + Prompt string `json:"prompt"` + SenderSessionID string `json:"sender_session_id"` + }{ + TaskID: "task-rollback", WorkflowID: "missing-workflow", + WorkflowStepID: "missing-step", Prompt: "handoff", + }, session) + require.NoError(t, err) + assertWSError(t, response, ws.ErrorCodeInternalError) + require.Len(t, queue.calls, 1) + assert.Equal(t, "preexisting", queue.calls[0].ID) +} + +type pendingMoveFailingQueuer struct { + recordingMessageQueuer + pendingErr error + removedIDs []string +} + +func (r *pendingMoveFailingQueuer) SetPendingMove( + _ context.Context, + _ string, + _ *messagequeue.PendingMove, +) error { + return r.pendingErr +} + +func (r *pendingMoveFailingQueuer) RemoveEntryForSession( + _ context.Context, + _ messagequeue.QueueSessionIdentity, + entryID string, +) (*messagequeue.QueueRemovalResult, error) { + r.removedIDs = append(r.removedIDs, entryID) + for index := range r.calls { + if r.calls[index].ID == entryID { + entry := r.calls[index] + r.calls = append(r.calls[:index], r.calls[index+1:]...) + return &messagequeue.QueueRemovalResult{Removed: []messagequeue.QueuedMessage{entry}}, nil + } + } + return nil, messagequeue.ErrEntryNotFound +} + +func TestDeferMoveTask_RollsBackHandoffWhenPendingMovePersistenceFails(t *testing.T) { + svc, repo := newTestTaskService(t) + seedRunningTask( + t, repo, + "ws-pending-failure", "wf-pending-failure", "task-pending-failure", + "session-pending-failure", "step-current", + ) + queue := &pendingMoveFailingQueuer{pendingErr: errors.New("persist pending move")} + queue.calls = append(queue.calls, messagequeue.QueuedMessage{ + ID: "preexisting", SessionID: "session-pending-failure", + TaskID: "task-pending-failure", Content: "keep me", + }) + h := &Handlers{taskSvc: svc, messageQueue: queue, logger: testLogger(t).WithFields()} + msg := makeWSMessage(t, ws.ActionMCPMoveTask, map[string]interface{}{ + "task_id": "task-pending-failure", + "workflow_id": "wf-pending-failure", + "workflow_step_id": "step-target", + "prompt": "handoff", + }) + + response, err := h.handleMoveTask(context.Background(), msg) + require.NoError(t, err) + assertWSError(t, response, ws.ErrorCodeInternalError) + require.Equal(t, []string{"queued-2"}, queue.removedIDs) + require.Len(t, queue.calls, 1) + assert.Equal(t, "preexisting", queue.calls[0].ID) } // TestQueueMoveTaskPrompt_NilQueueReturnsError ensures the call is safe (no panic) @@ -431,7 +546,9 @@ func (r *recordingMessageQueuer) TakeQueued(_ context.Context, _ string) (*messa func TestQueueMoveTaskPrompt_NilQueueReturnsError(t *testing.T) { h := &Handlers{logger: testLogger(t).WithFields()} - err := h.queueMoveTaskPrompt(context.Background(), "task-1", "session-1", "fix issues") + _, err := h.queueMoveTaskPrompt(context.Background(), messagequeue.QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "inc-1", + }, "fix issues") require.Error(t, err) assert.Contains(t, err.Error(), "message queue") } @@ -446,7 +563,9 @@ func TestQueueMoveTaskPrompt_EmptySessionIDReturnsError(t *testing.T) { logger: testLogger(t).WithFields(), } - err := h.queueMoveTaskPrompt(context.Background(), "task-1", "", "fix issues") + _, err := h.queueMoveTaskPrompt(context.Background(), messagequeue.QueueSessionIdentity{ + TaskID: "task-1", SessionIncarnationID: "inc-1", + }, "fix issues") require.Error(t, err) assert.Contains(t, err.Error(), "primary session") assert.Empty(t, queue.calls, "queue must not be invoked without a session ID") @@ -462,7 +581,9 @@ func TestQueueMoveTaskPrompt_QueuesWithExpectedFields(t *testing.T) { logger: testLogger(t).WithFields(), } - err := h.queueMoveTaskPrompt(context.Background(), "task-1", "session-99", "Please fix the failing test in foo_test.go") + _, err := h.queueMoveTaskPrompt(context.Background(), messagequeue.QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-99", SessionIncarnationID: "inc-99", + }, "Please fix the failing test in foo_test.go") require.NoError(t, err) require.Len(t, queue.calls, 1) diff --git a/apps/backend/internal/mcp/handlers/config_task_handlers.go b/apps/backend/internal/mcp/handlers/config_task_handlers.go index e264324429..86a1ec6379 100644 --- a/apps/backend/internal/mcp/handlers/config_task_handlers.go +++ b/apps/backend/internal/mcp/handlers/config_task_handlers.go @@ -157,24 +157,34 @@ func (h *Handlers) deferMoveTask( } moveID := uuid.NewString() + var handoff *queuedMoveTaskPrompt if req.Prompt != "" { wrapped := "You were moved to this step with the following message: " + req.Prompt - if err := h.queueMoveTaskPromptWithMoveID(ctx, req.TaskID, session.ID, wrapped, moveID); err != nil { + var err error + handoff, err = h.queueMoveTaskPromptWithMoveID(ctx, queueIdentityForSession(session), wrapped, moveID) + if err != nil { h.logger.Error("move_task: failed to queue hand-off prompt", zap.String("task_id", req.TaskID), zap.String("session_id", session.ID), zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "failed to queue move_task hand-off prompt", nil) } } - h.messageQueue.SetPendingMove(ctx, session.ID, &messagequeue.PendingMove{ - MoveID: moveID, - TaskID: req.TaskID, - WorkflowID: req.WorkflowID, - WorkflowStepID: req.WorkflowStepID, - Position: req.Position, - Actor: string(wfmodels.StepTransitionActorAgent), - SenderSessionID: req.SenderSessionID, - }) + if err := h.messageQueue.SetPendingMove(ctx, session.ID, &messagequeue.PendingMove{ + MoveID: moveID, + SessionIncarnationID: session.QueueIncarnationID, + TaskID: req.TaskID, + WorkflowID: req.WorkflowID, + WorkflowStepID: req.WorkflowStepID, + Position: req.Position, + Actor: string(wfmodels.StepTransitionActorAgent), + SenderSessionID: req.SenderSessionID, + }); err != nil { + h.rollbackMoveTaskPrompt(ctx, handoff) + h.logger.Error("move_task: failed to persist deferred move", + zap.String("task_id", req.TaskID), zap.String("session_id", session.ID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, + "failed to persist deferred move", nil) + } return ws.NewResponse(msg.ID, msg.Action, h.synthesizeMovedTaskDTO(ctx, req.TaskID, req.WorkflowID, req.WorkflowStepID, req.Position)) } @@ -196,16 +206,17 @@ func (h *Handlers) applyMoveTaskImmediate( }, session *models.TaskSession, ) (*ws.Message, error) { - queuedSessionID := "" + var handoff *queuedMoveTaskPrompt if req.Prompt != "" && session != nil { wrapped := "You were moved to this step with the following message: " + req.Prompt - if err := h.queueMoveTaskPrompt(ctx, req.TaskID, session.ID, wrapped); err != nil { + var err error + handoff, err = h.queueMoveTaskPrompt(ctx, queueIdentityForSession(session), wrapped) + if err != nil { h.logger.Error("move_task: failed to queue hand-off prompt for idle session", zap.String("task_id", req.TaskID), zap.String("session_id", session.ID), zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "failed to queue move_task hand-off prompt", nil) } - queuedSessionID = session.ID } // Attribution uses the CALLING session (req.SenderSessionID, injected @@ -225,15 +236,8 @@ func (h *Handlers) applyMoveTaskImmediate( result, err := h.taskSvc.MoveTaskWithOptions(moveCtx, req.TaskID, req.WorkflowID, req.WorkflowStepID, req.Position, service.MoveTaskOptions{StepHistoryActor: wfmodels.StepTransitionActorAgent}) if err != nil { - // Roll back the queued prompt — without this, the next turn would - // deliver a "You were moved to this step…" message for a transition - // that didn't actually happen. - if queuedSessionID != "" && h.messageQueue != nil { - if _, ok := h.messageQueue.TakeQueued(ctx, queuedSessionID); ok { - h.logger.Warn("move_task: dropped queued hand-off prompt after MoveTask failure", - zap.String("task_id", req.TaskID), zap.String("session_id", queuedSessionID)) - } - } + // Without the transition, its handoff must not reach a later turn. + h.rollbackMoveTaskPrompt(ctx, handoff) h.logger.Error("failed to move task", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, classifyMoveTaskError(err), moveTaskErrorMessage(err), nil) } @@ -319,36 +323,90 @@ func (h *Handlers) lookupSession(ctx context.Context, taskID string) (*models.Ta return session, nil } -// queueMoveTaskPrompt enqueues a user-supplied prompt on the task's primary session. -// Returns an error when the queue itself is missing or QueueMessage fails — the -// caller decides whether to fail the whole move (running-session deferred path) -// or proceed (idle path), since a queue failure makes the deferred contract -// impossible to honor. -func (h *Handlers) queueMoveTaskPrompt(ctx context.Context, taskID, sessionID, prompt string) error { - return h.queueMoveTaskPromptWithMoveID(ctx, taskID, sessionID, prompt, "") +type queuedMoveTaskPrompt struct { + identity messagequeue.QueueSessionIdentity + entryID string } -func (h *Handlers) queueMoveTaskPromptWithMoveID(ctx context.Context, taskID, sessionID, prompt, moveID string) error { - if h.messageQueue == nil { - return fmt.Errorf("message queue is unavailable") +func queueIdentityForSession(session *models.TaskSession) messagequeue.QueueSessionIdentity { + return messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, + SessionID: session.ID, + SessionIncarnationID: session.QueueIncarnationID, } - if sessionID == "" { - return fmt.Errorf("task has no primary session") +} + +// queueMoveTaskPrompt enqueues a handoff for one exact session incarnation and +// returns the entry identity needed to undo only that handoff. +func (h *Handlers) queueMoveTaskPrompt( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + prompt string, +) (*queuedMoveTaskPrompt, error) { + return h.queueMoveTaskPromptWithMoveID(ctx, identity, prompt, "") +} + +func (h *Handlers) queueMoveTaskPromptWithMoveID( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + prompt string, + moveID string, +) (*queuedMoveTaskPrompt, error) { + if h.messageQueue == nil { + return nil, fmt.Errorf("message queue is unavailable") + } + if identity.SessionID == "" { + return nil, fmt.Errorf("task has no primary session") } metadata := map[string]interface{}(nil) if moveID != "" { metadata = map[string]interface{}{messagequeue.MetadataDeferredMoveID: moveID} } - if queueWithMetadata, ok := h.messageQueue.(messageMetadataQueuer); ok { - if _, err := queueWithMetadata.QueueMessageWithMetadata(ctx, sessionID, taskID, prompt, "", messagequeue.QueuedByMoveTask, false, nil, metadata); err != nil { - return fmt.Errorf("queue message: %w", err) + + var ( + entry *messagequeue.QueuedMessage + err error + ) + if queue, ok := h.messageQueue.(*messagequeue.Service); ok { + entry, err = queue.QueueMessageWithMetadataForSession( + ctx, identity, prompt, "", messagequeue.QueuedByMoveTask, false, nil, metadata, + ) + if err == nil { + h.publishQueueStatusEvent(ctx, identity, queue) } - return nil + } else if queueWithMetadata, ok := h.messageQueue.(messageMetadataQueuer); ok { + entry, err = queueWithMetadata.QueueMessageWithMetadata( + ctx, identity.SessionID, identity.TaskID, prompt, "", messagequeue.QueuedByMoveTask, false, nil, metadata, + ) + } else { + entry, err = h.messageQueue.QueueMessage( + ctx, identity.SessionID, identity.TaskID, prompt, "", messagequeue.QueuedByMoveTask, false, nil, + ) } - if _, err := h.messageQueue.QueueMessage(ctx, sessionID, taskID, prompt, "", messagequeue.QueuedByMoveTask, false, nil); err != nil { - return fmt.Errorf("queue message: %w", err) + if err != nil { + return nil, fmt.Errorf("queue message: %w", err) + } + if entry == nil || entry.ID == "" { + return nil, fmt.Errorf("queue message returned no entry identity") + } + return &queuedMoveTaskPrompt{identity: identity, entryID: entry.ID}, nil +} + +func (h *Handlers) rollbackMoveTaskPrompt(ctx context.Context, handoff *queuedMoveTaskPrompt) { + if handoff == nil || h.messageQueue == nil { + return + } + if _, err := h.messageQueue.RemoveEntryForSession(ctx, handoff.identity, handoff.entryID); err != nil { + h.logger.Error("move_task: failed to roll back queued hand-off prompt", + zap.String("task_id", handoff.identity.TaskID), + zap.String("session_id", handoff.identity.SessionID), + zap.String("entry_id", handoff.entryID), + zap.Error(err)) + return + } + if queue, ok := h.messageQueue.(*messagequeue.Service); ok { + h.publishQueueStatusEvent(ctx, handoff.identity, queue) } - return nil } func (h *Handlers) handleDeleteTask(ctx context.Context, msg *ws.Message) (*ws.Message, error) { diff --git a/apps/backend/internal/mcp/handlers/handlers.go b/apps/backend/internal/mcp/handlers/handlers.go index c12bb1bc74..3a16296840 100644 --- a/apps/backend/internal/mcp/handlers/handlers.go +++ b/apps/backend/internal/mcp/handlers/handlers.go @@ -191,7 +191,7 @@ type SessionLauncher interface { // this call actually dispatched the message immediately; callers must // not report "sent" when it's false — the message is still only // queued, to be delivered later by whichever drain gets to it. - QueueAndInterruptForPeerMessage(ctx context.Context, taskID, sessionID, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) + QueueAndInterruptForPeerMessage(ctx context.Context, identity messagequeue.QueueSessionIdentity, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) // RenameSession sets the user-visible session tab label and broadcasts // the change. Used by spawn_session_kandev's optional name parameter. RenameSession(ctx context.Context, sessionID, name string) error @@ -218,14 +218,11 @@ type TaskTitleBranchRenamer interface { RenameGeneratedBranchesForTaskTitle(ctx context.Context, taskID, sessionID, title string) (orchestrator.TitleBranchRenameResult, error) } -// MessageQueuer queues a prompt message for delivery to a session on its next turn. -// TakeQueued is exposed so move_task can roll back the hand-off prompt when the -// underlying MoveTask call fails — without it, a queued "you were moved..." -// message would survive a failed move and be delivered on the next agent turn. +// MessageQueuer owns session-incarnation-bound move handoffs and pending moves. type MessageQueuer interface { QueueMessage(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []messagequeue.MessageAttachment) (*messagequeue.QueuedMessage, error) - SetPendingMove(ctx context.Context, sessionID string, move *messagequeue.PendingMove) - TakeQueued(ctx context.Context, sessionID string) (*messagequeue.QueuedMessage, bool) + SetPendingMove(ctx context.Context, sessionID string, move *messagequeue.PendingMove) error + RemoveEntryForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity, entryID string) (*messagequeue.QueueRemovalResult, error) } // messageMetadataQueuer is an optional extension implemented by the @@ -2940,6 +2937,7 @@ const ( keyBaseBranch = "base_branch" keyCheckoutBranch = "checkout_branch" keyPosition = "position" + keyAutoMergeEnabled = "auto_merge_enabled" ) // taskMessageStatusSent is the taskMessageDispatchResult.status value used @@ -2995,6 +2993,8 @@ type taskMessageReviewRollback struct { type taskMessageSessionRollback struct { sessionID string + taskID string + sessionIncarnationID string state models.TaskSessionState error string completedAt *time.Time @@ -3006,6 +3006,7 @@ type taskMessageSessionRollback struct { } type taskMessageQueueRollback struct { + identity messagequeue.QueueSessionIdentity entries []messagequeue.QueuedMessage hadPendingMove bool pendingMove *messagequeue.PendingMove @@ -3020,10 +3021,14 @@ type taskMessageSessionRollbackRepository interface { expected models.TaskSessionState, ) (bool, error) SetSessionPrimary(ctx context.Context, sessionID string) error - DeleteTaskSession(ctx context.Context, id string) error + DeleteTaskSession(ctx context.Context, session *models.TaskSession) error UpdateSessionMetadata(ctx context.Context, sessionID string, metadata map[string]interface{}) error } +type taskMessageSessionAttachmentDeleter interface { + DeleteTaskSessionWithAttachments(ctx context.Context, session *models.TaskSession) ([]*models.TaskMessageAttachment, error) +} + var errTaskMessageRollbackSuperseded = errors.New("task message rollback superseded by coordinator cancellation") type taskMessageExecutorReader interface { @@ -3062,12 +3067,21 @@ func (r *taskMessageReviewRollback) captureQueues(ctx context.Context, queue *me } queues := make(map[string]taskMessageQueueRollback, len(r.sessions)) for _, session := range r.sessions { - entries, move, err := queue.SnapshotSession(ctx, session.sessionID) + identity, err := queue.ResolveSessionIdentity(ctx, session.taskID, session.sessionID) + if err != nil { + return fmt.Errorf("resolve session queue identity: %w", err) + } + if session.sessionIncarnationID != "" && + identity.SessionIncarnationID != session.sessionIncarnationID { + return messagequeue.ErrSessionIdentityMismatch + } + entries, move, err := queue.SnapshotSessionForIdentity(ctx, identity) if err != nil { return err } snapshot := taskMessageQueueRollback{ - entries: cloneTaskMessageQueuedMessages(entries), + identity: identity, + entries: cloneTaskMessageQueuedMessages(entries), } if move != nil { snapshot.hadPendingMove = true @@ -3094,6 +3108,8 @@ func captureTaskMessageSession(session *models.TaskSession) taskMessageSessionRo } snapshot := taskMessageSessionRollback{ sessionID: session.ID, + taskID: session.TaskID, + sessionIncarnationID: session.QueueIncarnationID, state: session.State, error: session.ErrorMessage, completedAt: completedAt, @@ -3288,10 +3304,23 @@ func (h *Handlers) queueTaskMessage(ctx context.Context, taskID string, session if queue == nil { return taskMessageDispatchResult{}, errors.New("message queue not available") } - queued, err := queue.QueueMessageWithMetadata(ctx, session.ID, taskID, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata) + identity, err := queue.ResolveSessionIdentity(ctx, taskID, session.ID) + if err != nil { + return taskMessageDispatchResult{}, fmt.Errorf("resolve queue session identity: %w", err) + } + if session.QueueIncarnationID != "" && + identity.SessionIncarnationID != session.QueueIncarnationID { + return taskMessageDispatchResult{}, messagequeue.ErrSessionIdentityMismatch + } + queued, err := queue.QueueMessageWithMetadataForSession( + ctx, identity, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata, + ) if err != nil { if errors.Is(err, messagequeue.ErrQueueFull) { - status := queue.GetStatus(ctx, session.ID) + status, statusErr := queue.Snapshot(ctx, identity) + if statusErr != nil { + return taskMessageDispatchResult{}, fmt.Errorf("read full queue status: %w", statusErr) + } return taskMessageDispatchResult{}, &queueFullDispatchError{ sessionID: session.ID, queueSize: status.Count, @@ -3301,7 +3330,7 @@ func (h *Handlers) queueTaskMessage(ctx context.Context, taskID string, session } return taskMessageDispatchResult{}, fmt.Errorf("failed to queue message: %w", err) } - h.publishQueueStatusEvent(ctx, session.ID, queue) + h.publishQueueStatusEvent(ctx, identity, queue) return taskMessageDispatchResult{status: taskMessageStatusQueued, sessionID: session.ID, queuedEntryID: queued.ID}, nil } @@ -3340,11 +3369,25 @@ func (h *Handlers) queueTaskMessage(ctx context.Context, taskID string, session // "queued" status avoids inviting that duplicate. The failure is still // logged server-side for operators. func (h *Handlers) queueThenInterruptTaskMessage(ctx context.Context, taskID string, session *models.TaskSession, prompt string, metadata map[string]interface{}) (taskMessageDispatchResult, error) { - queued, dispatched, err := h.sessionLauncher.QueueAndInterruptForPeerMessage(ctx, taskID, session.ID, prompt, metadata) + queue := h.sessionLauncher.GetMessageQueue() + if queue == nil { + return taskMessageDispatchResult{}, errors.New("message queue not available") + } + identity, err := queue.ResolveSessionIdentity(ctx, taskID, session.ID) + if err != nil { + return taskMessageDispatchResult{}, fmt.Errorf("resolve queue session identity: %w", err) + } + if session.QueueIncarnationID != "" && + identity.SessionIncarnationID != session.QueueIncarnationID { + return taskMessageDispatchResult{}, messagequeue.ErrSessionIdentityMismatch + } + queued, dispatched, err := h.sessionLauncher.QueueAndInterruptForPeerMessage(ctx, identity, prompt, metadata) if err != nil { if errors.Is(err, messagequeue.ErrQueueFull) { - queue := h.sessionLauncher.GetMessageQueue() - status := queue.GetStatus(ctx, session.ID) + status, statusErr := queue.Snapshot(ctx, identity) + if statusErr != nil { + return taskMessageDispatchResult{}, fmt.Errorf("read full queue status: %w", statusErr) + } return taskMessageDispatchResult{}, &queueFullDispatchError{ sessionID: session.ID, queueSize: status.Count, @@ -3522,7 +3565,25 @@ func (h *Handlers) restoreSelectedTaskMessageSession(ctx context.Context, repo t return err } } - return repo.DeleteTaskSession(ctx, rollback.selectedID) + return h.deleteTaskMessageRollbackSession(ctx, repo, selected) +} + +func (h *Handlers) deleteTaskMessageRollbackSession( + ctx context.Context, + repo taskMessageSessionRollbackRepository, + session *models.TaskSession, +) error { + if deleter, ok := repo.(taskMessageSessionAttachmentDeleter); ok { + attachments, err := deleter.DeleteTaskSessionWithAttachments(ctx, session) + if err != nil { + return err + } + if attachmentSvc := h.taskSvc.AttachmentService(); attachmentSvc != nil { + attachmentSvc.RemoveBytes(attachments) + } + return nil + } + return repo.DeleteTaskSession(ctx, session) } func (r taskMessageReviewRollback) primarySessionID() string { @@ -3555,7 +3616,7 @@ func (h *Handlers) restoreTaskMessageQueue(ctx context.Context, queue *messagequ if snapshot.hadPendingMove { pendingMove = cloneTaskMessagePendingMove(snapshot.pendingMove) } - return queue.RestoreSession(ctx, sessionID, cloneTaskMessageQueuedMessages(snapshot.entries), pendingMove) + return queue.RestoreSessionForIdentity(ctx, snapshot.identity, cloneTaskMessageQueuedMessages(snapshot.entries), pendingMove) } func (h *Handlers) restoreTaskMessageQueueOwner(ctx context.Context, selectedID, primaryID string) error { @@ -3563,9 +3624,38 @@ func (h *Handlers) restoreTaskMessageQueueOwner(ctx context.Context, selectedID, if queue == nil { return nil } - return queue.TransferSession(ctx, selectedID, primaryID) + selected, err := h.sessionRepo.GetTaskSession(ctx, selectedID) + if err != nil { + return fmt.Errorf("load selected queue owner: %w", err) + } + if selected == nil { + return fmt.Errorf("load selected queue owner %q: not found", selectedID) + } + primary, err := h.sessionRepo.GetTaskSession(ctx, primaryID) + if err != nil { + return fmt.Errorf("load primary queue owner: %w", err) + } + if primary == nil { + return fmt.Errorf("load primary queue owner %q: not found", primaryID) + } + selectedIdentity, err := queue.ResolveSessionIdentity(ctx, selected.TaskID, selected.ID) + if err != nil { + return fmt.Errorf("resolve selected queue owner: %w", err) + } + if selected.QueueIncarnationID != "" && + selectedIdentity.SessionIncarnationID != selected.QueueIncarnationID { + return messagequeue.ErrSessionIdentityMismatch + } + primaryIdentity, err := queue.ResolveSessionIdentity(ctx, primary.TaskID, primary.ID) + if err != nil { + return fmt.Errorf("resolve primary queue owner: %w", err) + } + if primary.QueueIncarnationID != "" && + primaryIdentity.SessionIncarnationID != primary.QueueIncarnationID { + return messagequeue.ErrSessionIdentityMismatch + } + return queue.TransferSessionIdentities(ctx, selectedIdentity, primaryIdentity) } - func restoreTaskMessageSessionSnapshot(ctx context.Context, repo taskMessageSessionRollbackRepository, rollback taskMessageSessionRollback) error { session, err := repo.GetTaskSession(ctx, rollback.sessionID) if err != nil { @@ -3713,22 +3803,44 @@ func (h *Handlers) promptWithAutoResume(ctx context.Context, taskID, sessionID, // publishQueueStatusEvent fires a queue.status_changed event so the frontend // can update the queue indicator. -func (h *Handlers) publishQueueStatusEvent(ctx context.Context, sessionID string, queue *messagequeue.Service) { +func (h *Handlers) publishQueueStatusEvent( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queue *messagequeue.Service, +) { if h.eventBus == nil { return } - status := queue.GetStatus(ctx, sessionID) + status, err := queue.Snapshot(ctx, identity) + if err != nil { + return + } + eventData := map[string]interface{}{ + keyTaskID: status.TaskID, + keySessionID: status.SessionID, + "session_incarnation_id": status.SessionIncarnationID, + "status_epoch": status.StatusEpoch, + "status_generation": status.StatusGeneration, + "entries": status.Entries, + "count": status.Count, + "max": status.Max, + "auto_run": status.AutoRun, + "merge_enabled": status.MergeEnabled, + "auto_merge_available": status.AutoMergeAvailable, + } + if status.AutoMergeEnabled != nil { + eventData[keyAutoMergeEnabled] = *status.AutoMergeEnabled + } + if status.AutoMergeSource != "" { + eventData["auto_merge_source"] = status.AutoMergeSource + } + if status.AutoMergeRevision != nil { + eventData["auto_merge_revision"] = *status.AutoMergeRevision + } _ = h.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( events.MessageQueueStatusChanged, "mcp-handlers", - map[string]interface{}{ - "session_id": sessionID, - "entries": status.Entries, - "count": status.Count, - "max": status.Max, - "auto_run": status.AutoRun, - "merge_enabled": status.MergeEnabled, - }, + eventData, )) } diff --git a/apps/backend/internal/mcp/handlers/handlers_test.go b/apps/backend/internal/mcp/handlers/handlers_test.go index 551eba96c9..7b0f64a610 100644 --- a/apps/backend/internal/mcp/handlers/handlers_test.go +++ b/apps/backend/internal/mcp/handlers/handlers_test.go @@ -2192,7 +2192,7 @@ func (m *mockSessionLauncher) GetMessageQueue() *messagequeue.Service { return n // dispatch with a fake queued entry; tests exercising other outcomes // (failure, not-dispatched) use fakeOrchestrator in message_task_test.go // instead of this generic stub. -func (m *mockSessionLauncher) QueueAndInterruptForPeerMessage(context.Context, string, string, string, map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { +func (m *mockSessionLauncher) QueueAndInterruptForPeerMessage(context.Context, messagequeue.QueueSessionIdentity, string, map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { return &messagequeue.QueuedMessage{ID: "mock-entry"}, true, nil } diff --git a/apps/backend/internal/mcp/handlers/message_target_session_test.go b/apps/backend/internal/mcp/handlers/message_target_session_test.go index b704ef37b3..1361c59dfc 100644 --- a/apps/backend/internal/mcp/handlers/message_target_session_test.go +++ b/apps/backend/internal/mcp/handlers/message_target_session_test.go @@ -77,7 +77,7 @@ func TestHandleMessageTask_CancelledPrimaryFallsBackToTheLiveSession(t *testing. live := addSession(t, repo, target.ID, "sess-live", models.TaskSessionStateRunning, 1) cancelSession(t, repo, primary.ID) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "keep going", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -99,7 +99,7 @@ func TestHandleMessageTask_LivePrimaryKeepsPriorityOverNewerSessions(t *testing. sender, target, primary := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) newer := addSession(t, repo, target.ID, "sess-newer", models.TaskSessionStateRunning, 1) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "keep going", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -125,7 +125,7 @@ func TestHandleMessageTask_AllSessionsTerminalNamesTheRecoveryTool(t *testing.T) sender, target, primary := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) cancelSession(t, repo, primary.ID) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "keep going", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -148,7 +148,7 @@ func TestHandleMessageTask_ExplicitTerminalSessionNamesTheRecoveryTool(t *testin sender, target, primary := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) cancelSession(t, repo, primary.ID) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) payload := senderPayload(target.ID, "restart please", sender.ID) payload["session_id"] = primary.ID msg := makeWSMessage(t, ws.ActionMCPMessageTask, payload) @@ -169,7 +169,7 @@ func TestHandleMessageTask_ExplicitTerminalSessionIsNotRedirected(t *testing.T) live := addSession(t, repo, target.ID, "sess-live", models.TaskSessionStateRunning, 1) cancelSession(t, repo, primary.ID) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) payload := senderPayload(target.ID, "restart please", sender.ID) payload["session_id"] = primary.ID msg := makeWSMessage(t, ws.ActionMCPMessageTask, payload) @@ -198,7 +198,7 @@ func TestHandleMessageTask_NoPrimarySessionUsesTheLiveSpawnedSession(t *testing. require.NoError(t, err) spawned := addSession(t, repo, targetResult.Task.ID, "sess-spawned", models.TaskSessionStateRunning, 1) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(targetResult.Task.ID, "hello", senderResult.Task.ID)) resp, err := h.handleMessageTask(ctx, msg) @@ -223,7 +223,7 @@ func TestHandleMessageTask_NoSessionAtAllNamesTheRecoveryTool(t *testing.T) { }) require.NoError(t, err) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(targetResult.Task.ID, "hello", senderResult.Task.ID)) resp, err := h.handleMessageTask(ctx, msg) @@ -251,7 +251,7 @@ func TestHandleMessageTask_IdleFallbackSessionIsNotReroutedToTheCancelledPrimary })) cancelSession(t, repo, primary.ID) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "pick this up", sender.ID)) resp, err := h.handleMessageTask(ctx, msg) @@ -291,7 +291,7 @@ func TestHandleMessageTask_CompletedSiblingDoesNotShadowTheLiveFallback(t *testi require.Less(t, indexOfSession(t, sessions, retired.ID), indexOfSession(t, sessions, running.ID), "fixture must present the completed session ahead of the running one") - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "keep going", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) diff --git a/apps/backend/internal/mcp/handlers/message_task_test.go b/apps/backend/internal/mcp/handlers/message_task_test.go index 068a09c398..69e7c0bcf5 100644 --- a/apps/backend/internal/mcp/handlers/message_task_test.go +++ b/apps/backend/internal/mcp/handlers/message_task_test.go @@ -74,6 +74,16 @@ func (r *failingQueueSnapshotRepository) ListBySession( return r.Repository.ListBySession(ctx, sessionID) } +func (r *failingQueueSnapshotRepository) Snapshot( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, +) (messagequeue.RepositorySnapshot, error) { + if identity.SessionID == r.failSessionID { + return messagequeue.RepositorySnapshot{}, errors.New("snapshot failed") + } + return r.Repository.Snapshot(ctx, identity) +} + // interruptCall records one InterruptForPeerMessage invocation. type interruptCall struct { taskID, sessionID, entryID string @@ -175,14 +185,14 @@ func (f *fakeOrchestrator) GetMessageQueue() *messagequeue.Service { return f.qu // interruptSkippedNoError simulates a busy/failed-take outcome — real // interrupt/drain behavior is exercised by the orchestrator-level // QueueAndInterruptForPeerMessage tests, not this fake. -func (f *fakeOrchestrator) QueueAndInterruptForPeerMessage(ctx context.Context, taskID, sessionID, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { - queued, err := f.queue.QueueMessageWithMetadata(ctx, sessionID, taskID, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata) +func (f *fakeOrchestrator) QueueAndInterruptForPeerMessage(ctx context.Context, identity messagequeue.QueueSessionIdentity, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { + queued, err := f.queue.QueueMessageWithMetadataForSession(ctx, identity, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata) if err != nil { return nil, false, err } f.mu.Lock() - f.interruptCalls = append(f.interruptCalls, interruptCall{taskID: taskID, sessionID: sessionID, entryID: queued.ID}) + f.interruptCalls = append(f.interruptCalls, interruptCall{taskID: identity.TaskID, sessionID: identity.SessionID, entryID: queued.ID}) f.mu.Unlock() if f.interruptErr != nil { @@ -216,11 +226,28 @@ func (panicPromptReferenceResolver) ResolvePromptReferences(context.Context, str func newMessageTaskHandler(t *testing.T, svc *service.Service, taskRepo ...TaskRepository) (*Handlers, *fakeOrchestrator) { t.Helper() log := testLogger(t) - orch := &fakeOrchestrator{queue: messagequeue.NewServiceMemory(log)} var repo TaskRepository if len(taskRepo) > 0 { repo = taskRepo[0] } + queueRepo := messagequeue.NewMemoryRepository() + if sessionRepo, ok := repo.(SessionRepository); ok { + queueRepo = messagequeue.NewMemoryRepositoryWithAuthority( + func(ctx context.Context, taskID, sessionID string) (messagequeue.QueueSessionIdentity, error) { + session, err := sessionRepo.GetTaskSession(ctx, sessionID) + if err != nil { + return messagequeue.QueueSessionIdentity{}, err + } + if session == nil || session.TaskID != taskID || session.QueueIncarnationID == "" { + return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch + } + return messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: session.QueueIncarnationID, + }, nil + }, + ) + } + orch := &fakeOrchestrator{queue: messagequeue.NewService(queueRepo, messagequeue.DefaultMaxPerSession, log)} h := &Handlers{ taskSvc: svc, taskRepo: repo, @@ -429,7 +456,7 @@ func TestHandleMessageTask_SiblingSession_Queues(t *testing.T) { require.NoError(t, repo.CreateTaskSession(context.Background(), sibling)) require.NoError(t, repo.UpdateTaskSessionState(context.Background(), sibling.ID, models.TaskSessionStateRunning, "")) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) payload := map[string]interface{}{ "task_id": target.ID, @@ -492,7 +519,7 @@ func TestHandleMessageTask_IdleSiblingSession_PinsTargetSession(t *testing.T) { AgentExecutionID: "exec-" + sibling.ID, })) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) payload := map[string]interface{}{ "task_id": target.ID, @@ -528,7 +555,7 @@ func TestHandleMessageTask_IdleSiblingSession_PinsTargetSession(t *testing.T) { func TestHandleMessageTask_QueuesPromptWhenOnTurnStartQueuesTask(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, session := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.turnStartResult = orchestrator.ProcessOnTurnStartResult{Queued: true} payload := senderPayload(target.ID, "wait for admission", sender.ID) @@ -553,7 +580,7 @@ func TestHandleMessageTask_SessionIDWrongTask_Rejected(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) // sess belongs to target — sending to sender's task with that session must fail. payload := senderPayload(sender.ID, "hello", target.ID) @@ -601,7 +628,7 @@ func TestHandleMessageTask_RunningSession_Queues(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "follow-up message", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -650,7 +677,7 @@ func TestHandleMessageTask_ParentToChildRunningSession_Interrupts(t *testing.T) svc, repo := newTestTaskService(t) parent, child, sess := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(child.ID, "stop and pivot to X", parent.ID, "interrupt")) resp, err := h.handleMessageTask(context.Background(), msg) @@ -692,7 +719,7 @@ func TestHandleMessageTask_ParentToChildRunningSession_InterruptFailure_KeepsMes svc, repo := newTestTaskService(t) parent, child, sess := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.interruptErr = errors.New("cancel agent: agent manager unreachable") msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(child.ID, "stop now", parent.ID, "interrupt")) @@ -720,7 +747,7 @@ func TestHandleMessageTask_ParentToChildRunningSession_InterruptSkipped_KeepsQue svc, repo := newTestTaskService(t) parent, child, sess := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.interruptSkippedNoError = true msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(child.ID, "stop now", parent.ID, "interrupt")) @@ -756,7 +783,7 @@ func TestHandleMessageTask_ParentToChildRunningSession_OmittedDeliveryMode_DoesN svc, repo := newTestTaskService(t) parent, child, sess := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(child.ID, "fyi, no rush", parent.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -782,7 +809,7 @@ func TestHandleMessageTask_ParentToChildRunningSession_ExplicitQueued_DoesNotInt svc, repo := newTestTaskService(t) parent, child, sess := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(child.ID, "fyi, no rush", parent.ID, "queued")) resp, err := h.handleMessageTask(context.Background(), msg) @@ -811,7 +838,7 @@ func TestHandleMessageTask_NonParentSender_InterruptRequest_HardRejected(t *test svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(target.ID, "stop now", sender.ID, "interrupt")) resp, err := h.handleMessageTask(context.Background(), msg) @@ -838,7 +865,7 @@ func TestHandleMessageTask_InvalidDeliveryMode_Rejected(t *testing.T) { svc, repo := newTestTaskService(t) parent, child, _ := seedChildTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayloadWithMode(child.ID, "stop now", parent.ID, "immediately")) resp, err := h.handleMessageTask(context.Background(), msg) @@ -850,7 +877,7 @@ func TestHandleMessageTask_AppendsPromptReferenceExpansions(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) h.SetPromptReferenceResolver(fakePromptReferenceResolver{ expansions: []promptservice.PromptReferenceExpansion{ {Name: "improve-harness", Content: "Review this session for durable harness improvements."}, @@ -876,7 +903,7 @@ func TestHandleMessageTask_PromptResolverError_FallsBackToOriginal(t *testing.T) svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) h.SetPromptReferenceResolver(fakePromptReferenceResolver{err: errors.New("db error")}) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "plain @missing text", sender.ID)) @@ -895,7 +922,7 @@ func TestHandleMessageTask_NoPromptReferencesSkipsResolver(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) h.SetPromptReferenceResolver(panicPromptReferenceResolver{}) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "plain text", sender.ID)) @@ -922,7 +949,7 @@ func TestHandleMessageTask_QueueFull_ReturnsStructuredError(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateRunning) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) // Saturate the receiver's queue. for i := 0; i < messagequeue.DefaultMaxPerSession; i++ { @@ -953,7 +980,7 @@ func TestHandleMessageTask_WaitingForInput_PromptsAgent(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "next instruction", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -999,7 +1026,7 @@ func TestHandleMessageTask_WaitingForInput_FiresTurnStart(t *testing.T) { task.WorkflowStepID = "step-review" require.NoError(t, repo.UpdateTask(ctx, task)) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, taskID, sessionID string) error { assert.Equal(t, target.ID, taskID) assert.Equal(t, sess.ID, sessionID) @@ -1041,7 +1068,7 @@ func TestHandleMessageTask_KanbanRunnerTransitionsReviewToInProgress(t *testing. task.AssigneeAgentProfileID = "kanban-runner" require.NoError(t, repo.UpdateTask(ctx, task)) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, taskID, sessionID string) error { assert.Equal(t, target.ID, taskID) assert.Equal(t, sess.ID, sessionID) @@ -1077,7 +1104,7 @@ func TestHandleMessageTask_WaitingForInput_UsesSessionSelectedByTurnStart(t *tes } require.NoError(t, repo.CreateTaskSession(ctx, replacement)) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, _, _ string) error { oldSession, err := svc.GetTaskSession(ctx, sess.ID) require.NoError(t, err) @@ -1155,7 +1182,7 @@ func TestHandleMessageTask_CompletedSessionWithoutSwitch_PromptsSameSession(t *t svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateCompleted) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "follow up completed", sender.ID)) resp, err := h.handleMessageTask(ctx, msg) @@ -1192,7 +1219,7 @@ func TestHandleMessageTask_CompletedSession_UsesSessionSelectedByTurnStart(t *te } require.NoError(t, repo.CreateTaskSession(ctx, replacement)) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, _, _ string) error { oldSession, err := svc.GetTaskSession(ctx, sess.ID) require.NoError(t, err) @@ -1232,7 +1259,7 @@ func TestHandleMessageTask_WaitingForInputCompletedWithoutPrimarySwitchRejects(t svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, _, _ string) error { oldSession, err := svc.GetTaskSession(ctx, sess.ID) require.NoError(t, err) @@ -1261,7 +1288,7 @@ func TestHandleMessageTask_CreatedSessionStartsAfterTurnStartChangesState(t *tes svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateCreated) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.onTurnStart = func(ctx context.Context, _, sessionID string) error { require.Equal(t, sess.ID, sessionID) return repo.UpdateTaskSessionState(ctx, sess.ID, models.TaskSessionStateWaitingForInput, "") @@ -1415,6 +1442,7 @@ func TestTaskMessageReviewRollbackPreservesReservedLifecycleQueueEntry(t *testin rollback := taskMessageReviewRollback{ changed: true, sessions: []taskMessageSessionRollback{{ + taskID: target.ID, sessionID: sess.ID, }}, } @@ -1442,8 +1470,8 @@ func TestTaskMessageReviewRollbackCaptureQueuesIsAtomic(t *testing.T) { rollback := taskMessageReviewRollback{ changed: true, sessions: []taskMessageSessionRollback{ - {sessionID: "session-1"}, - {sessionID: "session-2"}, + {taskID: "task", sessionID: "session-1"}, + {taskID: "task", sessionID: "session-2"}, }, queues: original, } @@ -1501,12 +1529,12 @@ func TestHandleMessageTask_DispatchErrorAfterSessionSwitchRestoresReviewSession( ) require.NoError(t, err) require.True(t, accepted) - orch.queue.SetPendingMove(ctx, sess.ID, &messagequeue.PendingMove{ + require.NoError(t, orch.queue.SetPendingMove(ctx, sess.ID, &messagequeue.PendingMove{ TaskID: target.ID, WorkflowID: "workflow-1", WorkflowStepID: "step-review", Position: 2, - }) + })) replacementID := "sess-2" orch.onTurnStart = func(ctx context.Context, taskID, _ string) error { updatedTask, err := svc.GetTask(ctx, taskID) @@ -1614,12 +1642,12 @@ func TestHandleMessageTask_DispatchErrorAfterExistingSessionSwitchRestoresQueues require.NoError(t, err) _, err = orch.queue.QueueMessageWithMetadata(ctx, replacementID, target.ID, "replacement queued", "", "user", false, nil, nil) require.NoError(t, err) - orch.queue.SetPendingMove(ctx, replacementID, &messagequeue.PendingMove{ + require.NoError(t, orch.queue.SetPendingMove(ctx, replacementID, &messagequeue.PendingMove{ TaskID: target.ID, WorkflowID: "workflow-1", WorkflowStepID: "replacement-step", Position: 5, - }) + })) orch.onTurnStart = func(ctx context.Context, _, _ string) error { oldSession, err := svc.GetTaskSession(ctx, sess.ID) @@ -1871,7 +1899,7 @@ func TestHandleMessageTask_PromptFailsWithExecutionNotFound_AutoResumes(t *testi svc, repo := newTestTaskService(t) sender, target, _ := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) orch.promptErrFirst = executor.ErrExecutionNotFound msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "retry me", sender.ID)) @@ -1888,7 +1916,7 @@ func TestHandleMessageTask_CreatedSession_StartsAgent(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateCreated) - h, orch := newMessageTaskHandler(t, svc) + h, orch := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "kick off the work", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -1925,7 +1953,7 @@ func TestHandleMessageTask_FailedSession_Rejects(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, _ := seedTaskWithSession(t, svc, repo, models.TaskSessionStateFailed) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "hello", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -1937,7 +1965,7 @@ func TestHandleMessageTask_CancelledSession_Rejects(t *testing.T) { svc, repo := newTestTaskService(t) sender, target, _ := seedTaskWithSession(t, svc, repo, models.TaskSessionStateCancelled) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "hello", sender.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -1965,7 +1993,7 @@ func TestHandleMessageTask_NoPrimarySession_Rejects(t *testing.T) { sender := senderResult.Task require.NoError(t, err) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "hello", sender.ID)) resp, err := h.handleMessageTask(ctx, msg) @@ -1994,7 +2022,7 @@ func TestHandleMessageTask_NonexistentTask_ReportsTaskNotFound(t *testing.T) { sender := senderResult.Task require.NoError(t, err) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload("00000000-0000-0000-0000-000000000000", "hello", sender.ID)) @@ -2014,7 +2042,7 @@ func TestHandleMessageTask_MissingSenderTaskID_Rejects(t *testing.T) { svc, repo := newTestTaskService(t) _, target, _ := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, map[string]interface{}{ "task_id": target.ID, @@ -2030,7 +2058,7 @@ func TestHandleMessageTask_SelfMessage_Rejects(t *testing.T) { svc, repo := newTestTaskService(t) _, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "hello", target.ID)) resp, err := h.handleMessageTask(context.Background(), msg) @@ -2047,7 +2075,7 @@ func TestHandleMessageTask_UnknownSenderTask_Rejects(t *testing.T) { svc, repo := newTestTaskService(t) _, target, sess := seedTaskWithSession(t, svc, repo, models.TaskSessionStateWaitingForInput) - h, _ := newMessageTaskHandler(t, svc) + h, _ := newMessageTaskHandler(t, svc, repo) msg := makeWSMessage(t, ws.ActionMCPMessageTask, senderPayload(target.ID, "hello", "00000000-0000-0000-0000-000000000000")) resp, err := h.handleMessageTask(context.Background(), msg) diff --git a/apps/backend/internal/mcp/handlers/queue_status_event_test.go b/apps/backend/internal/mcp/handlers/queue_status_event_test.go index 38e6c6a05d..5e86aaa74c 100644 --- a/apps/backend/internal/mcp/handlers/queue_status_event_test.go +++ b/apps/backend/internal/mcp/handlers/queue_status_event_test.go @@ -16,11 +16,18 @@ func TestPublishQueueStatusEventIncludesQueuePolicy(t *testing.T) { eventBus := &mcpRecordingEventBus{} handlers := &Handlers{eventBus: eventBus} - handlers.publishQueueStatusEvent(ctx, "session-policy", queue) + identity, err := queue.ResolveSessionIdentity(ctx, "task-policy", "session-policy") + require.NoError(t, err) + handlers.publishQueueStatusEvent(ctx, identity, queue) require.Len(t, eventBus.events, 1) data, ok := eventBus.events[0].Data.(map[string]interface{}) require.True(t, ok) require.Equal(t, false, data["auto_run"]) require.Equal(t, false, data["merge_enabled"]) + require.Equal(t, "task-policy", data["task_id"]) + require.Equal(t, identity.SessionIncarnationID, data["session_incarnation_id"]) + require.NotEmpty(t, data["status_epoch"]) + require.NotEqual(t, identity.SessionIncarnationID, data["status_epoch"]) + require.Equal(t, true, data["auto_merge_available"]) } diff --git a/apps/backend/internal/orchestrator/automation_run_liveness_test.go b/apps/backend/internal/orchestrator/automation_run_liveness_test.go index 1669f2c972..deb3a7659d 100644 --- a/apps/backend/internal/orchestrator/automation_run_liveness_test.go +++ b/apps/backend/internal/orchestrator/automation_run_liveness_test.go @@ -108,7 +108,7 @@ func TestAutomationRunLiveNormalizesMissingTaskAndSession(t *testing.T) { } } if tt.deleteSession { - if err := repo.DeleteTaskSession(ctx, session.ID); err != nil { + if err := repo.DeleteTaskSession(ctx, session); err != nil { t.Fatal(err) } } diff --git a/apps/backend/internal/orchestrator/ci_automation_dispatch.go b/apps/backend/internal/orchestrator/ci_automation_dispatch.go index 5425f5f2f6..b4fca6c7b3 100644 --- a/apps/backend/internal/orchestrator/ci_automation_dispatch.go +++ b/apps/backend/internal/orchestrator/ci_automation_dispatch.go @@ -30,6 +30,7 @@ const ( type ciAutomationDispatchResult struct { kind ciAutomationDispatchKind + identity messagequeue.QueueSessionIdentity queueEntryID string turnID string } @@ -85,19 +86,24 @@ func (s *Service) dispatchCIAutomationPromptToIdleSession( if err != nil { return ciAutomationDispatchResult{}, err } + identity := result.identity if replaced { - outcome := s.drainQueuedMessageForPromptableSessionOutcome(ctx, session.ID) - if outcome == queueDrainPaused { - return result, nil + dispatched, drainErr := s.drainQueuedMessageForPromptableSessionForIdentity(ctx, identity) + if drainErr != nil { + return ciAutomationDispatchResult{}, drainErr } - if outcome != queueDrainDispatched { - return ciAutomationDispatchResult{}, fmt.Errorf("failed to dispatch replaced CI automation prompt") + if !dispatched { + return result, nil } return result, nil } if !params.AllowNewRound { return ciAutomationDispatchResult{}, errCIAutoFixRoundCapReached } + identity, err = s.resolveQueueIdentityForSession(ctx, session) + if err != nil { + return ciAutomationDispatchResult{}, err + } turnID, recorded := s.recordCIAutomationUserMessage(ctx, session.TaskID, session.ID, params.ChatPrompt, params.Metadata) if !recorded { return ciAutomationDispatchResult{}, fmt.Errorf("failed to record CI automation user message") @@ -108,6 +114,7 @@ func (s *Service) dispatchCIAutomationPromptToIdleSession( } } promptResult, promptErr := s.promptTask(ctx, session.TaskID, session.ID, params.ChatPrompt, "", false, nil, true, promptTaskOptions{ + expectedSessionIdentity: &identity, onAccepted: func(acceptedTurnID string) { if params.OnAccepted != nil { params.OnAccepted(acceptedTurnID, "") @@ -123,7 +130,7 @@ func (s *Service) dispatchCIAutomationPromptToIdleSession( if promptResult != nil && promptResult.TurnID != "" { turnID = promptResult.TurnID } - return ciAutomationDispatchResult{kind: ciAutomationDispatchDirect, turnID: turnID}, nil + return ciAutomationDispatchResult{kind: ciAutomationDispatchDirect, identity: identity, turnID: turnID}, nil } func (s *Service) replacePendingCIAutomationPrompt( @@ -148,18 +155,20 @@ func (s *Service) queueOrReplaceCIAutomationPrompt( if s.messageQueue == nil { return ciAutomationDispatchResult{}, fmt.Errorf("message queue is not configured") } + identity, err := s.resolveQueueIdentityForSession(ctx, session) + if err != nil { + return ciAutomationDispatchResult{}, err + } var queued *messagequeue.QueuedMessage var replaced, accepted bool - var err error if params.OnQueued == nil { - queued, replaced, err = s.messageQueue.QueueMessageWithCoalesceKey( - ctx, session.ID, session.TaskID, params.ChatPrompt, "", messagequeue.QueuedByWorkflow, + queued, replaced, accepted, err = s.messageQueue.QueueLifecycleMessageWithCoalesceKeyForSession( + ctx, identity, params.ChatPrompt, "", messagequeue.QueuedByWorkflow, false, nil, params.Metadata, params.CoalesceKey, allowInsert, ) - accepted = err == nil } else { - queued, replaced, accepted, err = s.messageQueue.QueueLifecycleMessageWithCoalesceKeyAfterInsert( - ctx, session.ID, session.TaskID, params.ChatPrompt, "", messagequeue.QueuedByWorkflow, + queued, replaced, accepted, err = s.messageQueue.QueueLifecycleMessageWithCoalesceKeyForSessionAfterInsert( + ctx, identity, params.ChatPrompt, "", messagequeue.QueuedByWorkflow, false, nil, params.Metadata, params.CoalesceKey, allowInsert, func(_ context.Context, queued *messagequeue.QueuedMessage, replaced bool) error { return params.OnQueued(queued.ID, replaced) @@ -178,11 +187,11 @@ func (s *Service) queueOrReplaceCIAutomationPrompt( if queued == nil { return ciAutomationDispatchResult{}, fmt.Errorf("CI automation queue returned no entry") } - s.publishQueueStatusEvent(ctx, session.ID) + s.publishQueueStatusEventForIdentity(ctx, identity) if replaced { - return ciAutomationDispatchResult{kind: ciAutomationDispatchQueuedReplace, queueEntryID: queued.ID}, nil + return ciAutomationDispatchResult{kind: ciAutomationDispatchQueuedReplace, identity: identity, queueEntryID: queued.ID}, nil } - return ciAutomationDispatchResult{kind: ciAutomationDispatchQueuedInsert, queueEntryID: queued.ID}, nil + return ciAutomationDispatchResult{kind: ciAutomationDispatchQueuedInsert, identity: identity, queueEntryID: queued.ID}, nil } // resolveAutoFixSession picks the session to receive the next auto-fix diff --git a/apps/backend/internal/orchestrator/event_handlers_agent.go b/apps/backend/internal/orchestrator/event_handlers_agent.go index 2085e72dea..26c25d259c 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent.go @@ -20,7 +20,11 @@ import ( v1 "github.com/kandev/kandev/pkg/api/v1" ) -const queueStatusMaxField = "max" +const ( + queueStatusMaxField = "max" + queueStatusScopeKey = "queue_status_scope" + queueStatusScopeTask = "task" +) type reservedPromptCallbackContextKey struct{} @@ -100,37 +104,107 @@ func (s *Service) handleAgentRunning(ctx context.Context, data watcher.AgentEven s.setSessionRunning(ctx, data.TaskID, data.SessionID, session) } -// publishQueueStatusEvent publishes a queue status changed event for the given session +func queueStatusEventData(status *messagequeue.QueueStatus) map[string]interface{} { + data := map[string]interface{}{ + metaKeyTaskID: status.TaskID, + metaKeySessionID: status.SessionID, + "session_incarnation_id": status.SessionIncarnationID, + "status_epoch": status.StatusEpoch, + "status_generation": status.StatusGeneration, + "entries": status.Entries, + "count": status.Count, + queueStatusMaxField: status.Max, + "auto_run": status.AutoRun, + "merge_enabled": status.MergeEnabled, + "auto_merge_available": status.AutoMergeAvailable, + } + if status.AutoMergeEnabled != nil { + data["auto_merge_enabled"] = *status.AutoMergeEnabled + } + if status.AutoMergeSource != "" { + data["auto_merge_source"] = status.AutoMergeSource + } + if status.AutoMergeRevision != nil { + data["auto_merge_revision"] = *status.AutoMergeRevision + } + return data +} + +func (s *Service) queueStatusSnapshot( + ctx context.Context, + sessionID string, +) (*messagequeue.QueueStatus, error) { + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return nil, err + } + identity, err := s.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID) + if err != nil { + return nil, err + } + return s.messageQueue.Snapshot(ctx, identity) +} + +func (s *Service) queueStatusSnapshotForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, +) (*messagequeue.QueueStatus, error) { + if identity.SessionIncarnationID == "" { + return s.queueStatusSnapshot(ctx, identity.SessionID) + } + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil { + return nil, err + } + if current != identity { + return nil, messagequeue.ErrSessionIdentityMismatch + } + return s.messageQueue.Snapshot(ctx, identity) +} + +// publishQueueStatusEvent publishes a queue status changed event for the given session. func (s *Service) publishQueueStatusEvent(ctx context.Context, sessionID string) { if s.eventBus == nil || s.messageQueue == nil { return } - - queueStatus := s.messageQueue.GetStatus(ctx, sessionID) - eventData := map[string]interface{}{ - metaKeySessionID: sessionID, - "entries": queueStatus.Entries, - "count": queueStatus.Count, - queueStatusMaxField: queueStatus.Max, - "auto_run": queueStatus.AutoRun, - "merge_enabled": queueStatus.MergeEnabled, - } - if taskID, err := s.SessionTaskID(ctx, sessionID); err != nil { - s.logger.Warn("resolve session task for queue status event", + queueStatus, err := s.queueStatusSnapshot(ctx, sessionID) + if err != nil { + s.logger.Warn("snapshot queue status event", zap.String("session_id", sessionID), zap.Error(err)) - } else if taskID != "" { - eventData["task_id"] = taskID + return } - s.logger.Debug("publishing queue status changed event", zap.String("session_id", sessionID), zap.Int("count", queueStatus.Count)) - _ = s.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( events.MessageQueueStatusChanged, "orchestrator", - eventData, + queueStatusEventData(queueStatus), + )) +} + +func (s *Service) publishQueueStatusEventForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, +) { + if s.eventBus == nil || s.messageQueue == nil { + return + } + queueStatus, err := s.queueStatusSnapshotForIdentity(ctx, identity) + if errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + return + } + if err != nil { + s.logger.Warn("snapshot queue status event", + zap.String("session_id", identity.SessionID), + zap.Error(err)) + return + } + _ = s.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( + events.MessageQueueStatusChanged, + "orchestrator", + queueStatusEventData(queueStatus), )) } @@ -148,16 +222,19 @@ func (s *Service) publishTaskQueueStatusEvent(ctx context.Context, taskID, sessi // leave queued_prompt_count stale on every live sidebar). ctx = context.WithoutCancel(ctx) eventData := map[string]interface{}{ - "task_id": taskID, + metaKeyTaskID: taskID, + queueStatusScopeKey: queueStatusScopeTask, } if sessionID != "" && s.messageQueue != nil { - queueStatus := s.messageQueue.GetStatus(ctx, sessionID) - eventData[metaKeySessionID] = sessionID - eventData["entries"] = queueStatus.Entries - eventData["count"] = queueStatus.Count - eventData[queueStatusMaxField] = queueStatus.Max - eventData["auto_run"] = queueStatus.AutoRun - eventData["merge_enabled"] = queueStatus.MergeEnabled + queueStatus, err := s.queueStatusSnapshot(ctx, sessionID) + if err != nil { + s.logger.Warn("snapshot task queue status event", + zap.String(metaKeyTaskID, taskID), + zap.String(metaKeySessionID, sessionID), + zap.Error(err)) + } else { + eventData = queueStatusEventData(queueStatus) + } } s.logger.Debug("publishing task queue status changed event", zap.String("task_id", taskID), @@ -173,22 +250,13 @@ func (s *Service) publishTaskQueueStatusEvent(ctx context.Context, taskID, sessi // Preserves the original Metadata (e.g. sender_task_id from message_task_kandev) // so attribution survives transient failures + retries. // -// User messages go through RequeueAtHead (FIFO-preserving) so a -// supersede→requeue cycle does not strand the original entry behind any -// new arrival — the busy-session starvation bug fix. Lifecycle messages -// keep their tail-requeue semantics (requeueLifecycleMessage) because -// they are gated by a generation token that already gives them -// priority; mixing the two paths would risk merging durable -// reservations with transient user-state. +// Same-session retries go through RequeueAtHead. Ordinary messages retain +// FIFO priority; reserved lifecycle messages release their exact durable row +// so a newer coalesced successor cannot overwrite the in-flight delivery. func (s *Service) requeueMessage(ctx context.Context, queuedMsg *messagequeue.QueuedMessage, queuedBy string) { - coalesceKey := messageCoalesceKey(queuedMsg) - if queuedMsg.QueuedBy != "" && coalesceKey != "" { + if queuedMsg.QueuedBy != "" { queuedBy = queuedMsg.QueuedBy } - if isLifecycleAutomationOrigin(queuedMsg.Metadata["origin"]) { - s.requeueLifecycleMessage(ctx, queuedMsg, queuedBy, coalesceKey) - return - } if err := s.messageQueue.RequeueAtHead(ctx, queuedMsg); err != nil { s.logger.Error("failed to requeue message at head", zap.String("session_id", queuedMsg.SessionID), @@ -206,7 +274,100 @@ func (s *Service) requeueMessage(ctx context.Context, queuedMsg *messagequeue.Qu s.publishQueueStatusEvent(ctx, queuedMsg.SessionID) } -func (s *Service) requeueLifecycleMessage(ctx context.Context, queuedMsg *messagequeue.QueuedMessage, queuedBy, coalesceKey string) bool { +func (s *Service) requeueMessageForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, + queuedBy string, +) error { + if queuedMsg.QueuedBy != "" { + queuedBy = queuedMsg.QueuedBy + } + if err := s.messageQueue.RequeueAtHeadForSession(ctx, identity, queuedMsg); err != nil { + s.logPostClaimQueueMutationFailure("requeue message at head", identity, queuedMsg, err) + return err + } + s.logger.Info("message requeued at head for exact session (FIFO preserved)", + zap.String("session_id", queuedMsg.SessionID), + zap.String("task_id", queuedMsg.TaskID), + zap.String("queue_id", queuedMsg.ID), + zap.String("queued_by", queuedBy)) + s.publishQueueStatusEventForIdentity(ctx, identity) + return nil +} + +func (s *Service) requeueMessageForReservation( + ctx context.Context, + reservation *queuedDispatchReservation, + queuedMsg *messagequeue.QueuedMessage, + queuedBy string, +) { + if reservation == nil || reservation.identity.SessionIncarnationID == "" { + s.requeueMessage(ctx, queuedMsg, queuedBy) + return + } + if err := s.requeueMessageForSession(ctx, reservation.identity, queuedMsg, queuedBy); err != nil && + errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + s.discardLifecycleReservationForReplacedSession(ctx, queuedMsg, reservation.identity) + } +} + +func (s *Service) requeueLifecycleMessageForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, + queuedBy, coalesceKey string, +) (bool, error) { + requeuedMsg, replaced, accepted, err := s.messageQueue.RequeueLifecycleMessageWithCoalesceKeyForSession( + ctx, identity, queuedMsg.Content, queuedMsg.Model, queuedBy, queuedMsg.PlanMode, + queuedMsg.Attachments, queuedMsg.Metadata, coalesceKey, true, + ) + if err != nil { + s.logPostClaimQueueMutationFailure("requeue lifecycle message", identity, queuedMsg, err) + return false, err + } + if !accepted { + return false, nil + } + s.logger.Info("lifecycle message requeued for exact session", + zap.String("session_id", identity.SessionID), + zap.String("task_id", identity.TaskID), + zap.String("old_queue_id", queuedMsg.ID), + zap.String("new_queue_id", requeuedMsg.ID), + zap.String("queued_by", queuedBy), + zap.String("coalesce_key", coalesceKey), + zap.Bool("replaced", replaced)) + s.publishQueueStatusEventForIdentity(ctx, identity) + return true, nil +} + +func (s *Service) logPostClaimQueueMutationFailure( + operation string, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, + err error, +) { + if errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + s.logger.Info("skipping stale post-claim queue mutation", + zap.String("operation", operation), + zap.String("session_id", identity.SessionID), + zap.String("task_id", identity.TaskID), + zap.String("queue_id", queuedMsg.ID)) + return + } + s.logger.Error("failed post-claim queue mutation", + zap.String("operation", operation), + zap.String("session_id", identity.SessionID), + zap.String("task_id", identity.TaskID), + zap.String("queue_id", queuedMsg.ID), + zap.Error(err)) +} + +func (s *Service) requeueLifecycleMessage( + ctx context.Context, + queuedMsg *messagequeue.QueuedMessage, + queuedBy, coalesceKey string, +) bool { requeuedMsg, replaced, accepted, err := s.messageQueue.RequeueLifecycleMessageWithCoalesceKey( ctx, queuedMsg.SessionID, queuedMsg.TaskID, queuedMsg.Content, queuedMsg.Model, queuedBy, queuedMsg.PlanMode, queuedMsg.Attachments, queuedMsg.Metadata, coalesceKey, true, @@ -397,6 +558,27 @@ func (s *Service) handleAgentBootReady(ctx context.Context, data watcher.AgentEv s.drainQueuedMessageForPromptableSession(ctx, data.SessionID) } +func (s *Service) drainQueuedBeforeWorkflowTransition( + ctx context.Context, + taskID string, + sessionID string, + session *models.TaskSession, +) bool { + if s.messageQueue == nil { + return false + } + if s.agentManager != nil && s.agentManager.IsPassthroughSession(ctx, sessionID) { + return false + } + status := s.messageQueue.GetStatus(ctx, sessionID) + if !status.AutoRun || len(status.Entries) == 0 { + return false + } + s.setSessionWaitingForInput(ctx, taskID, sessionID, session) + outcome := s.drainQueuedMessageForPromptableSessionLockedWithTaskAdmission(ctx, taskID, sessionID) + return outcome == queueDrainDispatched +} + // handleAgentReady handles turn-end ready events: the agent finished processing // a prompt and is waiting for the next input. This is the *only* event that // should evaluate workflow on_turn_complete actions — boot signals route @@ -442,9 +624,7 @@ func (s *Service) handleAgentReady(ctx context.Context, data watcher.AgentEventD // A passthrough lifecycle reservation cannot be executed while this ready // handler holds the per-session guard: executeQueuedMessage performs its // own final claim through that guard. Register the deferred dispatch before - // acquiring it, so LIFO defer ordering releases the guard first. Keeping - // execution in this handler preserves the legacy ready-event completion - // boundary while still using the normal lifecycle delivery pipeline. + // acquiring it, so LIFO defer ordering releases the guard first. var deferredLifecycleDispatch *messagequeue.QueuedMessage var deferredLifecycleReservation *queuedDispatchReservation defer func() { @@ -697,6 +877,10 @@ func (s *Service) handleAgentReady(ctx context.Context, data watcher.AgentEventD return } + if s.drainQueuedBeforeWorkflowTransition(ctx, data.TaskID, data.SessionID, session) { + return + } + // Check for workflow transition based on session's current step. // Uses the engine when available; falls back to legacy evaluation. // The ViaEngine method handles setSessionWaitingForInput internally when no transition occurs. @@ -715,53 +899,87 @@ func (s *Service) handleAgentReady(ctx context.Context, data watcher.AgentEventD // Passthrough sessions: deliver queued messages via PTY stdin instead of ACP. if s.agentManager.IsPassthroughSession(ctx, data.SessionID) { - queuedMsg, exists := s.messageQueue.ReserveQueued(ctx, data.SessionID) - if !exists { + dispatchCtx, finishDispatch := s.beginPassthroughDispatch(ctx, data.TaskID) + defer finishDispatch() + identity, err := s.messageQueue.ResolveSessionIdentity(dispatchCtx, data.TaskID, data.SessionID) + if err != nil { + return + } + queuedMsg, exists, autoRun, err := s.messageQueue. + ReserveQueuedForDeliveryWithAutoRunForSession(dispatchCtx, identity) + if err != nil || !autoRun || !exists { return } - // A durable lifecycle reservation must take the same guarded delivery - // path as an ACP session. In particular, executeQueuedMessage performs - // the final active-task/session claim, records the visible message, and - // only acknowledges the reservation after Executor.Prompt accepts it. - // Executor.Prompt itself is passthrough-aware and writes to PTY stdin. - // Ordinary entries retain the historical direct PTY behavior below. if queuedMsg.IsDurableLifecycle() { - // Reserve the durable row's dispatch token while this ready handler - // still owns the same guard used by all drains. Deferring this mark - // until after guard release lets a competing manual drain reserve the - // same durable row and begin a duplicate delivery attempt. - deferredLifecycleReservation = s.markQueuedDispatchInFlightWithSourceLocked(data.SessionID, queuedMsg.ID, queuedMsg) + deferredLifecycleReservation = s.markQueuedDispatchInFlightWithIdentityLocked(identity, queuedMsg.ID, queuedMsg) deferredLifecycleDispatch = queuedMsg return } + releaseReservation := func() { + if releaseErr := s.messageQueue.ReleaseQueuedDeliveryForSession( + context.WithoutCancel(ctx), identity, queuedMsg.ID, + ); releaseErr != nil && + !errors.Is(releaseErr, messagequeue.ErrSessionIdentityMismatch) && + !errors.Is(releaseErr, messagequeue.ErrTaskInactive) { + s.logger.Warn("failed to release passthrough queue reservation", + zap.String("session_id", data.SessionID), zap.Error(releaseErr)) + } + } + if hook := s.afterReadyLifecycleReservation; hook != nil { + hook() + } + current, resolveErr := s.messageQueue.ResolveSessionIdentity( + dispatchCtx, identity.TaskID, identity.SessionID, + ) + if dispatchCtx.Err() != nil || resolveErr != nil || current != identity { + releaseReservation() + return + } if queuedMsg.Content != "" { - var err error if preparer, ok := s.agentManager.(passthroughRunningPreparer); ok { - deferredPassthroughRunning, err = preparer.PreparePassthroughRunning(data.SessionID) - if err != nil { - s.logger.Warn("failed to prepare passthrough as running before queued prompt", - zap.String("session_id", data.SessionID), - zap.Error(err)) - return + finishRunning, prepareErr := preparer.PreparePassthroughRunning(data.SessionID) + err = prepareErr + if finishRunning != nil { + deferredPassthroughRunning = func() { + current, resolveErr := s.messageQueue.ResolveSessionIdentity( + context.WithoutCancel(ctx), identity.TaskID, identity.SessionID, + ) + if resolveErr == nil && current == identity { + finishRunning() + } + } + } + if err == nil { + err = s.writePassthroughPrompt(dispatchCtx, data.SessionID, queuedMsg.Content) } - err = s.writePassthroughPrompt(ctx, data.SessionID, queuedMsg.Content) } else { - err = s.deliverPassthroughPrompt(ctx, data.SessionID, queuedMsg.Content) + err = s.deliverPassthroughPrompt(dispatchCtx, data.SessionID, queuedMsg.Content) + } + if dispatchCtx.Err() != nil { + deferredPassthroughRunning = nil } if err != nil { s.logger.Warn("failed to deliver queued message to passthrough", - zap.String("session_id", data.SessionID), - zap.Error(err)) + zap.String("session_id", data.SessionID), zap.Error(err)) + releaseReservation() return } } + if err := s.messageQueue.AcknowledgeQueuedForSession( + context.WithoutCancel(ctx), identity, queuedMsg.ID, + ); err != nil && + !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) && + !errors.Is(err, messagequeue.ErrTaskInactive) { + s.logger.Warn("failed to acknowledge passthrough queue reservation", + zap.String("session_id", data.SessionID), zap.Error(err)) + } return } // Check for queued messages when no workflow transition occurred. Uses // the Locked variant directly: the guard above is held for this // entire function now, not just this final step. - s.drainQueuedMessageForPromptableSessionLocked(ctx, data.SessionID) + s.drainQueuedMessageForPromptableSessionLockedWithTaskAdmission(ctx, data.TaskID, data.SessionID) } const githubPRAutomationOrigin = "github_pr_automation" @@ -836,12 +1054,32 @@ func (s *Service) executeQueuedMessageWithReservation( return } + if reservation != nil && reservation.identity.SessionIncarnationID != "" { + current, err := s.messageQueue.ResolveSessionIdentity( + promptCtx, + reservation.identity.TaskID, + reservation.identity.SessionID, + ) + if err != nil || current != reservation.identity { + s.logger.Info("discarding queued dispatch for replaced session", + zap.String("session_id", callerSessionID), + zap.String("task_id", queuedMsg.TaskID), + zap.String("queue_id", queuedMsg.ID)) + s.discardLifecycleReservationForReplacedSession( + promptCtx, + queuedMsg, + reservation.identity, + ) + return + } + } + if s.isSessionResetInProgress(queuedMsg.SessionID) { s.logger.Warn("queued message execution deferred due to context reset in progress", zap.String("session_id", callerSessionID), zap.String("task_id", queuedMsg.TaskID), zap.String("queue_id", queuedMsg.ID)) - s.requeueMessage(promptCtx, queuedMsg, "workflow-auto-start-reset-retry") + s.requeueMessageForReservation(promptCtx, reservation, queuedMsg, "workflow-auto-start-reset-retry") return } if lifecyclePrompt && !s.lifecycleQueuedDispatchIsCurrent(promptCtx, queuedMsg) { @@ -849,6 +1087,13 @@ func (s *Service) executeQueuedMessageWithReservation( zap.String("session_id", callerSessionID), zap.String("task_id", queuedMsg.TaskID), zap.String("queue_id", queuedMsg.ID)) + if reservation != nil { + s.discardLifecycleReservationForReplacedSession( + promptCtx, + queuedMsg, + reservation.identity, + ) + } return } @@ -867,42 +1112,18 @@ func (s *Service) executeQueuedMessageWithReservation( references := entityrefs.NormalizePersisted(queuedMsg.Metadata[messagequeue.MetadataEntityReferences]) promptContent := AppendEntityReferenceContext(queuedMsg.Content, references) promptContent = appendStepHandoffToPrompt(promptContent, stepHandoffFromQueuedMetadata(queuedMsg.Metadata)) - - // Create user messages for ordinary queue entries now. Lifecycle entries - // persist their visible message only after the final active-task claim. - // Skip when the queued metadata is tagged user_message_recorded — that means - // autoStartStepPrompt already inserted the chat row via recordAutoStartMessage - // before queueing (the post-recordAutoStartMessage retry branches). Recording - // here would produce the duplicate user message observed when a workflow - // auto-start failed transiently and the queue drained on boot_ready. - alreadyRecorded, _ := queuedMsg.Metadata[metaKeyUserMessageRecorded].(bool) - userMessageRecorded := alreadyRecorded - if !lifecyclePrompt { - if err := s.recordQueuedUserMessage(promptCtx, queuedMsg, attachments); err == nil && s.messageCreator != nil { - userMessageRecorded = true - } - } - - // Process on_turn_start before sending the queued prompt, just like - // dispatchPromptAsync does for user-initiated messages. This allows - // workflow transitions (e.g. move_to_next) to fire on auto-started prompts. - if session, sErr := s.repo.GetTaskSession(promptCtx, queuedMsg.SessionID); sErr == nil { - if lifecyclePrompt && !s.lifecycleQueuedDispatchIsCurrent(promptCtx, queuedMsg) { - s.logger.Info("discarding stale lifecycle dispatch before workflow side effects", - zap.String("session_id", callerSessionID), - zap.String("task_id", queuedMsg.TaskID), - zap.String("queue_id", queuedMsg.ID)) - return - } - if !turnStartAlreadyProcessed(queuedMsg.Metadata) { - s.processOnTurnStartViaEngine(promptCtx, queuedMsg.TaskID, session) - } - } + userMessageRecorded := false // Call promptTask with this entry's ID as a second ownership check. The // worker already claimed the handoff before visible side effects; promptTask // revalidates that ownership while it marks the session RUNNING. - afterClaim := s.queuedLifecycleAfterClaim(promptCtx, queuedMsg, attachments, lifecyclePrompt) + var dispatchIdentity messagequeue.QueueSessionIdentity + if reservation != nil { + dispatchIdentity = reservation.identity + } + afterClaim := s.queuedMessageAfterClaim( + promptCtx, dispatchIdentity, queuedMsg, attachments, lifecyclePrompt, &userMessageRecorded, + ) _, err := s.promptTask(promptCtx, queuedMsg.TaskID, queuedMsg.SessionID, promptContent, queuedMsg.Model, queuedMsg.PlanMode, attachments, false, promptTaskOptions{ @@ -917,35 +1138,89 @@ func (s *Service) executeQueuedMessageWithReservation( s.reconcileQueuedCIAutoFixDispatchFailure(promptCtx, queuedMsg) } s.finishQueuedMessageExecution( - promptCtx, callerSessionID, reservedSessionID, queuedMsg, + promptCtx, callerSessionID, reservedSessionID, queuedMsg, reservation, lifecyclePrompt, userMessageRecorded, err, ) + s.clearQueuedDispatchInFlightIfCurrent(reservedSessionID, reservation) + if err == nil { + // A ready event can arrive while this worker still owns the reservation. + // Retry through the public guarded drain after releasing that marker so a + // later FIFO entry is not stranded when the ready handler backed off. + s.drainQueuedMessageForPromptableSession(promptCtx, reservedSessionID) + } + } -func (s *Service) queuedLifecycleAfterClaim( +func (s *Service) queuedMessageAfterClaim( ctx context.Context, + identity messagequeue.QueueSessionIdentity, queuedMsg *messagequeue.QueuedMessage, attachments []v1.MessageAttachment, lifecyclePrompt bool, + userMessageRecorded *bool, ) func() error { - if !lifecyclePrompt { - return nil + alreadyRecorded, _ := queuedMsg.Metadata[metaKeyUserMessageRecorded].(bool) + if userMessageRecorded != nil { + *userMessageRecorded = alreadyRecorded } return func() error { - if !s.lifecycleQueuedDispatchIsCurrent(ctx, queuedMsg) { + if !s.queuedDispatchIdentityIsCurrent(ctx, identity) { return errLifecyclePromptReservationSuperseded } - return s.recordQueuedUserMessage(ctx, queuedMsg, attachments) + if lifecyclePrompt && !s.lifecycleQueuedDispatchIsCurrent(ctx, queuedMsg) { + return errLifecyclePromptReservationSuperseded + } + if !alreadyRecorded { + if err := s.recordQueuedUserMessage(ctx, queuedMsg, attachments); err != nil { + if lifecyclePrompt { + return err + } + } else if userMessageRecorded != nil && s.messageCreator != nil { + *userMessageRecorded = true + } + } + if session, err := s.repo.GetTaskSession(ctx, queuedMsg.SessionID); err == nil && + s.queuedSessionMatchesIdentity(session, identity) && + !turnStartAlreadyProcessed(queuedMsg.Metadata) { + s.processOnTurnStartViaEngine(ctx, queuedMsg.TaskID, session) + } + return nil } } +func (s *Service) queuedDispatchIdentityIsCurrent( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, +) bool { + if identity.SessionIncarnationID == "" { + return true + } + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + return err == nil && current == identity +} + +func (s *Service) queuedSessionMatchesIdentity( + session *models.TaskSession, + identity messagequeue.QueueSessionIdentity, +) bool { + if session == nil { + return false + } + return identity.SessionIncarnationID == "" || + (session.TaskID == identity.TaskID && + session.ID == identity.SessionID && + session.QueueIncarnationID == identity.SessionIncarnationID) +} + func (s *Service) finishQueuedMessageExecution( ctx context.Context, callerSessionID, reservedSessionID string, queuedMsg *messagequeue.QueuedMessage, + reservation *queuedDispatchReservation, lifecyclePrompt, userMessageRecorded bool, err error, ) { + exactReservation := reservation != nil && reservation.identity.SessionIncarnationID != "" if errors.Is(err, errLifecyclePromptReservationSuperseded) { s.logger.Info("discarding stale lifecycle dispatch after final claim", zap.String("session_id", callerSessionID), @@ -954,16 +1229,38 @@ func (s *Service) finishQueuedMessageExecution( return } if errors.Is(err, errLifecyclePromptInactive) { - s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + if exactReservation { + s.acknowledgeLifecycleQueueEntryForSession(ctx, reservation.identity, queuedMsg) + } else { + s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + } return } var reselected *lifecyclePromptReselectedError if errors.As(err, &reselected) { - queuedMsg.SessionID = reselected.sessionID - if s.requeueLifecycleMessage( - ctx, queuedMsg, queuedMsg.QueuedBy, messageCoalesceKey(queuedMsg), - ) { - s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + if !exactReservation { + queuedMsg.SessionID = reselected.sessionID + if s.requeueLifecycleMessage( + ctx, queuedMsg, queuedMsg.QueuedBy, messageCoalesceKey(queuedMsg), + ) { + s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + } + return + } + destination, resolveErr := s.messageQueue.ResolveSessionIdentity( + ctx, queuedMsg.TaskID, reselected.sessionID, + ) + if resolveErr != nil { + s.logPostClaimQueueMutationFailure("resolve reselected lifecycle session", reservation.identity, queuedMsg, resolveErr) + return + } + retry := *queuedMsg + retry.SessionID = destination.SessionID + _, requeueErr := s.requeueLifecycleMessageForSession( + ctx, destination, &retry, retry.QueuedBy, messageCoalesceKey(&retry), + ) + if requeueErr == nil { + s.acknowledgeLifecycleQueueEntryForSession(ctx, reservation.identity, queuedMsg) } return } @@ -978,17 +1275,25 @@ func (s *Service) finishQueuedMessageExecution( zap.String("session_id", callerSessionID), zap.String("task_id", queuedMsg.TaskID), zap.String("queue_id", queuedMsg.ID)) - s.requeueMessage(ctx, queuedMsg, "superseded-by-newer-dispatch") + if exactReservation { + _ = s.requeueMessageForSession(ctx, reservation.identity, queuedMsg, "superseded-by-newer-dispatch") + } else { + s.requeueMessage(ctx, queuedMsg, "superseded-by-newer-dispatch") + } return } if err != nil { s.handleQueuedMessageExecutionError( - ctx, callerSessionID, queuedMsg, lifecyclePrompt, userMessageRecorded, err, + ctx, callerSessionID, queuedMsg, reservation, lifecyclePrompt, userMessageRecorded, err, ) return } if lifecyclePrompt { - s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + if exactReservation { + s.acknowledgeLifecycleQueueEntryForSession(ctx, reservation.identity, queuedMsg) + } else { + s.acknowledgeLifecycleQueueEntry(ctx, reservedSessionID, queuedMsg) + } } } @@ -996,6 +1301,7 @@ func (s *Service) handleQueuedMessageExecutionError( ctx context.Context, callerSessionID string, queuedMsg *messagequeue.QueuedMessage, + reservation *queuedDispatchReservation, lifecyclePrompt, userMessageRecorded bool, err error, ) { @@ -1021,9 +1327,14 @@ func (s *Service) handleQueuedMessageExecutionError( zap.String("task_id", queuedMsg.TaskID), zap.String("queue_id", queuedMsg.ID), zap.Bool("manual_recovery", manualRecovery)) - if manualRecovery { + switch { + case reservation != nil && reservation.identity.SessionIncarnationID != "" && manualRecovery: + s.restoreQueuedMessageForSession(ctx, reservation.identity, queuedMsg) + case reservation != nil && reservation.identity.SessionIncarnationID != "": + _ = s.requeueMessageForSession(ctx, reservation.identity, queuedMsg, "workflow-auto-start-retry") + case manualRecovery: s.restoreQueuedMessage(ctx, queuedMsg) - } else { + default: s.requeueMessage(ctx, queuedMsg, "workflow-auto-start-retry") } return @@ -1060,7 +1371,7 @@ func (s *Service) claimQueuedMessageHandoff( case accepted != nil && accepted.entryID == queuedMsg.ID: reservation = accepted case pending != nil || accepted != nil: - s.requeueMessage(ctx, queuedMsg, "superseded-by-newer-dispatch") + s.requeueMessageForReservation(ctx, reservation, queuedMsg, "superseded-by-newer-dispatch") return "", true } } @@ -1075,14 +1386,14 @@ func (s *Service) claimQueuedMessageHandoff( // must not requeue it or create any visible side effects. return "", true case errors.Is(claimErr, errQueuedDispatchSuperseded): - s.requeueMessage(ctx, queuedMsg, "superseded-by-newer-dispatch") + s.requeueMessageForReservation(ctx, reservation, queuedMsg, "superseded-by-newer-dispatch") return "", true case claimErr != nil: s.logger.Warn("failed to claim queued dispatch ownership", zap.String("session_id", callerSessionID), zap.String("queue_id", queuedMsg.ID), zap.Error(claimErr)) - s.requeueMessage(ctx, queuedMsg, "queued-dispatch-claim-retry") + s.requeueMessageForReservation(ctx, reservation, queuedMsg, "queued-dispatch-claim-retry") return "", true case tracked: return queuedMsg.ID, false @@ -1090,6 +1401,25 @@ func (s *Service) claimQueuedMessageHandoff( return "", false } } +func (s *Service) discardLifecycleReservationForReplacedSession( + ctx context.Context, + queuedMsg *messagequeue.QueuedMessage, + identity messagequeue.QueueSessionIdentity, +) { + if s.messageQueue == nil || queuedMsg == nil || + !queuedMsg.IsReservedLifecycleDelivery() || identity.SessionIncarnationID == "" { + return + } + if err := s.messageQueue.DiscardLifecycleReservation(ctx, identity, queuedMsg.ID); err != nil { + s.logger.Error("failed to discard lifecycle reservation for replaced session", + zap.String("session_id", identity.SessionID), + zap.String("task_id", identity.TaskID), + zap.String("queue_id", queuedMsg.ID), + zap.Error(err)) + return + } + s.publishQueueStatusEventForIdentity(ctx, identity) +} func (s *Service) lifecycleQueuedDispatchIsCurrent( ctx context.Context, @@ -1130,6 +1460,21 @@ func (s *Service) acknowledgeLifecycleQueueEntry( s.publishQueueStatusEvent(ctx, sessionID) } +func (s *Service) acknowledgeLifecycleQueueEntryForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, +) { + if s.messageQueue == nil || queuedMsg == nil || !queuedMsg.IsDurableLifecycle() { + return + } + if err := s.messageQueue.AcknowledgeQueuedForSession(ctx, identity, queuedMsg.ID); err != nil { + s.logPostClaimQueueMutationFailure("acknowledge lifecycle message", identity, queuedMsg, err) + return + } + s.publishQueueStatusEventForIdentity(ctx, identity) +} + func (s *Service) restoreQueuedMessage(ctx context.Context, queuedMsg *messagequeue.QueuedMessage) { restored, err := s.messageQueue.RestoreMessage(ctx, queuedMsg) if err != nil { @@ -1148,6 +1493,24 @@ func (s *Service) restoreQueuedMessage(ctx context.Context, queuedMsg *messagequ s.publishQueueStatusEvent(ctx, queuedMsg.SessionID) } +func (s *Service) restoreQueuedMessageForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, +) { + restored, err := s.messageQueue.RestoreMessageForSession(ctx, identity, queuedMsg) + if err != nil { + s.logPostClaimQueueMutationFailure("restore message", identity, queuedMsg, err) + return + } + s.logger.Info("message restored for manual recovery on exact session", + zap.String("session_id", restored.SessionID), + zap.String("task_id", restored.TaskID), + zap.String("queue_id", restored.ID), + zap.Int64("position", restored.Position)) + s.publishQueueStatusEventForIdentity(ctx, identity) +} + func markQueuedUserMessageRecorded(queuedMsg *messagequeue.QueuedMessage) { if queuedMsg.Metadata == nil { queuedMsg.Metadata = make(map[string]interface{}) @@ -1317,6 +1680,15 @@ func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.A return } + s.finishAgentCompleted(ctx, data, session, guard) +} + +func (s *Service) finishAgentCompleted( + ctx context.Context, + data watcher.AgentEventData, + session *models.TaskSession, + guard *lockedCancelInFlightGuard, +) { // A successful, still-live completion clears retry state and scheduler // ownership only after the guarded terminal/rotation checks above. s.resetTransientRetry(data.SessionID) @@ -1342,7 +1714,8 @@ func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.A return } - transitioned := s.processOnTurnCompleteViaEngine(ctx, data.TaskID, session) + transitioned := !s.drainQueuedBeforeWorkflowTransition(ctx, data.TaskID, data.SessionID, session) && + s.processOnTurnCompleteViaEngine(ctx, data.TaskID, session) s.finishAgentCompletedTurn(ctx, data, session, transitioned, guard) } @@ -1411,6 +1784,10 @@ func (s *Service) finishAgentCompletedTurn( zap.String("session_id", data.SessionID), zap.Error(err)) } + // Some runtimes report completion without a matching agent.ready event. + // Once completion has settled the session, give an enabled queue its normal + // promptable drain instead of leaving its head parked indefinitely. + s.drainQueuedMessageForPromptableSession(context.WithoutCancel(ctx), data.SessionID) } // handleAgentFailed handles agent failure events diff --git a/apps/backend/internal/orchestrator/event_handlers_agent_clarification_test.go b/apps/backend/internal/orchestrator/event_handlers_agent_clarification_test.go index 6809aa7119..2e649a0112 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent_clarification_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent_clarification_test.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/orchestrator/watcher" "github.com/kandev/kandev/internal/task/models" wfmodels "github.com/kandev/kandev/internal/workflow/models" + "github.com/stretchr/testify/require" ) type repoBackedTurnService struct { @@ -301,21 +302,15 @@ func TestHandleAgentCompleted_BlocksOnTurnCompleteWhileClarificationPending(t *t AgentExecutionID: "exec-1", }) - task, err = repo.GetTask(ctx, "t1") - if err != nil { - t.Fatalf("get task: %v", err) - } - if task.WorkflowStepID != "step1" { - t.Fatalf("expected workflow step to remain step1, got %q", task.WorkflowStepID) - } - - session, err = repo.GetTaskSession(ctx, "s1") - if err != nil { - t.Fatalf("get session: %v", err) - } - if session.State != models.TaskSessionStateWaitingForInput { - t.Fatalf("expected session %q, got %q", models.TaskSessionStateWaitingForInput, session.State) - } + require.Eventually(t, func() bool { + storedTask, taskErr := repo.GetTask(ctx, "t1") + storedSession, sessionErr := repo.GetTaskSession(ctx, "s1") + return taskErr == nil && + sessionErr == nil && + storedTask.WorkflowStepID == "step1" && + storedSession.State == models.TaskSessionStateWaitingForInput + }, 2*time.Second, 10*time.Millisecond, + "pending clarification should keep the task at step1 and settle the session waiting for input") turn, err := svc.turnService.GetActiveTurn(ctx, "s1") if err != nil && !errors.Is(err, sql.ErrNoRows) { t.Fatalf("get active turn: %v", err) diff --git a/apps/backend/internal/orchestrator/event_handlers_clarification.go b/apps/backend/internal/orchestrator/event_handlers_clarification.go index 99a7f0a2c5..0ae7b58c5b 100644 --- a/apps/backend/internal/orchestrator/event_handlers_clarification.go +++ b/apps/backend/internal/orchestrator/event_handlers_clarification.go @@ -584,14 +584,18 @@ func (s *Service) dispatchClarificationResumeLocked(ctx context.Context, data cl // silently the way a bare false return did. return fmt.Errorf("cannot resume clarification: message queue is not configured") } - queued, err := s.messageQueue.QueueMessageWithMetadata( - ctx, data.SessionID, data.TaskID, prompt, "", messagequeue.QueuedByAgent, false, nil, + identity, err := s.messageQueue.ResolveSessionIdentity(ctx, data.TaskID, data.SessionID) + if err != nil { + return fmt.Errorf("resolve clarification resume session: %w", err) + } + queued, err := s.messageQueue.QueueMessageWithMetadataForSession( + ctx, identity, prompt, "", messagequeue.QueuedByAgent, false, nil, map[string]interface{}{metaKeyUserMessageRecorded: true}, ) if err != nil { return fmt.Errorf("queue clarification resume prompt: %w", err) } - dispatched, err := s.takeAndDispatchEntryLocked(ctx, data.SessionID, queued.ID) + dispatched, err := s.takeAndDispatchEntryLocked(ctx, identity, queued.ID) if err != nil { return fmt.Errorf("dispatch clarification resume prompt: %w", err) } diff --git a/apps/backend/internal/orchestrator/event_handlers_duplicate_autostart_test.go b/apps/backend/internal/orchestrator/event_handlers_duplicate_autostart_test.go index 8c26f7335c..3f9c90125c 100644 --- a/apps/backend/internal/orchestrator/event_handlers_duplicate_autostart_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_duplicate_autostart_test.go @@ -133,7 +133,7 @@ func TestAutoStartTransientError_BootReadyDrainsOrphanedQueue(t *testing.T) { repo: repo, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(testLogger()), + messageQueue: newAuthoritativeMemoryQueue(repo, testLogger()), executor: exec, messageCreator: msgCreator, } @@ -441,7 +441,7 @@ func TestAutoStartTransientError_AutoResumesWhenAgentDead(t *testing.T) { repo: repo, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(testLogger()), + messageQueue: newAuthoritativeMemoryQueue(repo, testLogger()), executor: exec, messageCreator: msgCreator, } diff --git a/apps/backend/internal/orchestrator/event_handlers_github_ci_automation_test.go b/apps/backend/internal/orchestrator/event_handlers_github_ci_automation_test.go index 271aa7101f..f820544a16 100644 --- a/apps/backend/internal/orchestrator/event_handlers_github_ci_automation_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_github_ci_automation_test.go @@ -2576,7 +2576,7 @@ func TestDispatchCIAutomationPromptForPRQueuesWhenRunningUserMessageCannotBeReco repo := setupTestRepo(t) seedTaskAndSession(t, repo, "task-1", "session-1", models.TaskSessionStateRunning) svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) - svc.messageQueue = messagequeue.NewServiceMemory(testLogger()) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) messageCreator := &mockMessageCreator{userMessageErr: errors.New("message db unavailable")} svc.messageCreator = messageCreator session, err := repo.GetTaskSession(ctx, "session-1") diff --git a/apps/backend/internal/orchestrator/event_handlers_github_pr_automation_test.go b/apps/backend/internal/orchestrator/event_handlers_github_pr_automation_test.go index e5a6ff3d7c..536837b45d 100644 --- a/apps/backend/internal/orchestrator/event_handlers_github_pr_automation_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_github_pr_automation_test.go @@ -33,6 +33,12 @@ func (r *lifecycleQueueFailureRepository) InsertOrReplaceLifecycleByCoalesceKey( return nil, false, r.err } +func (r *lifecycleQueueFailureRepository) InsertOrReplaceLifecycleByCoalesceKeyForSession( + context.Context, messagequeue.QueueSessionIdentity, *messagequeue.QueuedMessage, string, int, bool, +) (*messagequeue.QueuedMessage, bool, error) { + return nil, false, r.err +} + // lifecycleAcknowledgingRepository exposes the durable lifecycle completion // boundary. ReserveHead intentionally retains lifecycle entries until the // executor accepts the prompt and AcknowledgeByID commits the removal. @@ -51,6 +57,18 @@ func (r *lifecycleAcknowledgingRepository) AcknowledgeByID( return nil } +func (r *lifecycleAcknowledgingRepository) AcknowledgeByIDForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + entryID string, +) error { + if err := r.Repository.AcknowledgeByIDForSession(ctx, identity, entryID); err != nil { + return err + } + close(r.acknowledged) + return nil +} + // archiveDuringLifecycleQueueRepository deterministically commits an archive // after lifecycle delivery has rechecked task activity, but before the queue // can accept the resulting prompt. It models the archive winning the TOCTOU @@ -78,6 +96,28 @@ func (r *lifecycleClaimObservingRepository) ClaimPromptableTaskSessionIfActive( return claim, err } +func (r *lifecycleClaimObservingRepository) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + claim, err := r.repoStore.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, taskID, sessionID, incarnationID, + ) + r.claimed <- claim.Status == models.PromptableTaskSessionClaimed + return claim, err +} + +func (r *lifecycleClaimObservingRepository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + return r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) +} + func (r *archiveDuringLifecycleQueueRepository) InsertOrReplaceByCoalesceKey( ctx context.Context, msg *messagequeue.QueuedMessage, @@ -110,6 +150,23 @@ func (r *archiveDuringLifecycleQueueRepository) InsertOrReplaceLifecycleByCoales return nil, false, messagequeue.ErrTaskInactive } +func (r *archiveDuringLifecycleQueueRepository) InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx context.Context, + _ messagequeue.QueueSessionIdentity, + _ *messagequeue.QueuedMessage, + _ string, + _ int, + _ bool, +) (*messagequeue.QueuedMessage, bool, error) { + if !r.archived { + r.archived = true + if err := r.archive(ctx); err != nil { + return nil, false, err + } + } + return nil, false, messagequeue.ErrTaskInactive +} + func TestDecideTaskPRAgentPrompt(t *testing.T) { trueValue := true falseValue := false @@ -315,7 +372,7 @@ func TestDispatchTaskPRAgentPrompt_ReadySessionsDrainImmediately(t *testing.T) { agent := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo, promptDone: make(chan struct{})} svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), agent) acknowledgingRepo := &lifecycleAcknowledgingRepository{ - Repository: messagequeue.NewMemoryRepository(), + Repository: newAuthoritativeMemoryRepository(repo), acknowledged: make(chan struct{}), } svc.messageQueue = messagequeue.NewService( @@ -469,7 +526,7 @@ func TestEvalTaskPRLifecycle_RecordsCheckpointOnlyAfterQueueAcceptance(t *testin svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) queueErr := errors.New("queue unavailable") svc.messageQueue = messagequeue.NewService(&lifecycleQueueFailureRepository{ - Repository: messagequeue.NewMemoryRepository(), err: queueErr, + Repository: newAuthoritativeMemoryRepository(repo), err: queueErr, }, messagequeue.DefaultMaxPerSession, testLogger()) delivered, err := svc.evalTaskPRLifecycle(ctx, pr, options, ghSvc) @@ -480,7 +537,7 @@ func TestEvalTaskPRLifecycle_RecordsCheckpointOnlyAfterQueueAcceptance(t *testin t.Fatalf("checkpoint recorded after failed queue acceptance: %+v", prompts) } - svc.messageQueue = messagequeue.NewServiceMemory(testLogger()) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) delivered, err = svc.evalTaskPRLifecycle(ctx, pr, options, ghSvc) if err != nil || !delivered { t.Fatalf("accepted queue = delivered %v, err %v", delivered, err) @@ -501,7 +558,7 @@ func TestEvalTaskPRLifecycle_ArchiveWinningAfterActivityCheckDoesNotQueueOrClaim } svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) svc.messageQueue = messagequeue.NewService(&archiveDuringLifecycleQueueRepository{ - Repository: messagequeue.NewMemoryRepository(), + Repository: newAuthoritativeMemoryRepository(repo), archive: func(ctx context.Context) error { return repo.ArchiveTask(ctx, "task-1") }, diff --git a/apps/backend/internal/orchestrator/event_handlers_gitlab_mr_automation_test.go b/apps/backend/internal/orchestrator/event_handlers_gitlab_mr_automation_test.go index 284f4576fc..b6ab06f500 100644 --- a/apps/backend/internal/orchestrator/event_handlers_gitlab_mr_automation_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_gitlab_mr_automation_test.go @@ -601,7 +601,7 @@ func TestDispatchTaskMRAgentPrompt_ReadySessionDrainsAndAcknowledges(t *testing. agent := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo, promptDone: make(chan struct{})} svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), agent) acknowledgingRepo := &lifecycleAcknowledgingRepository{ - Repository: messagequeue.NewMemoryRepository(), + Repository: newAuthoritativeMemoryRepository(repo), acknowledged: make(chan struct{}), } svc.messageQueue = messagequeue.NewService(acknowledgingRepo, messagequeue.DefaultMaxPerSession, testLogger()) diff --git a/apps/backend/internal/orchestrator/event_handlers_lifecycle_dispatch.go b/apps/backend/internal/orchestrator/event_handlers_lifecycle_dispatch.go index 3ad3aeb1fa..52c4d22080 100644 --- a/apps/backend/internal/orchestrator/event_handlers_lifecycle_dispatch.go +++ b/apps/backend/internal/orchestrator/event_handlers_lifecycle_dispatch.go @@ -21,22 +21,25 @@ func (s *Service) queueAndDrainLifecyclePrompt( if s.messageQueue == nil { return "", fmt.Errorf("message queue is not configured") } - if _, _, accepted, err := s.messageQueue.QueueLifecycleMessageWithCoalesceKey( - ctx, session.ID, taskID, prompt, "", messagequeue.QueuedByWorkflow, + identity, err := s.resolveQueueIdentityForSession(ctx, session) + if err != nil || identity.TaskID != taskID { + if err != nil { + return "", err + } + return "", messagequeue.ErrSessionIdentityMismatch + } + if _, _, accepted, err := s.messageQueue.QueueLifecycleMessageWithCoalesceKeyForSession( + ctx, identity, prompt, "", messagequeue.QueuedByWorkflow, false, nil, metadata, coalesceKey, true, ); err != nil { return "", err } else if !accepted { return "", inactiveErr } - s.publishQueueStatusEvent(ctx, session.ID) + s.publishQueueStatusEventForIdentity(ctx, identity) // Always attempt the drain rather than gating it on this stale `session` - // snapshot's state: drainQueuedMessageForPromptableSession reloads the - // session and re-checks promptability itself, so it safely no-ops when - // not ready. Gating here on the pre-queue snapshot left a real race — a - // session that became promptable between snapshot and queueing would - // have its prompt stuck until some unrelated later event triggered a - // drain. - s.drainQueuedMessageForPromptableSession(ctx, session.ID) + // snapshot's state. The exact drain reloads promptability while preserving + // the incarnation captured by the lifecycle admission. + _, _ = s.drainQueuedMessageForPromptableSessionForIdentity(ctx, identity) return session.ID, nil } diff --git a/apps/backend/internal/orchestrator/event_handlers_passthrough_lifecycle_test.go b/apps/backend/internal/orchestrator/event_handlers_passthrough_lifecycle_test.go index 93f102ceca..0b7c70eb11 100644 --- a/apps/backend/internal/orchestrator/event_handlers_passthrough_lifecycle_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_passthrough_lifecycle_test.go @@ -2,6 +2,7 @@ package orchestrator import ( "context" + "sync/atomic" "testing" "time" @@ -113,6 +114,120 @@ func TestHandleAgentReady_PassthroughLifecycleReservationBlocksConcurrentDrain(t } } +func TestHandleAgentReady_PassthroughOrdinaryReservationRejectsRecreatedSession(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + seedExecutorRunning(t, repo, "s1", "t1", "exec-1") + + agentMgr := &mockAgentManager{ + isPassthrough: true, + isAgentRunning: true, + repoForExecutionLookup: repo, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + if _, err := svc.messageQueue.QueueMessage( + ctx, "s1", "t1", "must not reach replacement", "", messagequeue.QueuedByAgent, false, nil, + ); err != nil { + t.Fatalf("queue ordinary prompt: %v", err) + } + svc.afterReadyLifecycleReservation = func() { + if _, err := repo.DB().ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + "s1", + ); err != nil { + t.Fatalf("replace session incarnation: %v", err) + } + } + + svc.handleAgentReady(ctx, watcher.AgentEventData{TaskID: "t1", SessionID: "s1"}) + + if got := len(agentMgr.passthroughStdinCalls); got != 0 { + t.Fatalf("replacement session PTY prompts = %d, want 0", got) + } +} + +func TestHandleAgentReady_PassthroughOrdinaryDispatchDoesNotHoldQueueLockDuringPTYWrite(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + seedExecutorRunning(t, repo, "s1", "t1", "exec-1") + + var queueWriteBlocked atomic.Bool + agentMgr := &mockAgentManager{ + isPassthrough: true, + isAgentRunning: true, + repoForExecutionLookup: repo, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, "t1", "s1") + if err != nil { + t.Fatalf("resolve session identity: %v", err) + } + agentMgr.passthroughStdinFunc = func(writeCtx context.Context, _, _ string) error { + writeDone := make(chan error, 1) + go func() { + _, queueErr := svc.messageQueue.QueueMessageWithMetadataForSession( + writeCtx, identity, "concurrent prompt", "", messagequeue.QueuedByAgent, false, nil, nil, + ) + writeDone <- queueErr + }() + select { + case queueErr := <-writeDone: + return queueErr + case <-time.After(100 * time.Millisecond): + queueWriteBlocked.Store(true) + return nil + } + } + if _, err := svc.messageQueue.QueueMessage( + ctx, "s1", "t1", "ordinary prompt", "", messagequeue.QueuedByAgent, false, nil, + ); err != nil { + t.Fatalf("queue ordinary prompt: %v", err) + } + + svc.handleAgentReady(ctx, watcher.AgentEventData{TaskID: "t1", SessionID: "s1"}) + + if queueWriteBlocked.Load() { + t.Fatal("PTY write ran while the queue session lock was held") + } +} + +func TestHandleAgentReady_PassthroughOrdinaryReservationRejectsArchivedTask(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + seedExecutorRunning(t, repo, "s1", "t1", "exec-1") + + agentMgr := &mockAgentManager{ + isPassthrough: true, + isAgentRunning: true, + repoForExecutionLookup: repo, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + if _, err := svc.messageQueue.QueueMessage( + ctx, "s1", "t1", "must not reach archived task", "", messagequeue.QueuedByAgent, false, nil, + ); err != nil { + t.Fatalf("queue ordinary prompt: %v", err) + } + svc.afterReadyLifecycleReservation = func() { + if err := repo.ArchiveTask(ctx, "t1"); err != nil { + t.Fatalf("archive task after reservation: %v", err) + } + } + + svc.handleAgentReady(ctx, watcher.AgentEventData{TaskID: "t1", SessionID: "s1"}) + + if got := len(agentMgr.passthroughStdinCalls); got != 0 { + t.Fatalf("archived task PTY prompts = %d, want 0", got) + } +} + // Archive may win after handleAgentReady reserves a durable lifecycle row and // releases its session guard, but before the deferred dispatch begins. That // stale entry must be discarded before it can trigger a workflow turn-start, diff --git a/apps/backend/internal/orchestrator/event_handlers_passthrough_running_test.go b/apps/backend/internal/orchestrator/event_handlers_passthrough_running_test.go index ddc2082ba3..4f3a37a4d9 100644 --- a/apps/backend/internal/orchestrator/event_handlers_passthrough_running_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_passthrough_running_test.go @@ -124,5 +124,5 @@ func TestHandleAgentReady_PassthroughQueuedMessagePublishesAfterWriteFailure(t * require.Equal(t, 1, runningEventCalls, "the running event must still publish after the PTY write fails") require.Empty(t, baseManager.markPassthroughCalls) require.Len(t, baseManager.passthroughStdinCalls, 1) - require.Zero(t, svc.messageQueue.GetStatus(ctx, "s1").Count) + require.Equal(t, 1, svc.messageQueue.GetStatus(ctx, "s1").Count) } diff --git a/apps/backend/internal/orchestrator/event_handlers_pending_move_test.go b/apps/backend/internal/orchestrator/event_handlers_pending_move_test.go index 1640031443..4be610fdbf 100644 --- a/apps/backend/internal/orchestrator/event_handlers_pending_move_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_pending_move_test.go @@ -396,7 +396,7 @@ func buildPendingMoveScenario(t *testing.T) *pendingMoveScenario { workflowStepGetter: stepGetter, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -412,11 +412,13 @@ func buildPendingMoveScenario(t *testing.T) *pendingMoveScenario { ); err != nil { t.Fatalf("queue hand-off prompt: %v", err) } - svc.messageQueue.SetPendingMove(ctx, reviewSessionID, &messagequeue.PendingMove{ + if err := svc.messageQueue.SetPendingMove(ctx, reviewSessionID, &messagequeue.PendingMove{ TaskID: "task-1", WorkflowID: "wf1", WorkflowStepID: stepInProgressID, - }) + }); err != nil { + t.Fatalf("set pending move: %v", err) + } return &pendingMoveScenario{ ctx: ctx, diff --git a/apps/backend/internal/orchestrator/event_handlers_queue_general_test.go b/apps/backend/internal/orchestrator/event_handlers_queue_general_test.go index f828ee3709..1238df99a7 100644 --- a/apps/backend/internal/orchestrator/event_handlers_queue_general_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_queue_general_test.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" "github.com/kandev/kandev/internal/github" @@ -17,6 +18,26 @@ import ( v1 "github.com/kandev/kandev/pkg/api/v1" ) +type sessionIdentityTransitionHookRepository struct { + repoStore + afterTransition func() +} + +func (r *sessionIdentityTransitionHookRepository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + changed, updatedAt, err := r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) + if err == nil && changed && r.afterTransition != nil { + r.afterTransition() + } + return changed, updatedAt, err +} + func TestExecuteQueuedMessage_RequeuesWhenResetInProgress(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) @@ -56,6 +77,59 @@ func TestExecuteQueuedMessage_RequeuesWhenResetInProgress(t *testing.T) { } } +func TestExecuteQueuedMessage_DoesNotPublishVisibleEffectsForReplacedIdentity(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t-replaced", "s-replaced", "step-replaced") + seedExecutorRunning(t, repo, "s-replaced", "t-replaced", "exec-replaced") + session, err := repo.GetTaskSession(ctx, "s-replaced") + if err != nil { + t.Fatalf("get session: %v", err) + } + session.State = models.TaskSessionStateWaitingForInput + if err := repo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("set waiting session: %v", err) + } + agentMgr := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + messages := &mockMessageCreator{} + svc.messageCreator = messages + svc.repo = &sessionIdentityTransitionHookRepository{ + repoStore: repo, + afterTransition: func() { + if _, updateErr := repo.DB().Exec( + `UPDATE task_sessions + SET queue_incarnation_id = ?, state = ? + WHERE id = ?`, + "replacement-incarnation", + models.TaskSessionStateWaitingForInput, + "s-replaced", + ); updateErr != nil { + t.Fatalf("replace session identity: %v", updateErr) + } + }, + } + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, "t-replaced", "s-replaced") + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } + queued := &messagequeue.QueuedMessage{ + ID: "replaced-dispatch", SessionID: "s-replaced", TaskID: "t-replaced", Content: "stale prompt", + } + lock, release := svc.acquireCancelInFlightGuard(identity.SessionID) + lock.Lock() + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, nil) + lock.Unlock() + release() + + svc.executeQueuedMessageWithReservation(identity.SessionID, queued, reservation) + + if len(messages.userMessages) != 0 { + t.Fatalf("replacement received %d stale visible messages, want 0", len(messages.userMessages)) + } +} + func TestExecuteQueuedMessage_RequeuesCancelReleaseFailure(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) @@ -291,6 +365,7 @@ func TestExecuteQueuedMessage_StoresAttachmentsInUserMessageMetadata(t *testing. ctx := context.Background() repo := setupTestRepo(t) seedSession(t, repo, "t1", "s1", "step1") + seedExecutorRunning(t, repo, "s1", "t1", "exec-1") session, err := repo.GetTaskSession(ctx, "s1") if err != nil { @@ -352,6 +427,14 @@ func TestExecuteQueuedMessageTransientRetryDoesNotDuplicateRecordedUserMessage(t repo := setupTestRepo(t) seedSession(t, repo, "t1", "s1", "step1") seedExecutorRunning(t, repo, "s1", "t1", "exec-1") + session, err := repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("get session: %v", err) + } + session.State = models.TaskSessionStateWaitingForInput + if err := repo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("set waiting session: %v", err) + } agentMgr := &mockAgentManager{ isAgentRunning: true, promptErr: fmt.Errorf("agent stream disconnected while prompting"), diff --git a/apps/backend/internal/orchestrator/event_handlers_queue_test.go b/apps/backend/internal/orchestrator/event_handlers_queue_test.go index 1588135455..d701561b3c 100644 --- a/apps/backend/internal/orchestrator/event_handlers_queue_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_queue_test.go @@ -51,6 +51,33 @@ func (r *lifecycleClaimSequenceRepository) ClaimPromptableTaskSessionIfActive( return r.repoStore.ClaimPromptableTaskSessionIfActive(ctx, sessionID) } +func (r *lifecycleClaimSequenceRepository) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + if r.claimCalls < len(r.claimErrors) { + err := r.claimErrors[r.claimCalls] + r.claimCalls++ + if err != nil { + return models.PromptableTaskSessionClaim{}, err + } + } + return r.repoStore.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, taskID, sessionID, incarnationID, + ) +} + +func (r *lifecycleClaimSequenceRepository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + return r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) +} + type lifecycleClaimBarrierRepository struct { repoStore claimEntered chan struct{} @@ -93,33 +120,26 @@ type hookedMessageCreator struct { } // archiveBeforeLifecycleRetryRepository commits an archive after a lifecycle -// entry has been dequeued and its claim failed, but before the generic retry -// insert runs. The channels make that schedule explicit without sleeps. +// entry has been reserved and its claim failed, but before the exact retry +// releases that reservation. The channels make that schedule explicit without +// sleeps. type archiveBeforeLifecycleRetryRepository struct { messagequeue.Repository archive func(context.Context) error requeueStarted chan struct{} archiveCommitted chan struct{} - lifecycleCalls int } -func (r *archiveBeforeLifecycleRetryRepository) InsertOrReplaceLifecycleByCoalesceKey( +func (r *archiveBeforeLifecycleRetryRepository) RequeuePreservingFIFO( ctx context.Context, - msg *messagequeue.QueuedMessage, - coalesceKey string, - maxPerSession int, - allowInsert bool, -) (*messagequeue.QueuedMessage, bool, error) { - r.lifecycleCalls++ - if r.lifecycleCalls == 1 { - return r.Repository.InsertOrReplaceLifecycleByCoalesceKey(ctx, msg, coalesceKey, maxPerSession, allowInsert) - } + _ *messagequeue.QueuedMessage, +) error { r.requeueStarted <- struct{}{} if err := r.archive(ctx); err != nil { - return nil, false, err + return err } r.archiveCommitted <- struct{}{} - return nil, false, messagequeue.ErrTaskInactive + return messagequeue.ErrTaskInactive } func (r *lifecycleClaimBarrierRepository) ClaimPromptableTaskSessionIfActive( @@ -130,6 +150,28 @@ func (r *lifecycleClaimBarrierRepository) ClaimPromptableTaskSessionIfActive( return r.repoStore.ClaimPromptableTaskSessionIfActive(ctx, sessionID) } +func (r *lifecycleClaimBarrierRepository) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + r.claimEntered <- struct{}{} + <-r.allowClaim + return r.repoStore.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, taskID, sessionID, incarnationID, + ) +} + +func (r *lifecycleClaimBarrierRepository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + return r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) +} + func (r *lifecycleClaimHookRepository) ClaimPromptableTaskSessionIfActive( ctx context.Context, sessionID string, ) (models.PromptableTaskSessionClaim, error) { @@ -140,6 +182,30 @@ func (r *lifecycleClaimHookRepository) ClaimPromptableTaskSessionIfActive( return claim, err } +func (r *lifecycleClaimHookRepository) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + claim, err := r.repoStore.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, taskID, sessionID, incarnationID, + ) + if err == nil && claim.Status == models.PromptableTaskSessionClaimed && r.afterClaim != nil { + r.afterClaim() + } + return claim, err +} + +func (r *lifecycleClaimHookRepository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + return r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) +} + func (r *lifecycleResolveHookRepository) ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error) { if r.beforeResolve != nil { r.beforeResolve() @@ -227,8 +293,8 @@ func TestExecuteQueuedMessage_LifecycleRequeueAfterArchiveIsDiscardedBeforeUnarc t.Fatal("lifecycle entry was not dequeued") } - // The failed claim reaches the generic requeue. Its repository barrier - // commits the archive before that insert is allowed to proceed. + // The failed claim reaches the exact reservation retry. Its repository + // barrier commits the archive before that release is allowed to proceed. svc.executeQueuedMessage("s1", queued) <-queueRepo.requeueStarted <-queueRepo.archiveCommitted @@ -256,6 +322,167 @@ func TestExecuteQueuedMessage_LifecycleRequeueAfterArchiveIsDiscardedBeforeUnarc } } +func TestDrainQueuedBeforeWorkflowTransition_DoesNotSuppressTransitionWhenAdmissionSkips(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t-queue-admission", "s-queue-admission", "step1") + + session, err := repo.GetTaskSession(ctx, "s-queue-admission") + if err != nil { + t.Fatalf("get session: %v", err) + } + session.State = models.TaskSessionStateWaitingForInput + if err := repo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("set session waiting: %v", err) + } + task, err := repo.GetTask(ctx, "t-queue-admission") + if err != nil { + t.Fatalf("get task: %v", err) + } + task.QueuedForStepID = "step1" + task.WIPAdmitted = false + if err := repo.UpdateTask(ctx, task); err != nil { + t.Fatalf("set task WIP wait: %v", err) + } + + svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) + if _, err := svc.messageQueue.QueueMessage( + ctx, + "s-queue-admission", + "t-queue-admission", + "queued while waiting for WIP admission", + "", + "user", + false, + nil, + ); err != nil { + t.Fatalf("queue message: %v", err) + } + + if drained := svc.drainQueuedBeforeWorkflowTransition( + ctx, + "t-queue-admission", + "s-queue-admission", + session, + ); drained { + t.Fatal("workflow transition was suppressed without dispatching a queued message") + } + if got := svc.messageQueue.GetStatus(ctx, "s-queue-admission").Count; got != 1 { + t.Fatalf("queue count after skipped drain = %d, want 1", got) + } +} + +func TestExecuteQueuedMessage_DiscardsReservedLifecycleMessageForRecreatedSession(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + seedExecutorRunning(t, repo, "s1", "t1", "exec-1") + session, err := repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("get session: %v", err) + } + session.State = models.TaskSessionStateWaitingForInput + if err := repo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("make session promptable: %v", err) + } + + agentMgr := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID) + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } + creator := &mockMessageCreator{} + svc.messageCreator = creator + _, _, accepted, err := svc.messageQueue.QueueLifecycleMessageWithCoalesceKey( + ctx, "s1", "t1", "must not reach replacement", "", messagequeue.QueuedByWorkflow, + false, nil, map[string]interface{}{"origin": githubPRAutomationOrigin}, + "github-pr:repo:1:merged", true, + ) + if err != nil || !accepted { + t.Fatalf("queue lifecycle message: accepted=%v err=%v", accepted, err) + } + queued, ok, _, err := svc.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + if err != nil || !ok { + t.Fatalf("reserve lifecycle message: ok=%v err=%v", ok, err) + } + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, queued) + + if _, err := repo.DB().ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + session.ID, + ); err != nil { + t.Fatalf("replace session incarnation: %v", err) + } + svc.executeQueuedMessageWithReservation("s1", queued, reservation) + + if got := len(agentMgr.capturedPrompts); got != 0 { + t.Fatalf("replacement session prompts = %d, want 0", got) + } + if got := len(creator.userMessages); got != 0 { + t.Fatalf("replacement session user messages = %d, want 0", got) + } + if svc.messageQueue.IsCurrentLifecycleReservation(ctx, queued) { + t.Fatal("stale lifecycle reservation remained hidden after worker exit") + } +} + +func TestFinishQueuedMessageExecution_SuccessLeavesRecreatedSessionQueueEmpty(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + session, err := repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("get session: %v", err) + } + + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID) + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } + svc.messageCreator = &mockMessageCreator{} + _, _, accepted, err := svc.messageQueue.QueueLifecycleMessageWithCoalesceKey( + ctx, "s1", "t1", "survive stale completion", "", messagequeue.QueuedByWorkflow, + false, nil, map[string]interface{}{"origin": githubPRAutomationOrigin}, + "github-pr:repo:1:merged", true, + ) + if err != nil || !accepted { + t.Fatalf("queue lifecycle message: accepted=%v err=%v", accepted, err) + } + queued, ok, _, err := svc.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + if err != nil || !ok || queued == nil { + t.Fatalf("reserve lifecycle message: queued=%+v ok=%v err=%v", queued, ok, err) + } + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, queued) + + if _, err := repo.DB().ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + session.ID, + ); err != nil { + t.Fatalf("replace session incarnation: %v", err) + } + svc.finishQueuedMessageExecution(ctx, "s1", "s1", queued, reservation, true, false, nil) + + replacement := identity + replacement.SessionIncarnationID = "replacement-incarnation" + entries, _, err := svc.messageQueue.SnapshotSessionForIdentity(ctx, replacement) + if err != nil { + t.Fatalf("snapshot replacement queue: %v", err) + } + if len(entries) != 0 { + t.Fatalf("replacement session retained stale queue state: %+v", entries) + } +} + func TestExecuteQueuedMessage_LifecycleClaimErrorClearsInFlightTokenForRetryDrain(t *testing.T) { ctx := context.Background() baseRepo := setupTestRepo(t) @@ -520,9 +747,9 @@ func TestHandleAgentReady_PassthroughAcknowledgesLifecycleOnlyAfterPTYAcceptance wantQueued: 0, }, { - name: "ordinary PTY failure keeps legacy destructive dequeue", + name: "ordinary PTY failure releases retained entry", passthroughErr: errors.New("PTY write failed"), - wantQueued: 0, + wantQueued: 1, }, } @@ -679,6 +906,36 @@ func TestArchiveTask_PersistentQueueCallbackDoesNotPurgeReacceptedGeneration(t * } } +func TestQueueStatusSnapshotForIdentityRejectsReplacementSession(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t-status", "s-status", "step-status") + db := sqlx.NewDb(repo.DB(), "sqlite3") + persistentRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatalf("new persistent queue repository: %v", err) + } + queue := messagequeue.NewService(persistentRepo, messagequeue.DefaultMaxPerSession, testLogger()) + identity, err := queue.ResolveSessionIdentity(ctx, "t-status", "s-status") + if err != nil { + t.Fatalf("resolve initial identity: %v", err) + } + if _, err := repo.DB().Exec( + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + "s-status", + ); err != nil { + t.Fatalf("replace session incarnation: %v", err) + } + service := &Service{repo: repo, messageQueue: queue} + if _, err := service.queueStatusSnapshotForIdentity(ctx, identity); !errors.Is( + err, + messagequeue.ErrSessionIdentityMismatch, + ) { + t.Fatalf("stale identity snapshot error = %v, want identity mismatch", err) + } +} + // Archive cancels accepted lifecycle work even when the currently selected // session is busy and therefore cannot drain it immediately. Unarchiving must // not resurrect that historical observation into a fresh agent prompt. @@ -860,8 +1117,16 @@ func TestExecuteQueuedMessage_LifecycleResetAfterClaimRestoresStateAndRetryDrain ID: "lifecycle-reset-claim", SessionID: "s1", TaskID: "t1", Content: "retry after reset", Metadata: map[string]interface{}{"origin": githubPRAutomationOrigin}, } - svc.markQueuedDispatchInFlight("s1", queued.ID) - svc.executeQueuedMessage("s1", queued) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, "t1", "s1") + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } + lock, release := svc.acquireCancelInFlightGuard("s1") + lock.Lock() + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, nil) + lock.Unlock() + release() + svc.executeQueuedMessageWithReservation("s1", queued, reservation) persisted, err := repo.GetTaskSession(ctx, "s1") if err != nil { diff --git a/apps/backend/internal/orchestrator/event_handlers_streaming.go b/apps/backend/internal/orchestrator/event_handlers_streaming.go index 495c0135d1..ba917ee1d4 100644 --- a/apps/backend/internal/orchestrator/event_handlers_streaming.go +++ b/apps/backend/internal/orchestrator/event_handlers_streaming.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/orchestrator/sessionstate" "github.com/kandev/kandev/internal/orchestrator/watcher" "github.com/kandev/kandev/internal/task/models" @@ -2335,6 +2336,54 @@ func (s *Service) writeTaskReviewStateOnCancel(ctx context.Context, taskID, sess zap.String("task_id", taskID)) } +func (s *Service) setQueuedSessionRunningForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + session *models.TaskSession, +) error { + s.taskRuntimeStateMu.Lock() + defer s.taskRuntimeStateMu.Unlock() + oldState := session.State + changed, updatedAt, err := s.repo.UpdateTaskSessionStateIfCurrentIdentity( + ctx, + identity.TaskID, + identity.SessionID, + identity.SessionIncarnationID, + oldState, + models.TaskSessionStateRunning, + "", + ) + if err != nil { + return err + } + if !changed { + return messagequeue.ErrSessionIdentityMismatch + } + session.State = models.TaskSessionStateRunning + session.ErrorMessage = "" + session.CompletedAt = nil + session.UpdatedAt = updatedAt + if oldState != models.TaskSessionStateRunning { + s.reconcileRunningTaskStateLocked(ctx, identity.TaskID, identity.SessionID) + s.clearTaskInterruptedMarker(ctx, identity.TaskID) + s.clearTaskAutoStartFailedMarker(ctx, identity.TaskID) + s.publishTaskSessionStateChanged( + ctx, + identity.TaskID, + identity.SessionID, + oldState, + models.TaskSessionStateRunning, + "", + &updatedAt, + session, + ) + s.republishTaskActivityOnSettle( + ctx, identity.TaskID, oldState, models.TaskSessionStateRunning, + ) + } + return nil +} + func (s *Service) setSessionRunning(ctx context.Context, taskID, sessionID string, preloadedSession ...*models.TaskSession) { s.setSessionRunningForExecution(ctx, taskID, sessionID, "", preloadedSession...) } diff --git a/apps/backend/internal/orchestrator/event_handlers_test.go b/apps/backend/internal/orchestrator/event_handlers_test.go index 6bb17896b3..8073767474 100644 --- a/apps/backend/internal/orchestrator/event_handlers_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_test.go @@ -843,6 +843,33 @@ func testLogger() *logger.Logger { return log } +func newAuthoritativeMemoryRepository(repo *sqliterepo.Repository) messagequeue.Repository { + return messagequeue.NewMemoryRepositoryWithAuthority( + func(ctx context.Context, taskID, sessionID string) (messagequeue.QueueSessionIdentity, error) { + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + return messagequeue.QueueSessionIdentity{}, err + } + if session.TaskID != taskID || session.QueueIncarnationID == "" { + return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch + } + return messagequeue.QueueSessionIdentity{ + TaskID: taskID, + SessionID: sessionID, + SessionIncarnationID: session.QueueIncarnationID, + }, nil + }, + ) +} + +func newAuthoritativeMemoryQueue(repo *sqliterepo.Repository, log *logger.Logger) *messagequeue.Service { + return messagequeue.NewService( + newAuthoritativeMemoryRepository(repo), + messagequeue.DefaultMaxPerSession, + log, + ) +} + func strPtr(s string) *string { return &s } // setupTestRepo creates a real in-memory SQLite repository for testing. @@ -1002,12 +1029,15 @@ func createTestServiceWithAgent(repo *sqliterepo.Repository, stepGetter *mockSte workflowStepGetter: stepGetter, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), agentFamilyResolver: agentRegistry, } repo.SetTaskQueuePurger(func(ctx context.Context, taskID string) { _, _ = svc.messageQueue.PurgeTask(ctx, taskID) }) + repo.SetTaskQueuePurgePreparer(func(_ context.Context, taskID string) { + svc.cancelPassthroughDispatches(taskID) + }) // Mirror production: after task-scoped queue purge, publish queue-status // so the status-summary projector can zero queued_prompt_count. repo.SetTaskQueuePurgeNotifier(func(ctx context.Context, taskID string) { @@ -1448,9 +1478,11 @@ func TestHandleAgentReadyGuards_ConcurrentInterruptRaces(t *testing.T) { if _, err := svc.turnService.StartTurn(ctx, "s1"); err != nil { t.Fatalf("seed original turn: %v", err) } - svc.messageQueue.SetPendingMove(ctx, "s1", &messagequeue.PendingMove{ + if err := svc.messageQueue.SetPendingMove(ctx, "s1", &messagequeue.PendingMove{ TaskID: "t1", WorkflowID: "wf1", WorkflowStepID: "step2", - }) + }); err != nil { + t.Fatalf("seed pending move: %v", err) + } turnB := raceGuardAgainstTurnReplacement(t, svc, "s1") @@ -1524,9 +1556,9 @@ func TestHandleAgentReady_DelayedOldGenerationDoesNotCompleteReplacementTurn(t * require.NoError(t, err) require.NotEqual(t, oldTurn.ID, replacementTurn.ID) if withPendingMove { - svc.messageQueue.SetPendingMove(ctx, "s1", &messagequeue.PendingMove{ + require.NoError(t, svc.messageQueue.SetPendingMove(ctx, "s1", &messagequeue.PendingMove{ TaskID: "t1", WorkflowID: "wf1", WorkflowStepID: "missing-step", - }) + })) } close(releaseOldReady) @@ -1727,7 +1759,7 @@ func TestExecuteQueuedMessage_FiresOnTurnStart(t *testing.T) { repo: repo, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), } svc.SetWorkflowStepGetter(stepGetter) svc.executor = executor.NewExecutor(agentMgr, repo, log, executor.ExecutorConfig{}) @@ -1782,7 +1814,7 @@ func TestExecuteQueuedMessage_NoOnTurnStart_StepUnchanged(t *testing.T) { repo: repo, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), } svc.SetWorkflowStepGetter(stepGetter) svc.executor = executor.NewExecutor(agentMgr, repo, log, executor.ExecutorConfig{}) @@ -1817,13 +1849,16 @@ func createTestServiceWithScheduler(repo *sqliterepo.Repository, stepGetter *moc workflowStepGetter: stepGetter, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } repo.SetTaskQueuePurger(func(ctx context.Context, taskID string) { _, _ = svc.messageQueue.PurgeTask(ctx, taskID) }) + repo.SetTaskQueuePurgePreparer(func(_ context.Context, taskID string) { + svc.cancelPassthroughDispatches(taskID) + }) // Mirror production: after task-scoped queue purge, publish queue-status // so the status-summary projector can zero queued_prompt_count. repo.SetTaskQueuePurgeNotifier(func(ctx context.Context, taskID string) { @@ -2432,6 +2467,35 @@ func TestDeliverPassthroughPrompt(t *testing.T) { t.Errorf("markPassthroughRunning should fire once before the write; got %d calls", len(agentMgr.markPassthroughCalls)) } }) + + t.Run("cancellation interrupts submit delay", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + writeCtx, cancel := context.WithCancel(context.Background()) + agentMgr := &mockAgentManager{ + isPassthrough: true, + passthroughConfig: agents.PassthroughConfig{ + Supported: true, + SubmitSequence: "\r", + DisableBracketedPaste: true, + SubmitDelay: time.Second, + }, + passthroughConfigSet: true, + } + agentMgr.passthroughStdinFunc = func(context.Context, string, string) error { + cancel() + return nil + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + + err := svc.writePassthroughPrompt(writeCtx, "s1", "hello") + if !errors.Is(err, context.Canceled) { + t.Fatalf("writePassthroughPrompt error = %v, want context cancellation", err) + } + if got := len(agentMgr.passthroughStdinCalls); got != 1 { + t.Fatalf("stdin calls after cancellation = %d, want 1", got) + } + }) } func TestHandleAgentReady_PassthroughQueuedMessage(t *testing.T) { diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow.go b/apps/backend/internal/orchestrator/event_handlers_workflow.go index 06bf71ec9e..e9269c9cb7 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow.go @@ -2559,31 +2559,59 @@ func (s *Service) reuseSessionForStepWithEndPolicy( } s.tagSessionAsWorkflowSwitched(ctx, existing.ID) - // Transfer any queued message and pending move from the session being - // switched away from to the reused session — without this, a hand-off - // prompt queued via move_task_kandev on the previous session is orphaned - // and gets delivered to the wrong agent the next time that previous - // session is reused (e.g. on the on_turn_complete bounce back). - if s.messageQueue != nil { - if err := s.messageQueue.TransferSession(ctx, currentSession.ID, existing.ID); err != nil { - // Fail closed: the workflow switch reuses an existing session, but - // orphaning a queued hand-off prompt on the previous session would - // silently misroute the next prompt. Stop here and surface the - // error so the caller can decide whether to retry. - return nil, fmt.Errorf("transfer queued state to reused session: %w", err) + // Parking can fail while persisting stop ownership. Do it before queue + // transfer so that failure leaves both source and destination queues under + // their original owners and needs no destructive reverse transfer. + if endPolicy == models.WorkflowProfileSessionEndPolicyPark { + if err := s.parkAndTransferReusedWorkflowSession(ctx, taskID, currentSession, existing.ID); err != nil { + return nil, err } + return existing, nil } - parked, err := s.finishWorkflowProfileSwitchSource(ctx, taskID, currentSession, endPolicy) - if err != nil { - if endPolicy == models.WorkflowProfileSessionEndPolicyPark && !parked && currentSession.IsPrimary { - s.restoreWorkflowProfileSwitchSourcePrimary(ctx, currentSession) - s.restoreWorkflowProfileSwitchSourceQueue(ctx, existing.ID, currentSession.ID) + if err := s.transferWorkflowProfileSwitchQueue(ctx, currentSession.ID, existing.ID); err != nil { + transferErr := fmt.Errorf("transfer queued state to reused session: %w", err) + if restoreErr := s.restoreWorkflowProfileSwitchSourcePrimary(ctx, currentSession); restoreErr != nil { + return nil, errors.Join(transferErr, restoreErr) } + return nil, transferErr + } + if _, err := s.finishWorkflowProfileSwitchSource(ctx, taskID, currentSession, endPolicy); err != nil { return nil, err } return existing, nil } +func (s *Service) parkAndTransferReusedWorkflowSession( + ctx context.Context, + taskID string, + currentSession *models.TaskSession, + destinationSessionID string, +) error { + parked, err := s.finishWorkflowProfileSwitchSource( + ctx, + taskID, + currentSession, + models.WorkflowProfileSessionEndPolicyPark, + ) + if err != nil { + if parked || !currentSession.IsPrimary { + return err + } + if restoreErr := s.restoreWorkflowProfileSwitchSourcePrimary(ctx, currentSession); restoreErr != nil { + return errors.Join(err, restoreErr) + } + return err + } + if err := s.transferWorkflowProfileSwitchQueue(ctx, currentSession.ID, destinationSessionID); err == nil { + return nil + } else { + transferErr := fmt.Errorf("transfer queued state to reused session: %w", err) + if restoreErr := s.restoreWorkflowProfileSwitchSourcePrimary(ctx, currentSession); restoreErr != nil { + return errors.Join(transferErr, restoreErr) + } + return transferErr + } +} // setNonterminalSessionPrimary promotes a workflow-reused session only when // its persisted state is still nonterminal. @@ -2629,19 +2657,26 @@ func (s *Service) createNewSessionForStepWithEndPolicy( return nil, fmt.Errorf("failed to promote new workflow session: %w", err) } - // Transfer any queued message (e.g. a move_task_kandev hand-off prompt) and - // pending move from the old session to the new one only after promotion - // succeeds. If promotion fails, the source remains primary and its queue - // must remain attached to that source rather than being stranded on an - // unpromoted destination. - s.transferWorkflowProfileSwitchQueue(ctx, currentSession.ID, newSession.ID) - - parked, err := s.finishWorkflowProfileSwitchSource(ctx, taskID, currentSession, endPolicy) - if err != nil { - if endPolicy == models.WorkflowProfileSessionEndPolicyPark && !parked && currentSession.IsPrimary { - s.restoreWorkflowProfileSwitchSourcePrimary(ctx, currentSession) - s.restoreWorkflowProfileSwitchSourceQueue(ctx, newSession.ID, currentSession.ID) + // Parking can still fail while recording stop ownership. Complete it before + // moving queue state so rollback never has to reverse a whole destination. + if endPolicy == models.WorkflowProfileSessionEndPolicyPark { + parked, err := s.finishWorkflowProfileSwitchSource(ctx, taskID, currentSession, endPolicy) + if err != nil { + if parked { + return nil, err + } + return nil, s.rollbackNewWorkflowProfileSwitch(ctx, taskID, currentSession, newSession, err) } + if err := s.transferWorkflowProfileSwitchQueue(ctx, currentSession.ID, newSession.ID); err != nil { + return nil, s.rollbackNewWorkflowProfileSwitch(ctx, taskID, currentSession, newSession, err) + } + return newSession, nil + } + + if err := s.transferWorkflowProfileSwitchQueue(ctx, currentSession.ID, newSession.ID); err != nil { + return nil, s.rollbackNewWorkflowProfileSwitch(ctx, taskID, currentSession, newSession, err) + } + if _, err := s.finishWorkflowProfileSwitchSource(ctx, taskID, currentSession, endPolicy); err != nil { return nil, err } return newSession, nil @@ -2683,7 +2718,7 @@ func (s *Service) prepareWorkflowReplacementSession( launchProfileID, err := s.resolveDynamicLaunchExecution(ctx, newSession, newAgentProfileID, true) if err != nil { resolutionErr := fmt.Errorf("failed to resolve workflow replacement profile: %w", err) - if deleteErr := s.repo.DeleteTaskSession(ctx, sessionID); deleteErr != nil { + if deleteErr := s.deleteSessionAndCleanAttachments(ctx, newSession); deleteErr != nil { s.logger.Warn("failed to delete workflow replacement after profile resolution failure", zap.String("task_id", taskID), zap.String("session_id", sessionID), @@ -2714,21 +2749,106 @@ func (s *Service) prepareWorkflowReplacementSession( return newSession, nil } -func (s *Service) transferWorkflowProfileSwitchQueue(ctx context.Context, fromSessionID, toSessionID string) { +func (s *Service) transferWorkflowProfileSwitchQueue(ctx context.Context, fromSessionID, toSessionID string) error { if s.messageQueue == nil { - return + return nil } - if err := s.messageQueue.TransferSession(ctx, fromSessionID, toSessionID); err != nil { - s.logger.Error("transfer queue to new session failed; queued prompts on the previous session will not be drained", - zap.String("from_session_id", fromSessionID), - zap.String("to_session_id", toSessionID), - zap.Error(err)) - // Continue anyway: the new session is already created and committed - // upstream. Failing closed here would leave the workflow in a - // half-switched state (new session exists but caller thinks it - // failed). The error is surfaced via logs and the orphaned entries - // stay safely in the old session for manual recovery. + if err := s.transferCurrentQueueSessions(ctx, fromSessionID, toSessionID); err != nil { + return fmt.Errorf("transfer queue to new workflow session: %w", err) } + return nil +} + +func (s *Service) rollbackNewWorkflowProfileSwitch( + ctx context.Context, + taskID string, + source, destination *models.TaskSession, + cause error, +) error { + if err := s.restoreWorkflowProfileSwitchSourcePrimary(ctx, source); err != nil { + return errors.Join(cause, err) + } + queueEmpty, inspectErr := s.workflowDestinationQueueEmpty(ctx, destination) + if inspectErr != nil { + retainErr := s.retainFailedWorkflowDestination(ctx, taskID, destination, inspectErr) + return errors.Join(cause, inspectErr, retainErr) + } + if !queueEmpty { + if err := s.retainFailedWorkflowDestination(ctx, taskID, destination, cause); err != nil { + return errors.Join(cause, err) + } + return cause + } + if s.agentManager != nil { + if err := s.agentManager.CleanupStaleExecutionBySessionID(ctx, destination.ID); err != nil { + cleanupErr := fmt.Errorf("clean up failed workflow destination runtime: %w", err) + if terminalErr := s.repo.UpdateTaskSessionState( + ctx, destination.ID, models.TaskSessionStateFailed, cleanupErr.Error(), + ); terminalErr != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("terminalize failed workflow destination: %w", terminalErr)) + } + return errors.Join(cause, cleanupErr) + } + } + if err := s.deleteSessionAndCleanAttachments(ctx, destination); err != nil { + deleteErr := fmt.Errorf("delete failed workflow destination: %w", err) + if terminalErr := s.repo.UpdateTaskSessionState( + ctx, destination.ID, models.TaskSessionStateFailed, deleteErr.Error(), + ); terminalErr != nil { + deleteErr = errors.Join(deleteErr, fmt.Errorf("terminalize failed workflow destination: %w", terminalErr)) + } + return errors.Join(cause, deleteErr) + } + s.logger.Warn("rolled back workflow profile switch after queue transfer failure", + zap.String("task_id", taskID), + zap.String("source_session_id", source.ID), + zap.String("destination_session_id", destination.ID), + zap.Error(cause)) + return cause +} + +func (s *Service) workflowDestinationQueueEmpty( + ctx context.Context, + destination *models.TaskSession, +) (bool, error) { + if s.messageQueue == nil { + return true, nil + } + identity := messagequeue.QueueSessionIdentity{ + TaskID: destination.TaskID, + SessionID: destination.ID, + SessionIncarnationID: destination.QueueIncarnationID, + } + if identity.SessionIncarnationID == "" { + var err error + identity, err = s.messageQueue.ResolveSessionIdentity(ctx, destination.TaskID, destination.ID) + if err != nil { + return false, fmt.Errorf("resolve failed workflow destination queue: %w", err) + } + } + entries, pendingMove, err := s.messageQueue.SnapshotSessionForIdentity(ctx, identity) + if err != nil { + return false, fmt.Errorf("inspect failed workflow destination queue: %w", err) + } + return len(entries) == 0 && pendingMove == nil, nil +} + +func (s *Service) retainFailedWorkflowDestination( + ctx context.Context, + taskID string, + destination *models.TaskSession, + cause error, +) error { + if err := s.repo.UpdateTaskSessionState( + ctx, destination.ID, models.TaskSessionStateFailed, cause.Error(), + ); err != nil { + return fmt.Errorf("terminalize retained workflow destination: %w", err) + } + s.logger.Warn("retained failed workflow destination for durable recovery", + zap.String("task_id", taskID), + zap.String("session_id", destination.ID), + zap.Error(cause)) + return nil } func (s *Service) finishWorkflowProfileSwitchSource( @@ -2744,24 +2864,50 @@ func (s *Service) finishWorkflowProfileSwitchSource( return true, nil } -func (s *Service) restoreWorkflowProfileSwitchSourcePrimary(ctx context.Context, session *models.TaskSession) { +func (s *Service) restoreWorkflowProfileSwitchSourcePrimary(ctx context.Context, session *models.TaskSession) error { if err := s.SetPrimarySession(ctx, session.ID); err != nil { - s.logger.Warn("failed to restore source session after parked profile switch failure", + s.logger.Warn("failed to restore source session after profile switch failure", zap.String("session_id", session.ID), zap.Error(err)) + return fmt.Errorf("restore source session primary: %w", err) } + return nil } -func (s *Service) restoreWorkflowProfileSwitchSourceQueue(ctx context.Context, fromSessionID, toSessionID string) { - if s.messageQueue == nil { - return +func (s *Service) transferCurrentQueueSessions(ctx context.Context, fromSessionID, toSessionID string) error { + from, err := s.repo.GetTaskSession(ctx, fromSessionID) + if err != nil { + return fmt.Errorf("load source queue session: %w", err) } - if err := s.messageQueue.TransferSession(ctx, fromSessionID, toSessionID); err != nil { - s.logger.Error("failed to return queued state to restored source session", - zap.String("from_session_id", fromSessionID), - zap.String("to_session_id", toSessionID), - zap.Error(err)) + if from == nil { + return fmt.Errorf("load source queue session %q: not found", fromSessionID) } + to, err := s.repo.GetTaskSession(ctx, toSessionID) + if err != nil { + return fmt.Errorf("load destination queue session: %w", err) + } + if to == nil { + return fmt.Errorf("load destination queue session %q: not found", toSessionID) + } + fromIdentity := messagequeue.QueueSessionIdentity{ + TaskID: from.TaskID, SessionID: from.ID, SessionIncarnationID: from.QueueIncarnationID, + } + if fromIdentity.SessionIncarnationID == "" { + fromIdentity, err = s.messageQueue.ResolveSessionIdentity(ctx, from.TaskID, from.ID) + if err != nil { + return fmt.Errorf("resolve source queue session: %w", err) + } + } + toIdentity := messagequeue.QueueSessionIdentity{ + TaskID: to.TaskID, SessionID: to.ID, SessionIncarnationID: to.QueueIncarnationID, + } + if toIdentity.SessionIncarnationID == "" { + toIdentity, err = s.messageQueue.ResolveSessionIdentity(ctx, to.TaskID, to.ID) + if err != nil { + return fmt.Errorf("resolve destination queue session: %w", err) + } + } + return s.messageQueue.TransferSessionIdentities(ctx, fromIdentity, toIdentity) } // completeAndStopSession stops the agent for a session and marks it COMPLETED. @@ -2988,6 +3134,10 @@ dispatchLoop: // this is this Build round's E1-only scope boundary, not a general // precondition of the marker CAS mechanism itself. func (s *Service) processOnEnter(ctx context.Context, taskID string, session *models.TaskSession, step *wfmodels.WorkflowStep, taskDescription string, entryID int64, sourceStep *wfmodels.WorkflowStep) { + // The step transition is already durable before on_enter runs. Its effects + // must finish even if the request or agent-event context that triggered the + // transition is cancelled. + ctx = context.WithoutCancel(ctx) // One GetWorkflowMeta read shared by profile resolution and prompt build. ctx = withWorkflowMetaCache(ctx) @@ -3452,56 +3602,55 @@ func (s *Service) engineOnEnterCallback(t wfmodels.OnEnterActionType) engine.Act // only when the move is already complete or rejected; unrelated queued // messages remain available for the normal drain path. func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string, session *models.TaskSession, move *messagequeue.PendingMove) { - // reinsertPendingMove restores the move so a future agent.ready can retry. - // Used on early failure paths (load errors, config issues) where the state - // hasn't been touched yet. NOT used after ApplyTransition has executed — - // at that point the workflow has either advanced or is in a corrupted state - // and re-attempting the move on the next turn would just re-trip the same - // failure (or worse, double-apply on a now-half-transitioned task). - reinsertPendingMove := func() { - if s.messageQueue == nil { - return - } - s.messageQueue.SetPendingMove(ctx, sessionID, move) + if move.SessionIncarnationID == "" && session != nil { + move.SessionIncarnationID = session.QueueIncarnationID } if s.workflowStepGetter == nil || s.workflowStore == nil { s.logger.Warn("cannot apply pending move: workflow components missing", zap.String("task_id", taskID), zap.String("session_id", sessionID)) - reinsertPendingMove() return } + record := messagequeue.PendingMoveRecord{SessionID: sessionID, Move: *move} if move.MoveID == "" { move.MoveID = legacyPendingMoveID(sessionID, move) } - task, err := s.repo.GetTask(ctx, taskID) if err != nil { s.logger.Error("failed to load task for pending move", zap.String("task_id", taskID), zap.Error(err)) - reinsertPendingMove() return } fromStepID := task.WorkflowStepID if fromStepID == move.WorkflowStepID { - if err := s.workflowStore.MarkDeferredMoveApplied(ctx, taskID, move.MoveID); errors.Is(err, errDeferredMoveAlreadyApplied) { + var applyErr error + if s.messageQueue.SupportsAtomicDeferredMoveTransition() { + applyErr = s.workflowStore.MarkDeferredMoveApplied(ctx, taskID, move.MoveID, record) + } else { + applyErr = s.workflowStore.markDeferredMoveAppliedUnfenced(ctx, taskID, move.MoveID) + } + if errors.Is(applyErr, errDeferredMoveAlreadyApplied) { s.logger.Info("dropping already-applied pending move at current target", zap.String("task_id", taskID), zap.String("move_id", move.MoveID)) - } else if err != nil { + } else if applyErr != nil { s.logger.Error("failed to record pending move at current target", - zap.String("task_id", taskID), zap.String("move_id", move.MoveID), zap.Error(err)) - reinsertPendingMove() + zap.String("task_id", taskID), zap.String("move_id", move.MoveID), zap.Error(applyErr)) return } - // Step already matches. The move is complete, so consume its hand-off - // prompt before the natural drain path handles unrelated messages. s.logger.Info("pending move target equals current step; skipping transition", zap.String("task_id", taskID), zap.String("step_id", fromStepID)) - s.removePendingMoveHandoffPrompt(ctx, sessionID, taskID, move.MoveID) - s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID) + s.removePendingMoveHandoffPromptForSession(ctx, messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + }, move.MoveID) + _, _ = s.drainQueuedMessageForPromptableSessionForIdentity(ctx, messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + }) + if !s.messageQueue.SupportsAtomicDeferredMoveTransition() { + s.consumeUnfencedPendingMove(ctx, taskID, sessionID, record) + } return } @@ -3511,7 +3660,6 @@ func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string zap.String("task_id", taskID), zap.String("target_step_id", move.WorkflowStepID), zap.Error(err)) - reinsertPendingMove() return } if targetStep.WorkflowID != move.WorkflowID { @@ -3520,14 +3668,32 @@ func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string zap.String("target_step_id", move.WorkflowStepID), zap.String("step_workflow_id", targetStep.WorkflowID), zap.String("move_workflow_id", move.WorkflowID)) - // Do NOT reinsert: the move is invalid and retrying would keep failing. - // The hand-off prompt was queued by handleMoveTask before the move was - // applied. Since the move is being dropped, the on_enter path that would - // have drained the queue won't run. Drop the orphan so it can't be - // misdelivered to the source step's agent on a future turn (it was - // authored for the move's *target* step) — mirrors the cleanup done on - // the ApplyTransition failure path below. - s.removePendingMoveHandoffPrompt(ctx, sessionID, taskID, move.MoveID) + if !s.messageQueue.SupportsAtomicDeferredMoveTransition() { + if !s.removePendingMoveHandoffPrompt(ctx, sessionID, taskID, move.MoveID) { + return + } + if _, deleteErr := s.messageQueue.DeletePendingMoveIfMatch(ctx, record, ""); deleteErr != nil { + s.logger.Warn("failed to discard invalid pending move", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(deleteErr)) + } + return + } + identity := messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + } + entryID, listed := s.pendingMoveHandoffPromptIDForSession(ctx, identity, move.MoveID) + if !listed { + return + } + removed, deleteErr := s.messageQueue.DeletePendingMoveIfMatch(ctx, record, entryID) + if deleteErr != nil { + s.logger.Warn("failed to discard invalid pending move", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(deleteErr)) + return + } + if removed && entryID != "" { + s.publishQueueStatusEventForIdentity(ctx, identity) + } return } @@ -3549,20 +3715,30 @@ func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string deferredMoveAttribution.SessionID = move.SenderSessionID } deferredMoveCtx := steptelemetry.WithAttribution(ctx, deferredMoveAttribution) - if err := s.workflowStore.ApplyDeferredMoveTransition(deferredMoveCtx, taskID, sessionID, fromStepID, move.WorkflowStepID, move.MoveID); errors.Is(err, errDeferredMoveAlreadyApplied) { + var transitionErr error + if s.messageQueue.SupportsAtomicDeferredMoveTransition() { + transitionErr = s.workflowStore.ApplyDeferredMoveTransition( + deferredMoveCtx, taskID, sessionID, fromStepID, move.WorkflowStepID, move.MoveID, record, + ) + } else { + transitionErr = s.workflowStore.applyTransition( + deferredMoveCtx, taskID, sessionID, fromStepID, move.WorkflowStepID, + engine.TriggerOnEnter, move.MoveID, nil, + ) + if transitionErr == nil { + s.consumeUnfencedPendingMove(ctx, taskID, sessionID, record) + } + } + if errors.Is(transitionErr, errDeferredMoveAlreadyApplied) { s.logger.Info("dropping already-applied pending move", zap.String("task_id", taskID), zap.String("move_id", move.MoveID)) - s.removePendingMoveHandoffPrompt(ctx, sessionID, taskID, move.MoveID) + s.removePendingMoveHandoffPromptForSession(deferredMoveCtx, messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + }, move.MoveID) return - } else if err != nil { + } else if transitionErr != nil { s.logger.Error("failed to apply pending move transition", zap.String("task_id", taskID), - zap.Error(err)) - // The hand-off prompt was queued by handleMoveTask before the move was - // applied. Now that the move failed, the on_enter path that would have - // drained the queue won't run, and handleAgentReady has already returned. - // Drop the orphan so it can't be misdelivered to the source step's agent - // on a future turn (it was authored for the move's *target* step). - s.removePendingMoveHandoffPrompt(ctx, sessionID, taskID, move.MoveID) + zap.Error(transitionErr)) return } @@ -3575,6 +3751,21 @@ func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string if move.Actor != "" { actor = wfmodels.StepTransitionActor(move.Actor) } + identity := messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + } + current, identityErr := s.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID) + if identityErr != nil || current != identity { + s.logger.Info("skipping deferred move session effects for replaced session", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.String("move_id", move.MoveID)) + return + } + freshSession, loadErr := s.repo.GetTaskSession(ctx, sessionID) + if loadErr != nil || freshSession == nil || freshSession.QueueIncarnationID != identity.SessionIncarnationID { + s.logger.Info("skipping deferred move session effects after identity changed", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.String("move_id", move.MoveID)) + return + } s.recordManualStepTransition(ctx, sessionID, fromStepID, move.WorkflowStepID, actor) s.logger.Info("applying pending move", @@ -3583,23 +3774,133 @@ func (s *Service) applyPendingMove(ctx context.Context, taskID, sessionID string zap.String("from_step_id", fromStepID), zap.String("to_step_id", move.WorkflowStepID)) if stored, loadErr := s.repo.GetTask(ctx, taskID); loadErr == nil && stored != nil && stored.QueuedForStepID == move.WorkflowStepID && !stored.WIPAdmitted { - // The destination is visible and queued, but its entry lifecycle is - // deferred until promotion. The source exit still runs once now. - go s.processStepExit(context.WithoutCancel(ctx), taskID, session, fromStepID) + go s.processStepExitForDeferredMove(context.WithoutCancel(ctx), identity, freshSession, fromStepID) return } s.syncTaskStateForPendingMove(ctx, taskID, fromStepID, move.WorkflowStepID) - - // Run on_exit + on_enter asynchronously. This call originated from - // handleAgentReady on the WS event reader goroutine; processStepExitAndEnter - // can take many seconds (resume + agentctl bootstrap) and would otherwise - // block that reader, queueing other agent events behind it and creating - // race conditions with concurrent on_turn_complete / agent.completed events - // for the same session. The DB transition is already persisted above, so - // it's safe to defer the rest. taskDescription := task.Description - go s.processStepExitAndEnter(context.WithoutCancel(ctx), taskID, session, fromStepID, move.WorkflowStepID, taskDescription) + go s.processStepExitAndEnterForDeferredMove( + context.WithoutCancel(ctx), identity, freshSession, + fromStepID, move.WorkflowStepID, taskDescription, + ) +} + +func (s *Service) consumeUnfencedPendingMove( + ctx context.Context, + taskID, sessionID string, + record messagequeue.PendingMoveRecord, +) { + removed, err := s.messageQueue.DeletePendingMoveIfMatch(ctx, record, "") + if err != nil || removed { + return + } + successor, exists, err := s.messageQueue.GetPendingMoveWithError(ctx, sessionID) + if err != nil || !exists { + return + } + task, err := s.repo.GetTask(ctx, taskID) + if err != nil || task == nil || task.WorkflowStepID != successor.WorkflowStepID { + return + } + successorRecord := messagequeue.PendingMoveRecord{SessionID: sessionID, Move: *successor} + moveID := normalizedPendingMoveID(sessionID, successor) + if err := s.workflowStore.markDeferredMoveAppliedUnfenced(ctx, taskID, moveID); err != nil && + !errors.Is(err, errDeferredMoveAlreadyApplied) { + return + } + _, _ = s.messageQueue.DeletePendingMoveIfMatch(ctx, successorRecord, "") +} + +func (s *Service) processStepExitForDeferredMove( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + session *models.TaskSession, + fromStepID string, +) { + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil || current != identity || session.QueueIncarnationID != identity.SessionIncarnationID { + return + } + s.processStepExit(ctx, identity.TaskID, session, fromStepID) +} + +func (s *Service) processStepExitAndEnterForDeferredMove( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + session *models.TaskSession, + fromStepID, toStepID, taskDescription string, +) { + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil || current != identity || session.QueueIncarnationID != identity.SessionIncarnationID { + return + } + fromStep, err := s.loadWorkflowStepForLifecycle(ctx, fromStepID, "deferred move source") + if err != nil { + return + } + s.processOnExit(ctx, identity.TaskID, session, fromStep) + + current, err = s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil || current != identity { + return + } + fresh, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil || fresh == nil || fresh.QueueIncarnationID != identity.SessionIncarnationID { + return + } + targetStep, err := s.loadWorkflowStepForLifecycle(ctx, toStepID, "deferred move target") + if err != nil { + return + } + s.processOnEnter(ctx, identity.TaskID, fresh, targetStep, taskDescription, 0, fromStep) +} + +func (s *Service) removePendingMoveHandoffPromptForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + moveID string, +) bool { + if s.messageQueue == nil { + return true + } + entryID, listed := s.pendingMoveHandoffPromptIDForSession(ctx, identity, moveID) + if !listed { + return false + } + if entryID == "" { + return true + } + _, removed, err := s.messageQueue.TakeQueuedEntryForSession(ctx, identity, entryID) + if err != nil { + return false + } + if removed { + s.publishQueueStatusEventForIdentity(ctx, identity) + } + return true +} + +func (s *Service) pendingMoveHandoffPromptIDForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + moveID string, +) (string, bool) { + entries, _, err := s.messageQueue.SnapshotSessionForIdentity(ctx, identity) + if err != nil { + return "", false + } + for _, entry := range entries { + if entry.TaskID != identity.TaskID || entry.QueuedBy != messagequeue.QueuedByMoveTask { + continue + } + entryMoveID, _ := entry.Metadata[messagequeue.MetadataDeferredMoveID].(string) + if entryMoveID != moveID && (entryMoveID != "" || !strings.HasPrefix(moveID, "legacy-")) { + continue + } + return entry.ID, true + } + return "", true } func legacyPendingMoveID(sessionID string, move *messagequeue.PendingMove) string { @@ -3840,48 +4141,74 @@ func (s *Service) drainQueuedMessageForPromptableSessionLockedWithTaskAdmission( s.isQueuedDispatchInFlight(sessionID) || s.isSteerInFlight(sessionID) { return queueDrainSkipped } - if taskID != "" { - task, err := s.repo.GetTask(ctx, taskID) - if err != nil { - s.logger.Warn("failed to reload task before admission-gated queue reservation", - zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) - return queueDrainTaskAdmissionReadFailed - } - if task == nil || (!task.WIPAdmitted && task.QueuedForStepID != "") { + if taskID == "" { + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil || session == nil { return queueDrainSkipped } + taskID = session.TaskID + } + task, err := s.repo.GetTask(ctx, taskID) + if err != nil { + s.logger.Warn("failed to reload task before admission-gated queue reservation", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) + return queueDrainTaskAdmissionReadFailed + } + if task == nil || (!task.WIPAdmitted && task.QueuedForStepID != "") { + return queueDrainSkipped + } + identity, err := s.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + return queueDrainSkipped + } + queuedMsg, ok, autoRun, err := s.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + if err != nil { + return queueDrainSkipped } - queuedMsg, ok, autoRun := s.messageQueue.ReserveQueuedWithAutoRun(ctx, sessionID) if !autoRun { return queueDrainPaused } - if s.dispatchTakenQueuedMessage(ctx, sessionID, queuedMsg, ok) { + if s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok) { return queueDrainDispatched } return queueDrainSkipped } -// dispatchTakenQueuedMessage publishes the queue-status update and dispatches -// queuedMsg for execution, given the (message, ok) pair returned by a Take* -// call on the message queue (TakeQueued or TakeQueuedEntry). Shared so -// InterruptForPeerMessage's targeted-entry take gets the same empty-message -// guard and dispatch behavior as the FIFO-head drain. -func (s *Service) dispatchTakenQueuedMessage(ctx context.Context, sessionID string, queuedMsg *messagequeue.QueuedMessage, ok bool) bool { +// dispatchTakenQueuedMessageForSession validates the captured session +// identity, publishes status, and dispatches a taken queue entry. +func (s *Service) dispatchTakenQueuedMessageForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + queuedMsg *messagequeue.QueuedMessage, + ok bool, +) bool { if !ok || queuedMsg == nil { return false } - s.publishQueueStatusEvent(ctx, sessionID) + if identity.SessionIncarnationID != "" { + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil || current != identity { + s.discardLifecycleReservationForReplacedSession(ctx, queuedMsg, identity) + return false + } + } + s.publishQueueStatusEventForIdentity(ctx, identity) if queuedMsg.Content == "" && len(queuedMsg.Attachments) == 0 { s.logger.Warn("skipping empty queued message after transition", - zap.String("session_id", sessionID), + zap.String("session_id", identity.SessionID), zap.String("queue_id", queuedMsg.ID)) return false } // Reserve entryID before handing off to the async goroutine. The worker // transitions this reservation to accepted under the same guard used by // Send Now before it performs any visible prompt side effects. - reservation := s.markQueuedDispatchInFlightWithSourceLocked(sessionID, queuedMsg.ID, queuedMsg) - go s.executeQueuedMessageWithReservation(sessionID, queuedMsg, reservation) + var reservation *queuedDispatchReservation + if identity.SessionIncarnationID != "" { + reservation = s.markQueuedDispatchInFlightWithIdentityLocked(identity, queuedMsg.ID, queuedMsg) + } else { + reservation = s.markQueuedDispatchInFlightWithSourceLocked(identity.SessionID, queuedMsg.ID, queuedMsg) + } + go s.executeQueuedMessageWithReservation(identity.SessionID, queuedMsg, reservation) return true } @@ -3930,7 +4257,18 @@ func (s *Service) writePassthroughPrompt(ctx context.Context, sessionID, content } for _, chunk := range agents.PlanPassthroughStdinChunks(content, pt) { if chunk.DelayBefore > 0 { - time.Sleep(chunk.DelayBefore) + timer := time.NewTimer(chunk.DelayBefore) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return ctx.Err() + case <-timer.C: + } } if err := s.agentManager.WritePassthroughStdin(ctx, sessionID, chunk.Data); err != nil { return fmt.Errorf("write to passthrough stdin: %w", err) diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_preflight_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_preflight_test.go index 385b522135..d91ce9a864 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_preflight_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_preflight_test.go @@ -267,7 +267,7 @@ func TestSwitchSessionForStepUsesReusableSessionExecutorProfileForCredentialAdmi exec.SetGitHubCredentialBroker(fakeSwitchSessionCredentialIssuer{}, "https://kandev.example/api/v1/github/credentials/resolve") svc := &Service{ logger: log, repo: repo, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), executor: exec, + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, } got, err := svc.switchSessionForStep(ctx, "t1", current, "profile-b") diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_rollback_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_rollback_test.go index 2ef86392da..be285ae4dc 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_rollback_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_rollback_test.go @@ -18,11 +18,13 @@ func TestSwitchSessionForStep_ParkOnEndPreservesSourceWhenPromotionFails(t *test ); err != nil { t.Fatalf("queue promotion handoff: %v", err) } - fixture.svc.messageQueue.SetPendingMove(ctx, fixture.current.ID, &messagequeue.PendingMove{ + if err := fixture.svc.messageQueue.SetPendingMove(ctx, fixture.current.ID, &messagequeue.PendingMove{ TaskID: "t1", WorkflowID: "wf1", WorkflowStepID: "step-b", - }) + }); err != nil { + t.Fatalf("set pending move: %v", err) + } promotionErr := errors.New("destination promotion failed") fixture.svc.repo = failProfileSwitchPromotionRepo{repoStore: fixture.repo, err: promotionErr} @@ -55,3 +57,201 @@ func TestSwitchSessionForStep_ParkOnEndPreservesSourceWhenPromotionFails(t *test t.Fatalf("source pending move after promotion failure = %+v exists=%t, want step-b preserved", move, exists) } } + +type failProfileSwitchQueueTransferRepository struct { + messagequeue.Repository + err error + beforeFailure func(context.Context, messagequeue.QueueSessionIdentity) error +} + +func (r *failProfileSwitchQueueTransferRepository) TransferSessionIdentities( + ctx context.Context, + _ messagequeue.QueueSessionIdentity, + destination messagequeue.QueueSessionIdentity, +) error { + if r.beforeFailure != nil { + if err := r.beforeFailure(ctx, destination); err != nil { + return err + } + } + return r.err +} + +func installFailingProfileSwitchQueue( + t *testing.T, + fixture *profileSwitchFixture, + transferErr error, +) *failProfileSwitchQueueTransferRepository { + t.Helper() + queueRepo := messagequeue.NewMemoryRepositoryWithAuthority( + func(ctx context.Context, taskID, sessionID string) (messagequeue.QueueSessionIdentity, error) { + session, err := fixture.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return messagequeue.QueueSessionIdentity{}, err + } + if session.TaskID != taskID || session.QueueIncarnationID == "" { + return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch + } + return messagequeue.QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: session.QueueIncarnationID, + }, nil + }, + ) + failingRepo := &failProfileSwitchQueueTransferRepository{Repository: queueRepo, err: transferErr} + fixture.svc.messageQueue = messagequeue.NewService( + failingRepo, + messagequeue.DefaultMaxPerSession, + fixture.svc.logger, + ) + return failingRepo +} + +func queueProfileSwitchHandoff(t *testing.T, fixture *profileSwitchFixture, content string) { + t.Helper() + if _, err := fixture.svc.messageQueue.QueueMessage( + context.Background(), fixture.current.ID, fixture.current.TaskID, + content, "", messagequeue.QueuedByUser, false, nil, + ); err != nil { + t.Fatalf("queue profile-switch handoff: %v", err) + } +} + +func TestCreateNewSessionForStep_TransferFailureRestoresSourceAndRemovesDestination(t *testing.T) { + ctx := context.Background() + fixture := newProfileSwitchFixture( + t, + models.WorkflowProfileSessionStartPolicyNew, + models.WorkflowProfileSessionEndPolicyComplete, + ) + transferErr := errors.New("queue transfer failed") + installFailingProfileSwitchQueue(t, fixture, transferErr) + queueProfileSwitchHandoff(t, fixture, "new-session handoff") + + _, err := fixture.svc.createNewSessionForStepWithEndPolicy( + ctx, "t1", fixture.current, "profile-b", fixture.endPolicy, + ) + if !errors.Is(err, transferErr) { + t.Fatalf("profile switch error = %v, want queue transfer failure", err) + } + + source, err := fixture.repo.GetTaskSession(ctx, fixture.current.ID) + if err != nil { + t.Fatalf("reload source: %v", err) + } + if source.State != models.TaskSessionStateRunning || !source.IsPrimary || source.CompletedAt != nil { + t.Fatalf("source after transfer failure = state %s primary %t completed %v", source.State, source.IsPrimary, source.CompletedAt) + } + sessions, err := fixture.repo.ListTaskSessions(ctx, fixture.current.TaskID) + if err != nil { + t.Fatalf("list task sessions: %v", err) + } + if len(sessions) != 1 || sessions[0].ID != fixture.current.ID { + t.Fatalf("sessions after transfer failure = %+v, want only source", sessions) + } + status := fixture.svc.messageQueue.GetStatus(ctx, fixture.current.ID) + if status.Count != 1 || status.Entries[0].Content != "new-session handoff" { + t.Fatalf("source queue after transfer failure = %+v", status.Entries) + } +} + +func TestReuseSessionForStep_TransferFailureRestoresSourcePrimary(t *testing.T) { + ctx := context.Background() + fixture := newProfileSwitchFixture( + t, + models.WorkflowProfileSessionStartPolicyReuse, + models.WorkflowProfileSessionEndPolicyPark, + ) + destination := &models.TaskSession{ + ID: "session-b-existing", TaskID: fixture.current.TaskID, AgentProfileID: "profile-b", + ExecutorID: "exec-local", ExecutorProfileID: "ep1", + State: models.TaskSessionStateWaitingForInput, StartedAt: fixture.current.StartedAt, + UpdatedAt: fixture.current.UpdatedAt, + } + if err := fixture.repo.CreateTaskSession(ctx, destination); err != nil { + t.Fatalf("create reusable destination: %v", err) + } + transferErr := errors.New("queue transfer failed") + installFailingProfileSwitchQueue(t, fixture, transferErr) + queueProfileSwitchHandoff(t, fixture, "reuse-session handoff") + + _, err := fixture.svc.reuseSessionForStepWithEndPolicy( + ctx, fixture.current.TaskID, fixture.current, destination, fixture.endPolicy, + ) + if !errors.Is(err, transferErr) { + t.Fatalf("profile switch error = %v, want queue transfer failure", err) + } + + source, err := fixture.repo.GetTaskSession(ctx, fixture.current.ID) + if err != nil { + t.Fatalf("reload source: %v", err) + } + if source.State != models.TaskSessionStateWaitingForInput || !source.IsPrimary { + t.Fatalf("source after transfer failure = state %s primary %t, want waiting primary", source.State, source.IsPrimary) + } + reused, err := fixture.repo.GetTaskSession(ctx, destination.ID) + if err != nil { + t.Fatalf("reload reusable destination: %v", err) + } + if reused.IsPrimary || reused.State != models.TaskSessionStateWaitingForInput { + t.Fatalf("destination after transfer failure = state %s primary %t", reused.State, reused.IsPrimary) + } + status := fixture.svc.messageQueue.GetStatus(ctx, fixture.current.ID) + if status.Count != 1 || status.Entries[0].Content != "reuse-session handoff" { + t.Fatalf("source queue after transfer failure = %+v", status.Entries) + } +} + +func TestCreateNewSessionForStep_TransferFailurePreservesDestinationArrival(t *testing.T) { + ctx := context.Background() + fixture := newProfileSwitchFixture( + t, + models.WorkflowProfileSessionStartPolicyNew, + models.WorkflowProfileSessionEndPolicyComplete, + ) + transferErr := errors.New("queue transfer failed") + queueRepo := installFailingProfileSwitchQueue(t, fixture, transferErr) + queueRepo.beforeFailure = func( + ctx context.Context, + destination messagequeue.QueueSessionIdentity, + ) error { + return queueRepo.InsertForSession(ctx, destination, &messagequeue.QueuedMessage{ + ID: "destination-arrival", + SessionID: destination.SessionID, + TaskID: destination.TaskID, + Content: "arrived during switch", + QueuedBy: messagequeue.QueuedByUser, + }, messagequeue.DefaultMaxPerSession) + } + queueProfileSwitchHandoff(t, fixture, "source handoff") + + _, err := fixture.svc.createNewSessionForStepWithEndPolicy( + ctx, fixture.current.TaskID, fixture.current, "profile-b", fixture.endPolicy, + ) + if !errors.Is(err, transferErr) { + t.Fatalf("profile switch error = %v, want queue transfer failure", err) + } + + source, err := fixture.repo.GetTaskSession(ctx, fixture.current.ID) + if err != nil || !source.IsPrimary { + t.Fatalf("source after transfer failure = %+v err=%v, want primary", source, err) + } + sessions, err := fixture.repo.ListTaskSessions(ctx, fixture.current.TaskID) + if err != nil { + t.Fatalf("list task sessions: %v", err) + } + if len(sessions) != 2 { + t.Fatalf("session count after destination arrival = %d, want 2", len(sessions)) + } + for _, session := range sessions { + if session.ID == source.ID { + continue + } + if session.State != models.TaskSessionStateFailed || session.IsPrimary { + t.Fatalf("retained destination = state %s primary %t, want failed nonprimary", session.State, session.IsPrimary) + } + status := fixture.svc.messageQueue.GetStatus(ctx, session.ID) + if status.Count != 1 || status.Entries[0].ID != "destination-arrival" { + t.Fatalf("retained destination queue = %+v", status.Entries) + } + } +} diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_test.go index e73f1e25c3..bca8934020 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_session_policy_test.go @@ -110,9 +110,11 @@ func newProfileSwitchFixture(t *testing.T, startPolicy models.WorkflowProfileSes return &executor.LaunchAgentResponse{AgentExecutionID: "execution-" + req.AgentProfileID}, nil }, } + svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, svc.logger) return &profileSwitchFixture{ repo: repo, stepGetter: stepGetter, current: current, agentMgr: agentMgr, startPolicy: startPolicy, endPolicy: endPolicy, - svc: createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr), + svc: svc, } } @@ -793,6 +795,11 @@ func TestSwitchSessionForStep_ParkOnEndRestoresQueueAfterReuseParkingFails(t *te ); err != nil { t.Fatalf("queue reuse handoff: %v", err) } + if _, err := fixture.svc.messageQueue.QueueMessage( + ctx, fixture.current.ID, "t1", "destination arrival", "", messagequeue.QueuedByUser, false, nil, + ); err != nil { + t.Fatalf("queue destination arrival: %v", err) + } fixture.svc.repo = failParkIntentRepo{repoStore: fixture.repo, remover: fixture.repo, marker: fixture.repo} fixture.stepGetter.workflowAgentProfileID = "profile-a" @@ -815,8 +822,8 @@ func TestSwitchSessionForStep_ParkOnEndRestoresQueueAfterReuseParkingFails(t *te t.Fatalf("source queue after reuse parking failure = %+v, want queued reuse handoff restored", status.Entries) } originalAStatus := fixture.svc.messageQueue.GetStatus(ctx, fixture.current.ID) - if originalAStatus.Count != 0 { - t.Fatalf("reused destination queue after rollback = %+v, want empty", originalAStatus.Entries) + if originalAStatus.Count != 1 || originalAStatus.Entries[0].Content != "destination arrival" { + t.Fatalf("reused destination queue after rollback = %+v, want destination arrival preserved", originalAStatus.Entries) } } diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_test.go index 1781246cbd..87dae22a38 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_profile_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_profile_test.go @@ -909,7 +909,7 @@ func TestSwitchSessionForStep(t *testing.T) { workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -1023,7 +1023,7 @@ func TestSwitchSessionForStep_ReusesNonterminalSession(t *testing.T) { workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, taskEvents: publisher, @@ -1112,7 +1112,7 @@ func TestSwitchSessionForStep_CreatesFreshSessionWhenCandidateTerminalizesBefore exec := executor.NewExecutor(agentMgr, repo, log, executor.ExecutorConfig{}) svc := &Service{ logger: log, workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), executor: exec, + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: scheduler.NewScheduler(queue.NewTaskQueue(100), exec, taskRepo, log, scheduler.SchedulerConfig{}), } barrierRepo := &terminalizeCandidateBeforePromotionRepo{ @@ -1243,7 +1243,7 @@ func TestSwitchSessionForStep_CompletedSessionNotReused(t *testing.T) { workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -1406,7 +1406,7 @@ func TestSwitchSessionForStep_FailedSessionNotReused(t *testing.T) { workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -1520,7 +1520,7 @@ func TestProcessOnEnter_ProfileSwitch(t *testing.T) { workflowStepGetter: sg, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -1799,7 +1799,7 @@ func TestProcessOnEnter_ProfileSwitch(t *testing.T) { workflowStepGetter: sg, taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } @@ -1876,7 +1876,7 @@ func TestSwitchSessionForStep_PreservesOldSessionOnFailure(t *testing.T) { workflowStepGetter: newMockStepGetter(), taskRepo: taskRepo, agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: exec, scheduler: sched, } diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_prompt_claim_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_prompt_claim_test.go index 2daeb46e53..c08ad19b94 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_prompt_claim_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_prompt_claim_test.go @@ -190,7 +190,7 @@ func TestWorkflowAutoStartPromptClaimRejectsSessionTerminalizedAfterPromotion(t errCh := make(chan error, 1) go func() { _, _, _, _, _, claimErr := svc.claimSessionRunningForPrompt( - ctx, session.TaskID, session.ID, "", false, nil, nil, "", true, + ctx, session.TaskID, session.ID, "", false, nil, nil, "", true, nil, ) errCh <- claimErr }() @@ -227,7 +227,7 @@ func TestWorkflowAutoStartRejectsSessionTerminalizedBeforeResume(t *testing.T) { allowClaim: make(chan struct{}), } agentMgr := &mockAgentManager{repoForExecutionLookup: repo} - queue := messagequeue.NewServiceMemory(testLogger()) + queue := newAuthoritativeMemoryQueue(repo, testLogger()) if _, err := queue.QueueMessage(ctx, session.ID, session.TaskID, "handoff", "", messagequeue.QueuedByUser, true, nil); err != nil { t.Fatalf("queue handoff: %v", err) } @@ -277,7 +277,7 @@ func TestWorkflowAutoStartRollsBackPromptClaimWhenPostClaimReloadFails(t *testin err: reloadErr, } agentMgr := &mockAgentManager{repoForExecutionLookup: repo, isAgentRunning: true} - queue := messagequeue.NewServiceMemory(testLogger()) + queue := newAuthoritativeMemoryQueue(repo, testLogger()) if _, err := queue.QueueMessage(ctx, session.ID, session.TaskID, "handoff", "", messagequeue.QueuedByUser, true, nil); err != nil { t.Fatalf("queue handoff: %v", err) } @@ -333,7 +333,7 @@ func TestWorkflowAutoStartRollsBackPromptClaimWhenTerminalizedAfterClaim(t *test allowReload: make(chan struct{}), } agentMgr := &mockAgentManager{repoForExecutionLookup: repo, isAgentRunning: true} - queue := messagequeue.NewServiceMemory(testLogger()) + queue := newAuthoritativeMemoryQueue(repo, testLogger()) if _, err := queue.QueueMessage(ctx, session.ID, session.TaskID, "handoff", "", messagequeue.QueuedByUser, true, nil); err != nil { t.Fatalf("queue handoff: %v", err) } diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_reset_quiescence_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_reset_quiescence_test.go index 89b724e740..e67c0bb8c3 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_reset_quiescence_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_reset_quiescence_test.go @@ -30,6 +30,30 @@ func (r lifecycleClaimSignalRepo) ClaimPromptableTaskSessionIfActive( return claim, err } +func (r lifecycleClaimSignalRepo) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + claim, err := r.repoStore.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, taskID, sessionID, incarnationID, + ) + if err == nil && claim.Status == models.PromptableTaskSessionClaimed { + r.claimed <- struct{}{} + } + return claim, err +} + +func (r lifecycleClaimSignalRepo) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, sessionID, incarnationID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + return r.repoStore.UpdateTaskSessionStateIfCurrentIdentity( + ctx, taskID, sessionID, incarnationID, expected, state, errorMessage, + ) +} + type orderedResetAgentManager struct { *mockAgentManager events chan string @@ -245,7 +269,7 @@ func TestResetAgentContext_ResetMarkerPrecedesCancellationWait(t *testing.T) { cancelledCtx, cancel := context.WithCancel(ctx) cancel() _, _, _, _, _, err := svc.claimSessionRunningForPrompt( - cancelledCtx, session.TaskID, session.ID, "", false, nil, nil, "", false, + cancelledCtx, session.TaskID, session.ID, "", false, nil, nil, "", false, nil, ) if !errors.Is(err, ErrSessionResetInProgress) { t.Fatalf("prompt admission error = %v, want %v", err, ErrSessionResetInProgress) @@ -318,7 +342,7 @@ func TestResetAgentContext_SerializesPromptAdmission(t *testing.T) { go func() { close(promptStarted) _, _, _, _, _, err := svc.claimSessionRunningForPrompt( - ctx, session.TaskID, session.ID, "", false, nil, nil, "", false, + ctx, session.TaskID, session.ID, "", false, nil, nil, "", false, nil, ) promptDone <- err }() diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction.go b/apps/backend/internal/orchestrator/executor/executor_interaction.go index 88e6c185f0..e43fcdfadf 100644 --- a/apps/backend/internal/orchestrator/executor/executor_interaction.go +++ b/apps/backend/internal/orchestrator/executor/executor_interaction.go @@ -379,6 +379,9 @@ func (e *Executor) prompt(ctx context.Context, taskID, sessionID string, prompt if err != nil || executionID == "" { return nil, ErrExecutionNotFound } + if session.AgentExecutionID != "" && executionID != session.AgentExecutionID { + return nil, ErrExecutionNotFound + } e.logger.Debug("sending prompt to agent", zap.String("task_id", taskID), diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction_test.go b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go index 889703e990..24118cfb82 100644 --- a/apps/backend/internal/orchestrator/executor/executor_interaction_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go @@ -769,3 +769,28 @@ func TestSwitchModelFallback_PreservesExecutorProfileMetadata(t *testing.T) { t.Fatalf("user namespace metadata = %q, want profile value true", got) } } + +func TestExecutorPromptRejectsRemappedPreloadedExecution(t *testing.T) { + repo := newMockRepository() + manager := &mockAgentManager{ + isPassthroughSessionFunc: func(context.Context, string) bool { return false }, + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-replacement", nil + }, + } + exec := newTestExecutor(t, manager, repo) + preloaded := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", AgentExecutionID: "execution-original", + State: models.TaskSessionStateWaitingForInput, + } + + _, err := exec.Prompt( + context.Background(), "task-1", "session-1", "prompt", nil, false, preloaded, + ) + if !errors.Is(err, ErrExecutionNotFound) { + t.Fatalf("expected stale execution rejection, got %v", err) + } + if manager.promptAgentCallCount != 0 { + t.Fatalf("stale preloaded session dispatched %d prompts", manager.promptAgentCallCount) + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index c826ad0681..87a2e62245 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -991,7 +991,9 @@ func (m *mockRepository) CountActiveTaskSessionsByRepository(ctx context.Context func (m *mockRepository) DeleteEphemeralTasksByAgentProfile(ctx context.Context, agentProfileID string) (int64, error) { return 0, nil } -func (m *mockRepository) DeleteTaskSession(ctx context.Context, id string) error { return nil } +func (m *mockRepository) DeleteTaskSession(ctx context.Context, session *models.TaskSession) error { + return nil +} // Workflow-related session operations func (m *mockRepository) GetPrimarySessionByTaskID(ctx context.Context, taskID string) (*models.TaskSession, error) { diff --git a/apps/backend/internal/orchestrator/handlers/queue_handlers.go b/apps/backend/internal/orchestrator/handlers/queue_handlers.go index afe0b9d8f9..3f43efe375 100644 --- a/apps/backend/internal/orchestrator/handlers/queue_handlers.go +++ b/apps/backend/internal/orchestrator/handlers/queue_handlers.go @@ -8,7 +8,6 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/entityrefs" - "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/orchestrator" "github.com/kandev/kandev/internal/orchestrator/messagequeue" @@ -45,10 +44,14 @@ const ( queueAccessDenied = "Session not found" // Payload field names — extracted to satisfy goconst (≥3 occurrences). - fieldSessionID = "session_id" - fieldEntryID = "entry_id" - fieldQueueSize = "queue_size" - fieldMax = "max" + fieldTaskID = "task_id" + fieldSessionID = "session_id" + fieldSessionIncarnation = "session_incarnation_id" + fieldEntryID = "entry_id" + fieldQueueSize = "queue_size" + fieldMax = "max" + fieldAutoRun = "auto_run" + fieldAutoMergeEnabled = "auto_merge_enabled" ) // QueueService is the surface the handlers depend on. Real implementation lives @@ -65,17 +68,84 @@ type QueueService interface { CancelAll(ctx context.Context, sessionID string) (int, error) GetStatus(ctx context.Context, sessionID string) *messagequeue.QueueStatus } +type QueueSnapshotService interface { + Snapshot(context.Context, messagequeue.QueueSessionIdentity) (*messagequeue.QueueStatus, error) +} +type QueueIdentityAdmissionService interface { + QueueMessageWithMetadataForSession( + context.Context, + messagequeue.QueueSessionIdentity, + string, + string, + string, + bool, + []messagequeue.MessageAttachment, + map[string]interface{}, + ) (*messagequeue.QueuedMessage, error) + QueueMessageWithMetadataForSessionAfterInsert( + context.Context, + messagequeue.QueueSessionIdentity, + string, + string, + string, + bool, + []messagequeue.MessageAttachment, + map[string]interface{}, + func(context.Context, *messagequeue.QueuedMessage) error, + ) (*messagequeue.QueuedMessage, error) +} +type QueueIdentityAttachmentAdmissionService interface { + QueueMessageWithMetadataForSessionWithClaim( + context.Context, + messagequeue.QueueSessionIdentity, + string, + string, + string, + bool, + []messagequeue.MessageAttachment, + map[string]interface{}, + messagequeue.QueueAttachmentClaim, + ) (*messagequeue.QueuedMessage, error) +} + +type QueueIdentityMutationService interface { + AppendContentForSession(context.Context, messagequeue.QueueSessionIdentity, string, string, string, bool, []messagequeue.MessageAttachment) (*messagequeue.QueuedMessage, bool, error) + UpdateMessageWithMetadataForSession(context.Context, messagequeue.QueueSessionIdentity, string, string, []messagequeue.MessageAttachment, map[string]interface{}, string) error + RemoveEntryForSession(context.Context, messagequeue.QueueSessionIdentity, string) (*messagequeue.QueueRemovalResult, error) + MergeIntoAboveForSession(context.Context, messagequeue.QueueSessionIdentity, string, string) (*messagequeue.QueuedMessage, error) + ReorderEntriesForSession(context.Context, messagequeue.QueueSessionIdentity, []string) error + CancelAllForSession(context.Context, messagequeue.QueueSessionIdentity) (*messagequeue.QueueRemovalResult, error) +} + +type QueueIdentityEntryService interface { + GetEntryForSession(context.Context, messagequeue.QueueSessionIdentity, string) (*messagequeue.QueuedMessage, error) +} + +type QueueIdentityAttachmentMutationService interface { + UpdateMessageWithMetadataForSessionWithClaim(context.Context, messagequeue.QueueSessionIdentity, string, string, []messagequeue.MessageAttachment, map[string]interface{}, string, messagequeue.QueueAttachmentClaim) error +} // QueueDrainer drains a single queued entry when the session is promptable. type QueueDrainer interface { DrainQueuedMessage(ctx context.Context, sessionID string) (bool, error) } +type QueueIdentityDrainer interface { + DrainQueuedMessageForSession(context.Context, messagequeue.QueueSessionIdentity) (bool, error) +} // QueueAutoRunController persists queue policy and may immediately dispatch // one FIFO head when enabling an eligible session. type QueueAutoRunController interface { SetQueueAutoRun(ctx context.Context, sessionID string, enabled bool) (autoRun bool, dispatched bool, err error) } +type QueueIdentityAutoRunController interface { + SetQueueAutoRunForSession(context.Context, messagequeue.QueueSessionIdentity, bool) (bool, bool, error) +} + +// QueueAutoMergeController persists and resolves per-session automatic-merge policy. +type QueueAutoMergeController interface { + SetSessionAutoMerge(context.Context, messagequeue.QueueSessionIdentity, bool) (messagequeue.AutoMergePolicy, error) +} // QueueSendNowDispatcher is implemented by the orchestrator service. It is // kept separate from QueueDrainer so queue-focused handlers can retain their @@ -83,12 +153,18 @@ type QueueAutoRunController interface { type QueueSendNowDispatcher interface { SendQueuedNow(ctx context.Context, sessionID, scope, entryID string) (int, error) } +type QueueIdentitySendNowDispatcher interface { + SendQueuedNowForSession(context.Context, messagequeue.QueueSessionIdentity, string, string) (int, error) +} // QueueAccessAuthorizer scopes queue reads and mutations to visible sessions. type QueueAccessAuthorizer interface { AuthorizeSessionAccess(ctx context.Context, sessionID string) error AuthorizeTaskSessionAccess(ctx context.Context, taskID, sessionID string) error } +type QueueSessionIdentityAuthorizer interface { + AuthorizeTaskSessionIncarnationAccess(ctx context.Context, taskID, sessionID, incarnationID string) error +} // SessionTaskResolver returns the task that owns a session. It enriches the // message.queue.status_changed event with task_id so task-scoped consumers @@ -102,6 +178,10 @@ type QueueAttachmentClaimer interface { ClaimMessageAttachments(ctx context.Context, taskID, sessionID string, attachments []v1.MessageAttachment) error } +type QueueAttachmentClaimPreparer interface { + PrepareQueueAttachmentClaim(context.Context, string, []v1.MessageAttachment) (messagequeue.QueueAttachmentClaim, error) +} + type QueueAttachmentReleaser interface { ReleaseMessageAttachments(ctx context.Context, taskID, sessionID string, attachments []v1.MessageAttachment) error } @@ -174,21 +254,23 @@ func (h *QueueHandlers) RegisterHandlers(d *ws.Dispatcher) { d.RegisterFunc(ws.ActionMessageQueueDrain, h.wsDrainQueue) d.RegisterFunc(ws.ActionMessageQueueSendNow, h.wsSendNow) d.RegisterFunc(ws.ActionMessageQueueAutoRunSet, h.wsSetAutoRun) + d.RegisterFunc(ws.ActionMessageQueueAutoMergeSet, h.wsSetAutoMerge) d.RegisterFunc(ws.ActionMessageQueueRemove, h.wsRemoveEntry) d.RegisterFunc(ws.ActionMessageQueueMerge, h.wsMergeIntoAbove) d.RegisterFunc(ws.ActionMessageQueueReorder, h.wsReorder) } type wsQueueMessageRequest struct { - SessionID string `json:"session_id"` - TaskID string `json:"task_id"` - Content string `json:"content"` - Model string `json:"model,omitempty"` - PlanMode bool `json:"plan_mode,omitempty"` - Attachments []messagequeue.MessageAttachment `json:"attachments,omitempty"` - ContextFiles []v1.ContextFileMeta `json:"context_files,omitempty"` - EntityReferences []v1.EntityReference `json:"entity_references,omitempty"` - UserID string `json:"user_id,omitempty"` + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + Content string `json:"content"` + Model string `json:"model,omitempty"` + PlanMode bool `json:"plan_mode,omitempty"` + Attachments []messagequeue.MessageAttachment `json:"attachments,omitempty"` + ContextFiles []v1.ContextFileMeta `json:"context_files,omitempty"` + EntityReferences []v1.EntityReference `json:"entity_references,omitempty"` + UserID string `json:"user_id,omitempty"` } // wsQueueMessage handles ActionMessageQueueAdd, appending a new entry to the session queue. @@ -204,7 +286,10 @@ func (h *QueueHandlers) wsQueueMessage(ctx context.Context, msg *ws.Message) (*w if req.TaskID == "" { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id is required", nil) } - if denied := h.authorizeTaskSession(ctx, msg, req.TaskID, req.SessionID); denied != nil { + if h.requiresQueueIdentity() && req.SessionIncarnationID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { return denied, nil } if req.Content == "" && len(req.Attachments) == 0 { @@ -241,12 +326,9 @@ func (h *QueueHandlers) wsQueueMessage(ctx context.Context, msg *ws.Message) (*w queued, err := h.admitQueuedMessage(ctx, &req, queuedBy, metadata) if err != nil { if errors.Is(err, messagequeue.ErrQueueFull) { - status := h.queueService.GetStatus(ctx, req.SessionID) - return ws.NewError(msg.ID, msg.Action, messagequeue.QueueFullErrorCode, "Queue is full", - map[string]interface{}{ - fieldQueueSize: status.Count, - fieldMax: status.Max, - }) + return h.queueFullResponse(ctx, msg, messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + }) } if errors.Is(err, messagequeue.ErrTaskInactive) { // The task was archived or deleted between the caller's @@ -264,7 +346,7 @@ func (h *QueueHandlers) wsQueueMessage(ctx context.Context, msg *ws.Message) (*w return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to queue message", nil) } - h.publishStatus(ctx, req.SessionID, queued) + h.publishStatusForIdentity(ctx, messagequeue.QueueSessionIdentity{TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID}, queued) return ws.NewResponse(msg.ID, msg.Action, queued) } @@ -274,243 +356,118 @@ var ( ) func (h *QueueHandlers) admitQueuedMessage(ctx context.Context, req *wsQueueMessageRequest, queuedBy string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, error) { - if h.attachmentClaimer == nil || len(req.Attachments) == 0 { - return h.queueService.QueueMessageWithMetadata( + if !h.requiresQueueIdentity() { + if h.attachmentClaimer == nil || len(req.Attachments) == 0 { + return h.queueService.QueueMessageWithMetadata( + ctx, req.SessionID, req.TaskID, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, + ) + } + return h.queueService.QueueMessageWithMetadataAfterInsert( ctx, req.SessionID, req.TaskID, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, + h.claimQueuedAttachmentsAfterInsert(req), ) } - return h.queueService.QueueMessageWithMetadataAfterInsert( - ctx, req.SessionID, req.TaskID, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, - func(admittedCtx context.Context, source *messagequeue.QueuedMessage) error { - claimErr := h.attachmentClaimer.ClaimMessageAttachments( - admittedCtx, req.TaskID, req.SessionID, queueAttachmentsToV1(req.Attachments), - ) - if claimErr == nil { - return nil - } - if rollbackErr := h.rollbackQueuedAttachmentClaim(admittedCtx, req.SessionID, source.ID); rollbackErr != nil { - h.logger.Error("failed to roll back queued attachment", zap.Error(rollbackErr)) - return fmt.Errorf("%w: %v", errQueuedAttachmentRollback, rollbackErr) - } - return fmt.Errorf("%w: %v", errQueuedAttachmentUnavailable, claimErr) - }, + admissions, ok := h.queueService.(QueueIdentityAdmissionService) + if !ok { + return nil, errors.New("identity-bound queue admission is unavailable") + } + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + if h.attachmentClaimer == nil || len(req.Attachments) == 0 { + return admissions.QueueMessageWithMetadataForSession( + ctx, identity, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, + ) + } + if preparer, ok := h.attachmentClaimer.(QueueAttachmentClaimPreparer); ok { + claim, err := preparer.PrepareQueueAttachmentClaim(ctx, req.TaskID, queueAttachmentsToV1(req.Attachments)) + if err != nil { + return nil, fmt.Errorf("%w: %v", errQueuedAttachmentUnavailable, err) + } + atomicAdmissions, ok := h.queueService.(QueueIdentityAttachmentAdmissionService) + if !ok { + return nil, errors.New("transactional attachment admission is unavailable") + } + return atomicAdmissions.QueueMessageWithMetadataForSessionWithClaim( + ctx, identity, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, claim, + ) + } + return admissions.QueueMessageWithMetadataForSessionAfterInsert( + ctx, identity, req.Content, req.Model, queuedBy, req.PlanMode, req.Attachments, metadata, + h.claimQueuedAttachmentsAfterInsert(req), ) } -type wsCancelAllRequest struct { - SessionID string `json:"session_id"` -} - -// wsCancelAll handles ActionMessageQueueCancel, clearing every pending entry for a session. -func (h *QueueHandlers) wsCancelAll(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsCancelAllRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - - removed, err := h.queueService.CancelAll(ctx, req.SessionID) - if err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ - fieldSessionID: req.SessionID, - "removed": removed, - }) -} - -type wsDrainQueueRequest struct { - SessionID string `json:"session_id"` -} - -// wsDrainQueue handles ActionMessageQueueDrain, dispatching one queued entry when the session is promptable. -func (h *QueueHandlers) wsDrainQueue(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsDrainQueueRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if h.queueDrainer == nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue drain is unavailable", nil) - } - - drained, err := h.queueDrainer.DrainQueuedMessage(ctx, req.SessionID) - if err != nil { - switch { - case errors.Is(err, orchestrator.ErrAgentPromptInProgress): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSessionBusy, "Session is busy", nil) - case errors.Is(err, orchestrator.ErrSessionNotPromptable): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeNotPromptable, "Session is not ready for input", nil) - default: - h.logger.Error("failed to drain queued message", zap.String(fieldSessionID, req.SessionID), zap.Error(err)) - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to drain queued message", nil) +func (h *QueueHandlers) claimQueuedAttachmentsAfterInsert(req *wsQueueMessageRequest) func(context.Context, *messagequeue.QueuedMessage) error { + return func(admittedCtx context.Context, source *messagequeue.QueuedMessage) error { + claimErr := h.attachmentClaimer.ClaimMessageAttachments( + admittedCtx, req.TaskID, req.SessionID, queueAttachmentsToV1(req.Attachments), + ) + if claimErr == nil { + return nil } + if rollbackErr := h.rollbackQueuedAttachmentClaim(admittedCtx, req.SessionID, source.ID); rollbackErr != nil { + h.logger.Error("failed to roll back queued attachment", zap.Error(rollbackErr)) + return fmt.Errorf("%w: %v", errQueuedAttachmentRollback, rollbackErr) + } + return fmt.Errorf("%w: %v", errQueuedAttachmentUnavailable, claimErr) } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ - fieldSessionID: req.SessionID, - "drained": drained, - }) -} - -type wsSetAutoRunRequest struct { - SessionID string `json:"session_id"` - Enabled *bool `json:"enabled"` -} - -// wsSetAutoRun persists automatic queue processing and optionally starts the -// promptable FIFO head when enabling it. -func (h *QueueHandlers) wsSetAutoRun(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsSetAutoRunRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if req.Enabled == nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "enabled is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if h.queueAutoRun == nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue Auto-run is unavailable", nil) - } - autoRun, dispatched, err := h.queueAutoRun.SetQueueAutoRun(ctx, req.SessionID, *req.Enabled) - if err != nil { - h.logger.Error("failed to set queue Auto-run", zap.String(fieldSessionID, req.SessionID), zap.Error(err)) - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to set queue Auto-run", nil) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ - fieldSessionID: req.SessionID, - "auto_run": autoRun, - "dispatched": dispatched, - }) -} - -type wsSendNowRequest struct { - SessionID string `json:"session_id"` - Scope string `json:"scope"` - EntryID string `json:"entry_id,omitempty"` -} - -// wsSendNow handles ActionMessageQueueSendNow, interrupting the active turn with an exact queue selection. -func (h *QueueHandlers) wsSendNow(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsSendNowRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if validation := validateSendNowRequest(req); validation != "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, validation, nil) - } - if h.queueDispatcher == nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue send-now is unavailable", nil) - } - - sentCount, err := h.queueDispatcher.SendQueuedNow(ctx, req.SessionID, req.Scope, req.EntryID) - if err != nil { - return h.sendNowErrorResponse(msg, req.SessionID, err) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ - fieldSessionID: req.SessionID, - "dispatched": true, - "sent_count": sentCount, - }) -} - -// validateSendNowRequest validates the send-now scope and entry id combination. -func validateSendNowRequest(req wsSendNowRequest) string { - switch { - case req.Scope != orchestrator.QueueSendNowScopeEntry && req.Scope != orchestrator.QueueSendNowScopeAll: - return "scope must be entry or all" - case req.Scope == orchestrator.QueueSendNowScopeEntry && req.EntryID == "": - return "entry_id is required for entry scope" - case req.Scope == orchestrator.QueueSendNowScopeAll && req.EntryID != "": - return "entry_id is not allowed for all scope" - default: - return "" - } -} - -// sendNowErrorResponse maps send-now failures to their stable websocket error codes. -func (h *QueueHandlers) sendNowErrorResponse(msg *ws.Message, sessionID string, err error) (*ws.Message, error) { - switch { - case errors.Is(err, orchestrator.ErrSendNowEntryNotFound): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry is no longer pending", nil) - case errors.Is(err, orchestrator.ErrSendNowQueueEmpty): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowQueueEmpty, "Queue is empty", nil) - case errors.Is(err, orchestrator.ErrSendNowQueueChanged): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowQueueChanged, "Queue changed before Send Now could start", nil) - case errors.Is(err, orchestrator.ErrSendNowConflict): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowConflict, "Another cancellation or Send Now operation is in progress", nil) - case errors.Is(err, orchestrator.ErrSendNowTurnChanged): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowTurnChanged, "The active turn changed before Send Now could start", nil) - case errors.Is(err, messagequeue.ErrSendNowAttachmentOverflow): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowAttachmentOverflow, "Combined attachments exceed the message limits", nil) - case errors.Is(err, messagequeue.ErrSendNowReferenceOverflow): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowReferenceOverflow, "Combined entity references exceed the message limit", nil) - case errors.Is(err, orchestrator.ErrSessionNotPromptable): - return ws.NewError(msg.ID, msg.Action, queueErrorCodeNotPromptable, "Session is not ready for input", nil) - default: - h.logger.Error("failed to send queued message now", zap.String(fieldSessionID, sessionID), zap.Error(err)) - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to send queued message now", nil) - } -} - -type wsGetQueueStatusRequest struct { - SessionID string `json:"session_id"` -} - -// wsGetQueueStatus handles ActionMessageQueueGet, returning the pending list and capacity. -func (h *QueueHandlers) wsGetQueueStatus(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsGetQueueStatusRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - - status := h.queueService.GetStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, status) } type wsUpdateMessageRequest struct { - SessionID string `json:"session_id"` - EntryID string `json:"entry_id"` - Content string `json:"content"` - Attachments []messagequeue.MessageAttachment `json:"attachments,omitempty"` - EntityReferences []v1.EntityReference `json:"entity_references,omitempty"` - UserID string `json:"user_id,omitempty"` + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + EntryID string `json:"entry_id"` + Content string `json:"content"` + Attachments []messagequeue.MessageAttachment `json:"attachments,omitempty"` + EntityReferences []v1.EntityReference `json:"entity_references,omitempty"` + UserID string `json:"user_id,omitempty"` +} + +// updateQueuedMessage selects the identity-bound mutation and optional atomic attachment claim. +func (h *QueueHandlers) updateQueuedMessage( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + req wsUpdateMessageRequest, + metadataUpdates map[string]interface{}, + queuedBy string, + atomicClaim *messagequeue.QueueAttachmentClaim, +) error { + if !h.requiresQueueIdentity() { + return h.queueService.UpdateMessageWithMetadata( + ctx, req.SessionID, req.EntryID, req.Content, req.Attachments, metadataUpdates, queuedBy, + ) + } + if atomicClaim == nil { + return h.queueService.(QueueIdentityMutationService).UpdateMessageWithMetadataForSession( + ctx, identity, req.EntryID, req.Content, req.Attachments, metadataUpdates, queuedBy, + ) + } + atomicMutations, ok := h.queueService.(QueueIdentityAttachmentMutationService) + if !ok { + return errors.New("transactional attachment update is unavailable") + } + return atomicMutations.UpdateMessageWithMetadataForSessionWithClaim( + ctx, identity, req.EntryID, req.Content, req.Attachments, metadataUpdates, queuedBy, *atomicClaim, + ) +} + +func (h *QueueHandlers) releaseFailedQueueAttachmentClaims( + ctx context.Context, + releaser QueueAttachmentReleaser, + previous *messagequeue.QueuedMessage, + sessionID string, + attachments []messagequeue.MessageAttachment, +) { + if releaser == nil || previous == nil { + return + } + if err := releaser.ReleaseMessageAttachments( + ctx, previous.TaskID, sessionID, queueAttachmentsToV1(attachments), + ); err != nil { + h.logger.Warn("failed to release attachments after queue update failure", zap.Error(err)) + } } // wsUpdateMessage handles ActionMessageQueueUpdate, replacing a queued entry's content. @@ -520,11 +477,12 @@ func (h *QueueHandlers) wsUpdateMessage(ctx context.Context, msg *ws.Message) (* return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) } if req.SessionID == "" { - // Required so publishStatus can broadcast the post-update list to other - // connected clients; without it they'd be left with a stale view. return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { return denied, nil } if req.EntryID == "" { @@ -571,34 +529,57 @@ func (h *QueueHandlers) wsUpdateMessage(ctx context.Context, msg *ws.Message) (* } metadataUpdates = map[string]interface{}{messagequeue.MetadataEntityReferences: referenceMetadata} } + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } var previous *messagequeue.QueuedMessage var releaseClaims QueueAttachmentReleaser var newlyAdded []messagequeue.MessageAttachment + var atomicClaim *messagequeue.QueueAttachmentClaim if h.attachmentClaimer != nil { var err error - previous, err = h.queueService.GetEntry(ctx, req.SessionID, req.EntryID) + if h.requiresQueueIdentity() { + entryService, ok := h.queueService.(QueueIdentityEntryService) + if !ok { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Identity-bound queue lookup is unavailable", nil) + } + previous, err = entryService.GetEntryForSession(ctx, identity, req.EntryID) + } else { + previous, err = h.queueService.GetEntry(ctx, req.SessionID, req.EntryID) + } if err != nil { if errors.Is(err, messagequeue.ErrEntryNotFound) { return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry was already drained or not owned by caller", nil) } + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) } newlyAdded = newlyAddedQueueAttachments(previous.Attachments, req.Attachments) - if err := h.attachmentClaimer.ClaimMessageAttachments(ctx, previous.TaskID, req.SessionID, queueAttachmentsToV1(newlyAdded)); err != nil { + if preparer, ok := h.attachmentClaimer.(QueueAttachmentClaimPreparer); ok && h.requiresQueueIdentity() { + prepared, prepareErr := preparer.PrepareQueueAttachmentClaim(ctx, previous.TaskID, queueAttachmentsToV1(newlyAdded)) + if prepareErr != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "Attachment is no longer available", nil) + } + atomicClaim = &prepared + } else if err := h.attachmentClaimer.ClaimMessageAttachments(ctx, previous.TaskID, req.SessionID, queueAttachmentsToV1(newlyAdded)); err != nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "Attachment is no longer available", nil) } releaseClaims, _ = h.attachmentClaimer.(QueueAttachmentReleaser) } - if err := h.queueService.UpdateMessageWithMetadata(ctx, req.SessionID, req.EntryID, req.Content, req.Attachments, metadataUpdates, queuedBy); err != nil { - if releaseClaims != nil && previous != nil { - if releaseErr := releaseClaims.ReleaseMessageAttachments(ctx, previous.TaskID, req.SessionID, queueAttachmentsToV1(newlyAdded)); releaseErr != nil { - h.logger.Warn("failed to release attachments after queue update failure", zap.Error(releaseErr)) - } + updateErr := h.updateQueuedMessage(ctx, identity, req, metadataUpdates, queuedBy, atomicClaim) + if updateErr != nil { + if atomicClaim == nil { + h.releaseFailedQueueAttachmentClaims(ctx, releaseClaims, previous, req.SessionID, newlyAdded) } - if errors.Is(err, messagequeue.ErrEntryNotFound) { + if errors.Is(updateErr, messagequeue.ErrEntryNotFound) { return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry was already drained or not owned by caller", nil) } - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) + if isQueueIdentityError(updateErr) { + return queueAccessDeniedResponse(msg), nil + } + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, updateErr.Error(), nil) } if releaseClaims != nil && previous != nil { if superseded := supersededQueueAttachments(previous.Attachments, req.Attachments); len(superseded) > 0 { @@ -607,7 +588,7 @@ func (h *QueueHandlers) wsUpdateMessage(ctx context.Context, msg *ws.Message) (* } } } - h.publishStatus(ctx, req.SessionID) + h.publishStatusForIdentity(ctx, identity) return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{fieldEntryID: req.EntryID}) } @@ -755,150 +736,14 @@ func queueAttachmentsToV1(attachments []messagequeue.MessageAttachment) []v1.Mes return converted } -type wsRemoveEntryRequest struct { - SessionID string `json:"session_id"` - EntryID string `json:"entry_id"` -} - -// wsRemoveEntry handles ActionMessageQueueRemove, deleting a single queued entry. -func (h *QueueHandlers) wsRemoveEntry(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsRemoveEntryRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - // Required so publishStatus can broadcast the post-removal list. - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if req.EntryID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "entry_id is required", nil) - } - - if err := h.queueService.RemoveEntry(ctx, req.SessionID, req.EntryID); err != nil { - if errors.Is(err, messagequeue.ErrEntryNotFound) { - return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry is no longer pending", nil) - } - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{fieldEntryID: req.EntryID}) -} - -// wsMergeIntoAboveRequest is the payload for ActionMessageQueueMerge: the -// session whose queue is modified and the id of the entry to fold into the -// entry directly above it. user_id is forwarded for ownership checks and is -// optional (the server defaults to the reserved "user" identity). -type wsMergeIntoAboveRequest struct { - SessionID string `json:"session_id"` - EntryID string `json:"entry_id"` - UserID string `json:"user_id,omitempty"` -} - -// wsMergeIntoAbove handles ActionMessageQueueMerge, folding the referenced -// queued entry into the entry above it and broadcasting the updated queue. -func (h *QueueHandlers) wsMergeIntoAbove(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsMergeIntoAboveRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - // Required so publishStatus can broadcast the post-merge list. - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if req.EntryID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "entry_id is required", nil) - } - if messagequeue.IsReservedQueuedBy(req.UserID) { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, reservedIdentityError(req.UserID), nil) - } - // Default empty user_id to QueuedByUser so the merge ownership guard runs - // against a non-empty owner, mirroring wsUpdateMessage. - queuedBy := req.UserID - if queuedBy == "" { - queuedBy = messagequeue.QueuedByUser - } - - merged, err := h.queueService.MergeIntoAbove(ctx, req.SessionID, req.EntryID, queuedBy) - if err != nil { - if errors.Is(err, messagequeue.ErrEntryNotFound) { - return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry was already drained or not owned by caller", nil) - } - if errors.Is(err, messagequeue.ErrNoMergeTarget) { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "No mergeable message above this entry", nil) - } - if errors.Is(err, messagequeue.ErrMergeReferenceOverflow) { - return ws.NewError(msg.ID, msg.Action, queueErrorCodeMergeReferenceOverflow, err.Error(), nil) - } - if errors.Is(err, messagequeue.ErrMergeDisabled) { - return ws.NewError(msg.ID, msg.Action, queueErrorCodeMergeDisabled, "Message merging is disabled", nil) - } - h.logger.Error("failed to merge queued message", zap.Error(err)) - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to merge queued message", nil) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{fieldEntryID: merged.ID}) -} - -// wsReorderRequest is the payload for ActionMessageQueueReorder: the session -// whose queue is modified and the complete ordered list of visible pending -// entry ids the caller wants as the new FIFO order. -type wsReorderRequest struct { - SessionID string `json:"session_id"` - OrderedIDs []string `json:"ordered_ids"` -} - -// wsReorder handles ActionMessageQueueReorder, rewriting the session's visible -// pending order to match ordered_ids and broadcasting the updated queue. Any -// drift from the persisted visible set (a drain/remove/merge raced the drag) -// is rejected atomically with queue_changed so the client refetches. -func (h *QueueHandlers) wsReorder(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - var req wsReorderRequest - if err := msg.ParsePayload(&req); err != nil { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - } - if req.SessionID == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - } - if denied := h.authorizeSession(ctx, msg, req.SessionID); denied != nil { - return denied, nil - } - if len(req.OrderedIDs) == 0 { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "ordered_ids is required", nil) - } - if hasDuplicateIDs(req.OrderedIDs) { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "ordered_ids must not contain duplicates", nil) - } - - if err := h.queueService.ReorderEntries(ctx, req.SessionID, req.OrderedIDs); err != nil { - if errors.Is(err, messagequeue.ErrQueueChanged) { - return ws.NewError(msg.ID, msg.Action, queueErrorCodeQueueChanged, "Queue changed before the reorder could be applied", nil) - } - h.logger.Error("failed to reorder queued messages", zap.Error(err)) - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to reorder queued messages", nil) - } - - h.publishStatus(ctx, req.SessionID) - return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ - fieldSessionID: req.SessionID, - "reordered": len(req.OrderedIDs), - }) -} - type wsAppendToQueueRequest struct { - SessionID string `json:"session_id"` - TaskID string `json:"task_id"` - Content string `json:"content"` - Model string `json:"model,omitempty"` - PlanMode bool `json:"plan_mode,omitempty"` - UserID string `json:"user_id,omitempty"` + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + Content string `json:"content"` + Model string `json:"model,omitempty"` + PlanMode bool `json:"plan_mode,omitempty"` + UserID string `json:"user_id,omitempty"` } // wsAppendToQueue handles ActionMessageQueueAppend, appending or inserting a user message. @@ -914,7 +759,10 @@ func (h *QueueHandlers) wsAppendToQueue(ctx context.Context, msg *ws.Message) (* if req.TaskID == "" { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id is required", nil) } - if denied := h.authorizeTaskSession(ctx, msg, req.TaskID, req.SessionID); denied != nil { + if h.requiresQueueIdentity() && req.SessionIncarnationID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { return denied, nil } if req.Content == "" { @@ -928,27 +776,65 @@ func (h *QueueHandlers) wsAppendToQueue(ctx context.Context, msg *ws.Message) (* if queuedBy == "" { queuedBy = messagequeue.QueuedByUser } - queued, appended, err := h.queueService.AppendContent(ctx, req.SessionID, req.TaskID, req.Content, req.Model, queuedBy, req.PlanMode, nil) + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var queued *messagequeue.QueuedMessage + var appended bool + var err error + if h.requiresQueueIdentity() { + queued, appended, err = h.queueService.(QueueIdentityMutationService).AppendContentForSession(ctx, identity, req.Content, req.Model, queuedBy, req.PlanMode, nil) + } else { + queued, appended, err = h.queueService.AppendContent(ctx, req.SessionID, req.TaskID, req.Content, req.Model, queuedBy, req.PlanMode, nil) + } if err != nil { if errors.Is(err, messagequeue.ErrQueueFull) { - status := h.queueService.GetStatus(ctx, req.SessionID) - return ws.NewError(msg.ID, msg.Action, messagequeue.QueueFullErrorCode, "Queue is full", - map[string]interface{}{ - fieldQueueSize: status.Count, - fieldMax: status.Max, - }) + return h.queueFullResponse(ctx, msg, identity) + } + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil } h.logger.Error("failed to append to queue", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to queue message", nil) } - h.publishStatus(ctx, req.SessionID) + h.publishStatusForIdentity(ctx, identity) return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ fieldEntryID: queued.ID, "was_append": appended, }) } +func (h *QueueHandlers) queueFullResponse( + ctx context.Context, + msg *ws.Message, + identity messagequeue.QueueSessionIdentity, +) (*ws.Message, error) { + if !h.requiresQueueIdentity() { + return queueFullErrorResponse(msg, h.queueService.GetStatus(ctx, identity.SessionID)) + } + snapshots, ok := h.queueService.(QueueSnapshotService) + if !ok { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue status is unavailable", nil) + } + status, err := snapshots.Snapshot(ctx, identity) + if err == nil { + return queueFullErrorResponse(msg, status) + } + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to read queue status", nil) +} + +func queueFullErrorResponse(msg *ws.Message, status *messagequeue.QueueStatus) (*ws.Message, error) { + return ws.NewError(msg.ID, msg.Action, messagequeue.QueueFullErrorCode, "Queue is full", + map[string]interface{}{ + fieldQueueSize: status.Count, + fieldMax: status.Max, + }) +} + // hasDuplicateIDs reports whether ids contains any id more than once. func hasDuplicateIDs(ids []string) bool { seen := make(map[string]struct{}, len(ids)) @@ -994,44 +880,34 @@ func (h *QueueHandlers) authorizeTaskSession( } return nil } +func (h *QueueHandlers) requiresQueueIdentity() bool { + _, ok := h.accessAuthorizer.(QueueSessionIdentityAuthorizer) + return ok +} + +func (h *QueueHandlers) authorizeQueueIdentity( + ctx context.Context, + msg *ws.Message, + taskID, sessionID, incarnationID string, +) *ws.Message { + if taskID == "" { + return h.authorizeSession(ctx, msg, sessionID) + } + if incarnationID == "" { + return h.authorizeTaskSession(ctx, msg, taskID, sessionID) + } + authorizer, ok := h.accessAuthorizer.(QueueSessionIdentityAuthorizer) + if !ok { + return h.authorizeTaskSession(ctx, msg, taskID, sessionID) + } + if err := authorizer.AuthorizeTaskSessionIncarnationAccess(ctx, taskID, sessionID, incarnationID); err != nil { + return queueAccessDeniedResponse(msg) + } + return nil +} // queueAccessDeniedResponse builds the non-enumerating session-not-found error response. func queueAccessDeniedResponse(msg *ws.Message) *ws.Message { response, _ := ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, queueAccessDenied, nil) return response } - -// publishStatus emits the latest QueueStatus on the event bus so the frontend -// updates its store after every mutation. -func (h *QueueHandlers) publishStatus(ctx context.Context, sessionID string, admitted ...*messagequeue.QueuedMessage) { - if h.eventBus == nil { - return - } - status := h.queueService.GetStatus(ctx, sessionID) - eventData := map[string]interface{}{ - fieldSessionID: sessionID, - "entries": status.Entries, - "count": status.Count, - fieldMax: status.Max, - "auto_run": status.AutoRun, - "merge_enabled": status.MergeEnabled, - } - if len(admitted) > 0 && admitted[0] != nil && admitted[0].QueuedBy != "" && !messagequeue.IsReservedQueuedBy(admitted[0].QueuedBy) { - eventData["queued_by"] = admitted[0].QueuedBy - eventData["queued_at"] = admitted[0].QueuedAt - } - if h.sessionTaskResolver != nil { - if taskID, err := h.sessionTaskResolver(ctx, sessionID); err != nil { - h.logger.Warn("resolve session task for queue status event", - zap.String("session_id", sessionID), - zap.Error(err)) - } else if taskID != "" { - eventData["task_id"] = taskID - } - } - _ = h.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( - events.MessageQueueStatusChanged, - "queue-handlers", - eventData, - )) -} diff --git a/apps/backend/internal/orchestrator/handlers/queue_handlers_attachment_cleanup_test.go b/apps/backend/internal/orchestrator/handlers/queue_handlers_attachment_cleanup_test.go new file mode 100644 index 0000000000..9d9eeeea2e --- /dev/null +++ b/apps/backend/internal/orchestrator/handlers/queue_handlers_attachment_cleanup_test.go @@ -0,0 +1,114 @@ +package handlers + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + ws "github.com/kandev/kandev/pkg/websocket" + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +func setupSQLiteQueueAttachmentCleanup(t *testing.T) (*QueueHandlers, *messagequeue.Service, messagequeue.Repository, messagequeue.QueueSessionIdentity, *recordingQueueAttachmentClaimer) { + t.Helper() + raw, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "queue.db")+"?_foreign_keys=on") + require.NoError(t, err) + raw.SetMaxOpenConns(1) + raw.SetMaxIdleConns(1) + database := sqlx.NewDb(raw, "sqlite3") + t.Cleanup(func() { require.NoError(t, database.Close()) }) + _, err = database.Exec(` + CREATE TABLE tasks (id TEXT PRIMARY KEY, archived_at TIMESTAMP, updated_at TIMESTAMP); + CREATE TABLE task_sessions ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + queue_incarnation_id TEXT NOT NULL + ); + `) + require.NoError(t, err) + identity := messagequeue.QueueSessionIdentity{ + TaskID: "task-cleanup", SessionID: "session-cleanup", SessionIncarnationID: "incarnation-cleanup", + } + _, err = database.Exec(`INSERT INTO tasks (id) VALUES (?)`, identity.TaskID) + require.NoError(t, err) + _, err = database.Exec( + `INSERT INTO task_sessions (id, task_id, queue_incarnation_id) VALUES (?, ?, ?)`, + identity.SessionID, identity.TaskID, identity.SessionIncarnationID, + ) + require.NoError(t, err) + + repository, err := messagequeue.NewSQLiteRepository(database, database) + require.NoError(t, err) + service := messagequeue.NewService(repository, 10, logger.Default()) + service.SetAutoMergeEnabled(false) + handlers := NewQueueHandlers( + service, &mockEventBus{}, logger.Default(), nil, allowQueueIdentityAccess{}, nil, + ) + claimer := &recordingQueueAttachmentClaimer{} + handlers.SetAttachmentClaimer(claimer) + return handlers, service, repository, identity, claimer +} + +func queuedFileAttachment(id string) messagequeue.MessageAttachment { + return messagequeue.MessageAttachment{ + Type: "resource", AttachmentID: id, Name: id + ".txt", MimeType: "text/plain", DeliveryMode: "path", + } +} + +func TestQueueRemovalReleasesOnlyUnreferencedAttachmentsSQLite(t *testing.T) { + handlers, service, _, identity, claimer := setupSQLiteQueueAttachmentCleanup(t) + ctx := context.Background() + removed, err := service.QueueMessageWithMetadataForSession( + ctx, identity, "remove", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{queuedFileAttachment("shared"), queuedFileAttachment("remove-only")}, nil, + ) + require.NoError(t, err) + _, err = service.QueueMessageWithMetadataForSession( + ctx, identity, "retain", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{queuedFileAttachment("shared")}, nil, + ) + require.NoError(t, err) + + response, err := handlers.wsRemoveEntry(ctx, createTestMessage(t, ws.ActionMessageQueueRemove, map[string]interface{}{ + "task_id": identity.TaskID, "session_id": identity.SessionID, + "session_incarnation_id": identity.SessionIncarnationID, "entry_id": removed.ID, + })) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + require.Equal(t, []string{"remove-only"}, claimer.releases) +} + +func TestQueueCancelAllReleasesOnlyUnreferencedAttachmentsSQLite(t *testing.T) { + handlers, service, repository, identity, claimer := setupSQLiteQueueAttachmentCleanup(t) + ctx := context.Background() + _, err := service.QueueMessageWithMetadataForSession( + ctx, identity, "in flight", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{queuedFileAttachment("shared")}, nil, + ) + require.NoError(t, err) + reserved, _, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.NotNil(t, reserved) + _, err = service.QueueMessageWithMetadataForSession( + ctx, identity, "remove", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{queuedFileAttachment("shared"), queuedFileAttachment("remove-only")}, nil, + ) + require.NoError(t, err) + + response, err := handlers.wsCancelAll(ctx, createTestMessage(t, ws.ActionMessageQueueCancel, map[string]interface{}{ + "task_id": identity.TaskID, "session_id": identity.SessionID, + "session_incarnation_id": identity.SessionIncarnationID, + })) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + require.Equal(t, []string{"remove-only"}, claimer.releases) + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.True(t, stored[0].IsReservedInFlight()) +} diff --git a/apps/backend/internal/orchestrator/handlers/queue_handlers_auto_merge_policy_test.go b/apps/backend/internal/orchestrator/handlers/queue_handlers_auto_merge_policy_test.go new file mode 100644 index 0000000000..dee1d8d651 --- /dev/null +++ b/apps/backend/internal/orchestrator/handlers/queue_handlers_auto_merge_policy_test.go @@ -0,0 +1,39 @@ +package handlers + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ws "github.com/kandev/kandev/pkg/websocket" +) + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.7 +func TestQueueHandlersAutoMergeSetReturnsEffectiveSessionPolicy(t *testing.T) { + const action = "message.queue.auto_merge.set" + handlers, _ := setupQueueHandlers(t) + dispatcher := ws.NewDispatcher() + handlers.RegisterHandlers(dispatcher) + require.True(t, dispatcher.HasHandler(action)) + + response, err := dispatcher.Dispatch(context.Background(), createTestMessage(t, action, map[string]interface{}{ + "task_id": "task", + "session_id": "session", + "session_incarnation_id": "memory:session", + "enabled": false, + })) + require.NoError(t, err) + assert.Equal(t, ws.MessageTypeResponse, response.Type) + var payload map[string]interface{} + require.NoError(t, json.Unmarshal(response.Payload, &payload)) + assert.Equal(t, "task", payload["task_id"]) + assert.Equal(t, "session", payload["session_id"]) + assert.Equal(t, "memory:session", payload["session_incarnation_id"]) + assert.Equal(t, false, payload["auto_merge_enabled"]) + assert.Equal(t, "session", payload["auto_merge_source"]) + assert.Equal(t, float64(1), payload["auto_merge_revision"]) +} diff --git a/apps/backend/internal/orchestrator/handlers/queue_handlers_snapshot_test.go b/apps/backend/internal/orchestrator/handlers/queue_handlers_snapshot_test.go new file mode 100644 index 0000000000..4ed07761bd --- /dev/null +++ b/apps/backend/internal/orchestrator/handlers/queue_handlers_snapshot_test.go @@ -0,0 +1,149 @@ +package handlers + +import ( + "context" + "encoding/json" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + ws "github.com/kandev/kandev/pkg/websocket" + "github.com/stretchr/testify/require" +) + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.5 +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 +func TestQueueStatusRequestReturnsIdentityBoundPolicySnapshot(t *testing.T) { + handlers, service := setupQueueHandlers(t) + identity := messagequeue.QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "memory:session", + } + _, err := service.SetSessionAutoMerge(context.Background(), identity, false) + require.NoError(t, err) + + response, err := handlers.wsGetQueueStatus(context.Background(), createTestMessage(t, ws.ActionMessageQueueGet, map[string]interface{}{ + "task_id": identity.TaskID, + "session_id": identity.SessionID, + "session_incarnation_id": identity.SessionIncarnationID, + })) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + var status messagequeue.QueueStatus + require.NoError(t, json.Unmarshal(response.Payload, &status)) + require.Equal(t, identity.TaskID, status.TaskID) + require.NotEmpty(t, status.StatusEpoch) + require.NotEqual(t, identity.SessionIncarnationID, status.StatusEpoch) + require.True(t, status.AutoMergeAvailable) + require.NotNil(t, status.AutoMergeEnabled) + require.False(t, *status.AutoMergeEnabled) + require.Equal(t, messagequeue.AutoMergeSourceSession, status.AutoMergeSource) +} + +type allowQueueIdentityAccess struct { + allowQueueAccess +} + +func (allowQueueIdentityAccess) AuthorizeTaskSessionIncarnationAccess( + context.Context, + string, + string, + string, +) error { + return nil +} + +type queueFullIdentityService struct { + *messagequeue.Service + snapshot *messagequeue.QueueStatus + snapshotErr error + legacyStatusRead bool +} + +func (s *queueFullIdentityService) QueueMessageWithMetadataForSession( + context.Context, + messagequeue.QueueSessionIdentity, + string, + string, + string, + bool, + []messagequeue.MessageAttachment, + map[string]interface{}, +) (*messagequeue.QueuedMessage, error) { + return nil, messagequeue.ErrQueueFull +} + +func (s *queueFullIdentityService) AppendContentForSession( + context.Context, + messagequeue.QueueSessionIdentity, + string, + string, + string, + bool, + []messagequeue.MessageAttachment, +) (*messagequeue.QueuedMessage, bool, error) { + return nil, false, messagequeue.ErrQueueFull +} + +func (s *queueFullIdentityService) Snapshot( + context.Context, + messagequeue.QueueSessionIdentity, +) (*messagequeue.QueueStatus, error) { + return s.snapshot, s.snapshotErr +} + +func (s *queueFullIdentityService) GetStatus( + context.Context, + string, +) *messagequeue.QueueStatus { + s.legacyStatusRead = true + return &messagequeue.QueueStatus{Count: 99, Max: 99} +} + +func TestQueueFullResponseUsesIdentityBoundSnapshot(t *testing.T) { + testQueueFullResponses(t, func(t *testing.T, response *ws.Message, service *queueFullIdentityService) { + require.Equal(t, ws.MessageTypeError, response.Type) + errPayload := parseError(t, response) + require.Equal(t, messagequeue.QueueFullErrorCode, errPayload.Code) + require.EqualValues(t, 7, errPayload.Details[fieldQueueSize]) + require.EqualValues(t, 11, errPayload.Details[fieldMax]) + require.False(t, service.legacyStatusRead) + }, &messagequeue.QueueStatus{Count: 7, Max: 11}, nil) +} + +func TestQueueFullResponseRejectsStaleIdentityWithoutLegacyStatus(t *testing.T) { + testQueueFullResponses(t, func(t *testing.T, response *ws.Message, service *queueFullIdentityService) { + require.Equal(t, ws.MessageTypeError, response.Type) + require.Equal(t, ws.ErrorCodeNotFound, parseError(t, response).Code) + require.False(t, service.legacyStatusRead) + }, nil, messagequeue.ErrSessionIdentityMismatch) +} + +func testQueueFullResponses( + t *testing.T, + assertResponse func(*testing.T, *ws.Message, *queueFullIdentityService), + snapshot *messagequeue.QueueStatus, + snapshotErr error, +) { + t.Helper() + calls := map[string]func(*QueueHandlers, context.Context, *ws.Message) (*ws.Message, error){ + "queue": (*QueueHandlers).wsQueueMessage, + "append": (*QueueHandlers).wsAppendToQueue, + } + for name, call := range calls { + t.Run(name, func(t *testing.T) { + handlers, base := setupQueueHandlers(t) + service := &queueFullIdentityService{ + Service: base, snapshot: snapshot, snapshotErr: snapshotErr, + } + handlers.queueService = service + handlers.accessAuthorizer = allowQueueIdentityAccess{} + response, err := call(handlers, context.Background(), createTestMessage(t, ws.ActionMessageQueueAdd, map[string]interface{}{ + fieldTaskID: "task-1", + fieldSessionID: "session-1", + fieldSessionIncarnation: "incarnation-1", + "content": "overflow", + })) + require.NoError(t, err) + assertResponse(t, response, service) + }) + } +} diff --git a/apps/backend/internal/orchestrator/handlers/queue_operations_handlers.go b/apps/backend/internal/orchestrator/handlers/queue_operations_handlers.go new file mode 100644 index 0000000000..a98b8c9524 --- /dev/null +++ b/apps/backend/internal/orchestrator/handlers/queue_operations_handlers.go @@ -0,0 +1,709 @@ +package handlers + +import ( + "context" + "errors" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/orchestrator" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + ws "github.com/kandev/kandev/pkg/websocket" + "go.uber.org/zap" +) + +type wsCancelAllRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` +} + +// wsCancelAll handles ActionMessageQueueCancel, clearing every pending entry for a session. +func (h *QueueHandlers) wsCancelAll(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsCancelAllRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var removal *messagequeue.QueueRemovalResult + var removed int + var err error + if h.requiresQueueIdentity() { + removal, err = h.queueService.(QueueIdentityMutationService).CancelAllForSession(ctx, identity) + if removal != nil { + removed = len(removal.Removed) + } + } else { + before := h.queueService.GetStatus(ctx, req.SessionID) + removed, err = h.queueService.CancelAll(ctx, req.SessionID) + if err == nil { + removal = &messagequeue.QueueRemovalResult{ + Removed: append([]messagequeue.QueuedMessage(nil), before.Entries...), + Retained: append([]messagequeue.QueuedMessage(nil), h.queueService.GetStatus(ctx, req.SessionID).Entries...), + } + } + } + if err != nil { + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + + h.releaseQueueRemovalAttachments(ctx, removal) + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldSessionID: req.SessionID, + "removed": removed, + }) +} + +type wsDrainQueueRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` +} + +// wsDrainQueue handles ActionMessageQueueDrain, dispatching one queued entry when the session is promptable. +func (h *QueueHandlers) wsDrainQueue(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsDrainQueueRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if h.queueDrainer == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue drain is unavailable", nil) + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var drained bool + var err error + if h.requiresQueueIdentity() { + drained, err = h.queueDrainer.(QueueIdentityDrainer).DrainQueuedMessageForSession(ctx, identity) + } else { + drained, err = h.queueDrainer.DrainQueuedMessage(ctx, req.SessionID) + } + if err != nil { + switch { + case errors.Is(err, orchestrator.ErrAgentPromptInProgress): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSessionBusy, "Session is busy", nil) + case errors.Is(err, orchestrator.ErrSessionNotPromptable): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeNotPromptable, "Session is not ready for input", nil) + case isQueueIdentityError(err): + return queueAccessDeniedResponse(msg), nil + default: + h.logger.Error("failed to drain queued message", zap.String(fieldSessionID, req.SessionID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to drain queued message", nil) + } + } + + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldSessionID: req.SessionID, + "drained": drained, + }) +} + +type wsSetAutoRunRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + Enabled *bool `json:"enabled"` +} + +// wsSetAutoRun persists automatic queue processing and optionally starts the +// promptable FIFO head when enabling it. +func (h *QueueHandlers) wsSetAutoRun(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsSetAutoRunRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if req.Enabled == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "enabled is required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if h.queueAutoRun == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue Auto-run is unavailable", nil) + } + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var autoRun, dispatched bool + var err error + if h.requiresQueueIdentity() { + autoRun, dispatched, err = h.queueAutoRun.(QueueIdentityAutoRunController).SetQueueAutoRunForSession(ctx, identity, *req.Enabled) + } else { + autoRun, dispatched, err = h.queueAutoRun.SetQueueAutoRun(ctx, req.SessionID, *req.Enabled) + } + if err != nil { + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + h.logger.Error("failed to set queue Auto-run", zap.String(fieldSessionID, req.SessionID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to set queue Auto-run", nil) + } + + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldSessionID: req.SessionID, + fieldAutoRun: autoRun, + "dispatched": dispatched, + }) +} + +type wsSetAutoMergeRequest struct { + TaskID string `json:"task_id"` + SessionID string `json:"session_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + Enabled *bool `json:"enabled"` +} + +func (h *QueueHandlers) wsSetAutoMerge(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsSetAutoMergeRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.TaskID == "" || req.SessionID == "" || req.SessionIncarnationID == "" || req.Enabled == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, session_incarnation_id, and enabled are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + controller, ok := h.queueService.(QueueAutoMergeController) + if !ok { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue Auto-merge is unavailable", nil) + } + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + policy, err := controller.SetSessionAutoMerge(ctx, identity, *req.Enabled) + if err != nil { + if errors.Is(err, messagequeue.ErrSessionIdentityMismatch) || + errors.Is(err, messagequeue.ErrTaskInactive) { + return queueAccessDeniedResponse(msg), nil + } + h.logger.Error("failed to set queue Auto-merge", zap.String(fieldSessionID, req.SessionID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to set queue Auto-merge", nil) + } + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldTaskID: identity.TaskID, + fieldSessionID: identity.SessionID, + fieldSessionIncarnation: identity.SessionIncarnationID, + fieldAutoMergeEnabled: policy.Enabled, + "auto_merge_source": policy.Source, + "auto_merge_revision": policy.Revision, + }) +} + +type wsSendNowRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + Scope string `json:"scope"` + EntryID string `json:"entry_id,omitempty"` +} + +// wsSendNow handles ActionMessageQueueSendNow, interrupting the active turn with an exact queue selection. +func (h *QueueHandlers) wsSendNow(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsSendNowRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if validation := validateSendNowRequest(req); validation != "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, validation, nil) + } + if h.queueDispatcher == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue send-now is unavailable", nil) + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var sentCount int + var err error + if h.requiresQueueIdentity() { + sentCount, err = h.queueDispatcher.(QueueIdentitySendNowDispatcher).SendQueuedNowForSession(ctx, identity, req.Scope, req.EntryID) + } else { + sentCount, err = h.queueDispatcher.SendQueuedNow(ctx, req.SessionID, req.Scope, req.EntryID) + } + if err != nil { + return h.sendNowErrorResponse(msg, req.SessionID, err) + } + + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldSessionID: req.SessionID, + "dispatched": true, + "sent_count": sentCount, + }) +} + +// validateSendNowRequest validates the send-now scope and entry id combination. +func validateSendNowRequest(req wsSendNowRequest) string { + switch { + case req.Scope != orchestrator.QueueSendNowScopeEntry && req.Scope != orchestrator.QueueSendNowScopeAll: + return "scope must be entry or all" + case req.Scope == orchestrator.QueueSendNowScopeEntry && req.EntryID == "": + return "entry_id is required for entry scope" + case req.Scope == orchestrator.QueueSendNowScopeAll && req.EntryID != "": + return "entry_id is not allowed for all scope" + default: + return "" + } +} + +// sendNowErrorResponse maps send-now failures to their stable websocket error codes. +func (h *QueueHandlers) sendNowErrorResponse(msg *ws.Message, sessionID string, err error) (*ws.Message, error) { + switch { + case errors.Is(err, orchestrator.ErrSendNowEntryNotFound): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry is no longer pending", nil) + case errors.Is(err, orchestrator.ErrSendNowQueueEmpty): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowQueueEmpty, "Queue is empty", nil) + case errors.Is(err, orchestrator.ErrSendNowQueueChanged): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowQueueChanged, "Queue changed before Send Now could start", nil) + case errors.Is(err, orchestrator.ErrSendNowConflict): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowConflict, "Another cancellation or Send Now operation is in progress", nil) + case errors.Is(err, orchestrator.ErrSendNowTurnChanged): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowTurnChanged, "The active turn changed before Send Now could start", nil) + case errors.Is(err, messagequeue.ErrSendNowAttachmentOverflow): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowAttachmentOverflow, "Combined attachments exceed the message limits", nil) + case errors.Is(err, messagequeue.ErrSendNowReferenceOverflow): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeSendNowReferenceOverflow, "Combined entity references exceed the message limit", nil) + case isQueueIdentityError(err): + return queueAccessDeniedResponse(msg), nil + case errors.Is(err, orchestrator.ErrSessionNotPromptable): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeNotPromptable, "Session is not ready for input", nil) + default: + h.logger.Error("failed to send queued message now", zap.String(fieldSessionID, sessionID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to send queued message now", nil) + } +} + +type wsGetQueueStatusRequest struct { + TaskID string `json:"task_id"` + SessionID string `json:"session_id"` + SessionIncarnationID string `json:"session_incarnation_id"` +} + +// wsGetQueueStatus handles ActionMessageQueueGet, returning the pending list and capacity. +func (h *QueueHandlers) wsGetQueueStatus(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsGetQueueStatusRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if req.TaskID == "" || req.SessionIncarnationID == "" { + return ws.NewResponse(msg.ID, msg.Action, h.queueService.GetStatus(ctx, req.SessionID)) + } + snapshots, ok := h.queueService.(QueueSnapshotService) + if !ok { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Queue status is unavailable", nil) + } + status, err := snapshots.Snapshot(ctx, messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + }) + if err != nil { + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to read queue status", nil) + } + return ws.NewResponse(msg.ID, msg.Action, status) +} + +type wsRemoveEntryRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + EntryID string `json:"entry_id"` +} + +// wsRemoveEntry handles ActionMessageQueueRemove, deleting a single queued entry. +func (h *QueueHandlers) wsRemoveEntry(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsRemoveEntryRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if req.EntryID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "entry_id is required", nil) + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var removal *messagequeue.QueueRemovalResult + var err error + if h.requiresQueueIdentity() { + removal, err = h.queueService.(QueueIdentityMutationService).RemoveEntryForSession(ctx, identity, req.EntryID) + } else { + previous, getErr := h.queueService.GetEntry(ctx, req.SessionID, req.EntryID) + err = h.queueService.RemoveEntry(ctx, req.SessionID, req.EntryID) + if err == nil && getErr == nil { + removal = &messagequeue.QueueRemovalResult{ + Removed: []messagequeue.QueuedMessage{*previous}, + Retained: append([]messagequeue.QueuedMessage(nil), h.queueService.GetStatus(ctx, req.SessionID).Entries...), + } + } + } + if err != nil { + if errors.Is(err, messagequeue.ErrEntryNotFound) { + return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry is no longer pending", nil) + } + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + + h.releaseQueueRemovalAttachments(ctx, removal) + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{fieldEntryID: req.EntryID}) +} + +func (h *QueueHandlers) releaseQueueRemovalAttachments(ctx context.Context, removal *messagequeue.QueueRemovalResult) { + releaser, ok := h.attachmentClaimer.(QueueAttachmentReleaser) + if !ok || removal == nil || len(removal.Removed) == 0 { + return + } + retained := make(map[string]struct{}) + for _, entry := range removal.Retained { + for _, attachment := range entry.Attachments { + if attachment.AttachmentID != "" { + retained[attachment.AttachmentID] = struct{}{} + } + } + } + taskID := removal.Removed[0].TaskID + sessionID := removal.Removed[0].SessionID + seen := make(map[string]struct{}) + released := make([]messagequeue.MessageAttachment, 0) + for _, entry := range removal.Removed { + for _, attachment := range entry.Attachments { + if attachment.AttachmentID == "" { + continue + } + if _, ok := retained[attachment.AttachmentID]; ok { + continue + } + if _, ok := seen[attachment.AttachmentID]; ok { + continue + } + seen[attachment.AttachmentID] = struct{}{} + released = append(released, attachment) + } + } + if len(released) == 0 { + return + } + if err := releaser.ReleaseMessageAttachments(ctx, taskID, sessionID, queueAttachmentsToV1(released)); err != nil { + h.logger.Warn("failed to release removed queue attachments", zap.Error(err)) + } +} + +// wsMergeIntoAboveRequest is the payload for ActionMessageQueueMerge: the +// session whose queue is modified and the id of the entry to fold into the +// entry directly above it. user_id is forwarded for ownership checks and is +// optional (the server defaults to the reserved "user" identity). +type wsMergeIntoAboveRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + EntryID string `json:"entry_id"` + UserID string `json:"user_id,omitempty"` +} + +func (h *QueueHandlers) mergeIntoAbove( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + entryID, queuedBy string, +) (*messagequeue.QueuedMessage, error) { + if h.requiresQueueIdentity() { + return h.queueService.(QueueIdentityMutationService). + MergeIntoAboveForSession(ctx, identity, entryID, queuedBy) + } + return h.queueService.MergeIntoAbove(ctx, identity.SessionID, entryID, queuedBy) +} + +func (h *QueueHandlers) mergeIntoAboveError(msg *ws.Message, err error) (*ws.Message, error) { + switch { + case errors.Is(err, messagequeue.ErrEntryNotFound): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeEntryNotFound, "Queue entry was already drained or not owned by caller", nil) + case errors.Is(err, messagequeue.ErrNoMergeTarget): + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "No mergeable message above this entry", nil) + case errors.Is(err, messagequeue.ErrMergeReferenceOverflow): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeMergeReferenceOverflow, err.Error(), nil) + case errors.Is(err, messagequeue.ErrMergeDisabled): + return ws.NewError(msg.ID, msg.Action, queueErrorCodeMergeDisabled, "Message merging is disabled", nil) + case isQueueIdentityError(err): + return queueAccessDeniedResponse(msg), nil + default: + h.logger.Error("failed to merge queued message", zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to merge queued message", nil) + } +} + +// wsMergeIntoAbove handles ActionMessageQueueMerge, folding the referenced +// queued entry into the entry above it and broadcasting the updated queue. +func (h *QueueHandlers) wsMergeIntoAbove(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsMergeIntoAboveRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if req.EntryID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "entry_id is required", nil) + } + if messagequeue.IsReservedQueuedBy(req.UserID) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, reservedIdentityError(req.UserID), nil) + } + // Default empty user_id to QueuedByUser so the merge ownership guard runs + // against a non-empty owner, mirroring wsUpdateMessage. + queuedBy := req.UserID + if queuedBy == "" { + queuedBy = messagequeue.QueuedByUser + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + merged, err := h.mergeIntoAbove(ctx, identity, req.EntryID, queuedBy) + if err != nil { + return h.mergeIntoAboveError(msg, err) + } + + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{fieldEntryID: merged.ID}) +} + +// wsReorderRequest is the payload for ActionMessageQueueReorder: the session +// whose queue is modified and the complete ordered list of visible pending +// entry ids the caller wants as the new FIFO order. +type wsReorderRequest struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + SessionIncarnationID string `json:"session_incarnation_id"` + OrderedIDs []string `json:"ordered_ids"` +} + +// wsReorder handles ActionMessageQueueReorder, rewriting the session's visible +// pending order to match ordered_ids and broadcasting the updated queue. Any +// drift from the persisted visible set (a drain/remove/merge raced the drag) +// is rejected atomically with queue_changed so the client refetches. +func (h *QueueHandlers) wsReorder(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var req wsReorderRequest + if err := msg.ParsePayload(&req); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) + } + if req.SessionID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) + } + if h.requiresQueueIdentity() && (req.TaskID == "" || req.SessionIncarnationID == "") { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id, session_id, and session_incarnation_id are required", nil) + } + if denied := h.authorizeQueueIdentity(ctx, msg, req.TaskID, req.SessionID, req.SessionIncarnationID); denied != nil { + return denied, nil + } + if len(req.OrderedIDs) == 0 { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "ordered_ids is required", nil) + } + if hasDuplicateIDs(req.OrderedIDs) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "ordered_ids must not contain duplicates", nil) + } + + identity := messagequeue.QueueSessionIdentity{ + TaskID: req.TaskID, SessionID: req.SessionID, SessionIncarnationID: req.SessionIncarnationID, + } + var err error + if h.requiresQueueIdentity() { + err = h.queueService.(QueueIdentityMutationService).ReorderEntriesForSession(ctx, identity, req.OrderedIDs) + } else { + err = h.queueService.ReorderEntries(ctx, req.SessionID, req.OrderedIDs) + } + if err != nil { + if errors.Is(err, messagequeue.ErrQueueChanged) { + return ws.NewError(msg.ID, msg.Action, queueErrorCodeQueueChanged, "Queue changed before the reorder could be applied", nil) + } + if isQueueIdentityError(err) { + return queueAccessDeniedResponse(msg), nil + } + h.logger.Error("failed to reorder queued messages", zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to reorder queued messages", nil) + } + + h.publishStatusForIdentity(ctx, identity) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + fieldSessionID: req.SessionID, + "reordered": len(req.OrderedIDs), + }) +} + +// publishStatus emits the latest QueueStatus on the event bus so the frontend +// updates its store after every mutation. +func (h *QueueHandlers) publishStatus(ctx context.Context, sessionID string, admitted ...*messagequeue.QueuedMessage) { + if h.eventBus == nil { + return + } + status := h.queueService.GetStatus(ctx, sessionID) + eventData := map[string]interface{}{ + fieldSessionID: sessionID, + "entries": status.Entries, + "count": status.Count, + fieldMax: status.Max, + fieldAutoRun: status.AutoRun, + "merge_enabled": status.MergeEnabled, + } + if len(admitted) > 0 && admitted[0] != nil && admitted[0].QueuedBy != "" && !messagequeue.IsReservedQueuedBy(admitted[0].QueuedBy) { + eventData["queued_by"] = admitted[0].QueuedBy + eventData["queued_at"] = admitted[0].QueuedAt + } + if h.sessionTaskResolver != nil { + if taskID, err := h.sessionTaskResolver(ctx, sessionID); err != nil { + h.logger.Warn("resolve session task for queue status event", + zap.String("session_id", sessionID), + zap.Error(err)) + } else if taskID != "" { + eventData["task_id"] = taskID + } + } + _ = h.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( + events.MessageQueueStatusChanged, + "queue-handlers", + eventData, + )) +} +func (h *QueueHandlers) publishStatusForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + admitted ...*messagequeue.QueuedMessage, +) { + if identity.TaskID == "" || identity.SessionIncarnationID == "" { + h.publishStatus(ctx, identity.SessionID, admitted...) + return + } + snapshots, ok := h.queueService.(QueueSnapshotService) + if !ok { + h.publishStatus(ctx, identity.SessionID, admitted...) + return + } + status, err := snapshots.Snapshot(ctx, identity) + if err != nil { + h.logger.Warn("read identity-bound queue status", + zap.String(fieldSessionID, identity.SessionID), + zap.Error(err)) + return + } + h.publishIdentityStatus(ctx, status, admitted...) +} +func (h *QueueHandlers) publishIdentityStatus( + ctx context.Context, + status *messagequeue.QueueStatus, + admitted ...*messagequeue.QueuedMessage, +) { + if h.eventBus == nil || status == nil { + return + } + eventData := map[string]interface{}{ + fieldTaskID: status.TaskID, + fieldSessionID: status.SessionID, + fieldSessionIncarnation: status.SessionIncarnationID, + "status_epoch": status.StatusEpoch, + "status_generation": status.StatusGeneration, + "entries": status.Entries, + "count": status.Count, + fieldMax: status.Max, + fieldAutoRun: status.AutoRun, + "merge_enabled": status.MergeEnabled, + "auto_merge_available": status.AutoMergeAvailable, + } + if status.AutoMergeEnabled != nil { + eventData[fieldAutoMergeEnabled] = *status.AutoMergeEnabled + } + if status.AutoMergeSource != "" { + eventData["auto_merge_source"] = status.AutoMergeSource + } + if status.AutoMergeRevision != nil { + eventData["auto_merge_revision"] = *status.AutoMergeRevision + } + if len(admitted) > 0 && admitted[0] != nil && + admitted[0].QueuedBy != "" && + !messagequeue.IsReservedQueuedBy(admitted[0].QueuedBy) { + eventData["queued_by"] = admitted[0].QueuedBy + eventData["queued_at"] = admitted[0].QueuedAt + } + _ = h.eventBus.Publish(ctx, events.MessageQueueStatusChanged, bus.NewEvent( + events.MessageQueueStatusChanged, + "queue-handlers", + eventData, + )) +} + +func isQueueIdentityError(err error) bool { + return errors.Is(err, messagequeue.ErrSessionIdentityMismatch) || + errors.Is(err, messagequeue.ErrTaskInactive) +} diff --git a/apps/backend/internal/orchestrator/messagequeue/pending_move_expiry_test.go b/apps/backend/internal/orchestrator/messagequeue/pending_move_expiry_test.go index be5897f6c6..54815a430d 100644 --- a/apps/backend/internal/orchestrator/messagequeue/pending_move_expiry_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/pending_move_expiry_test.go @@ -214,7 +214,7 @@ func TestRepository_DeletePendingMoveIfMatchPreservesReplacement(t *testing.T) { } moveB := &PendingMove{ - MoveID: "move-b", TaskID: "task-b", WorkflowID: "wf-b", + MoveID: "move-b", TaskID: "task-a", WorkflowID: "wf-b", WorkflowStepID: "step-b", QueuedAt: queuedAtA.Add(time.Hour), } if err := repo.SetPendingMove(ctx, "sess", moveB); err != nil { diff --git a/apps/backend/internal/orchestrator/messagequeue/pending_move_sweep.go b/apps/backend/internal/orchestrator/messagequeue/pending_move_sweep.go index 46d3db93de..d318b8f089 100644 --- a/apps/backend/internal/orchestrator/messagequeue/pending_move_sweep.go +++ b/apps/backend/internal/orchestrator/messagequeue/pending_move_sweep.go @@ -19,7 +19,8 @@ import ( // ListPendingMoves returns every armed deferred move, keyed by session. func (r *sqliteRepository) ListPendingMoves(ctx context.Context) ([]PendingMoveRecord, error) { rows, err := r.ro.QueryxContext(ctx, ` - SELECT session_id, move_id, task_id, workflow_id, workflow_step_id, step_position, queued_at, actor, sender_session_id + SELECT session_id, move_id, session_incarnation_id, task_id, workflow_id, + workflow_step_id, step_position, queued_at, actor, sender_session_id FROM pending_moves `) if err != nil { @@ -30,26 +31,29 @@ func (r *sqliteRepository) ListPendingMoves(ctx context.Context) ([]PendingMoveR var records []PendingMoveRecord for rows.Next() { var ( - sessionID, moveID, taskID, workflowID, workflowStepID string - position int - queuedAt time.Time - actor, senderSessionID string + sessionID, moveID, sessionIncarnationID, taskID, workflowID, workflowStepID string + position int + queuedAt time.Time + actor, senderSessionID string ) - if err := rows.Scan(&sessionID, &moveID, &taskID, &workflowID, &workflowStepID, - &position, &queuedAt, &actor, &senderSessionID); err != nil { + if err := rows.Scan( + &sessionID, &moveID, &sessionIncarnationID, &taskID, &workflowID, + &workflowStepID, &position, &queuedAt, &actor, &senderSessionID, + ); err != nil { return nil, fmt.Errorf("scan pending move: %w", err) } records = append(records, PendingMoveRecord{ SessionID: sessionID, Move: PendingMove{ - MoveID: moveID, - TaskID: taskID, - WorkflowID: workflowID, - WorkflowStepID: workflowStepID, - Position: position, - QueuedAt: queuedAt, - Actor: actor, - SenderSessionID: senderSessionID, + MoveID: moveID, + SessionIncarnationID: sessionIncarnationID, + TaskID: taskID, + WorkflowID: workflowID, + WorkflowStepID: workflowStepID, + Position: position, + QueuedAt: queuedAt, + Actor: actor, + SenderSessionID: senderSessionID, }, }) } @@ -82,8 +86,8 @@ func (r *sqliteRepository) DeletePendingMoveIfMatch( } result, err := tx.ExecContext(ctx, r.db.Rebind(` DELETE FROM pending_moves - WHERE session_id = ? AND move_id = ? AND queued_at = ? - `), expected.SessionID, expected.Move.MoveID, expected.Move.QueuedAt) + WHERE session_id = ? AND move_id = ? AND session_incarnation_id = ? AND queued_at = ? + `), expected.SessionID, expected.Move.MoveID, expected.Move.SessionIncarnationID, expected.Move.QueuedAt) if err != nil { return false, fmt.Errorf("delete pending move: %w", err) } @@ -136,7 +140,10 @@ func (r *memoryRepository) DeletePendingMoveIfMatch( r.mu.Lock() defer r.mu.Unlock() stored, ok := r.pendingMoves[expected.SessionID] - if !ok || stored.MoveID != expected.Move.MoveID || !stored.QueuedAt.Equal(expected.Move.QueuedAt) { + if !ok || + stored.MoveID != expected.Move.MoveID || + stored.SessionIncarnationID != expected.Move.SessionIncarnationID || + !stored.QueuedAt.Equal(expected.Move.QueuedAt) { return false, nil } delete(r.pendingMoves, expected.SessionID) diff --git a/apps/backend/internal/orchestrator/messagequeue/repository.go b/apps/backend/internal/orchestrator/messagequeue/repository.go index 2d492bb04a..afd2389ddc 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository.go @@ -2,34 +2,53 @@ package messagequeue import "context" +// RepositorySnapshot is one identity-validated queue/state read. A policy-only +// failure preserves independently readable queue entries and Auto-run state. +type RepositorySnapshot struct { + Entries []QueuedMessage + PendingMove *PendingMove + AutoRun bool + AutoMergeOverride *AutoMergeOverride + AutoMergeError error + StatusGeneration int64 +} + // Repository abstracts persistent storage of queued messages and pending moves. // Operations on queued messages are atomic per session. type Repository interface { + ResolveSessionIdentity(ctx context.Context, taskID, sessionID string) (QueueSessionIdentity, error) + Snapshot(ctx context.Context, identity QueueSessionIdentity) (RepositorySnapshot, error) + // Insert appends a new entry at the tail of the session's FIFO queue. // Returns ErrQueueFull if maxPerSession > 0 and the count would exceed it. Insert(ctx context.Context, msg *QueuedMessage, maxPerSession int) error + InsertForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, maxPerSession int) error // Restore reinserts a previously dequeued entry at its original FIFO // position. It is used when a delivery fails after TakeHead so later // messages cannot overtake the failed one. Restore(ctx context.Context, msg *QueuedMessage, maxPerSession int) error + RestoreForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, maxPerSession int) error // AppendOrInsertTail concatenates content onto the tail entry when the tail // exists AND its QueuedBy matches the supplied queuedBy. Otherwise inserts a // new entry. Returns the resulting entry and whether the call appended (true) // or inserted (false). Honors maxPerSession when inserting. AppendOrInsertTail(ctx context.Context, sessionID, taskID, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) + AppendOrInsertTailForSession(ctx context.Context, identity QueueSessionIdentity, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) // InsertOrReplaceByCoalesceKey replaces an existing queued entry with the // same session, queued_by, and metadata coalesce key. If no matching entry // exists, it inserts a new tail entry when allowInsert is true. Returns // ErrEntryNotFound when allowInsert is false and no matching entry exists. InsertOrReplaceByCoalesceKey(ctx context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) + InsertOrReplaceByCoalesceKeyForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) // InsertOrReplaceLifecycleByCoalesceKey is the lifecycle-specific variant // that accepts the entry only while msg.TaskID exists and is not archived. // SQLite implementations make that check and queue mutation one transaction. InsertOrReplaceLifecycleByCoalesceKey(ctx context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) + InsertOrReplaceLifecycleByCoalesceKeyForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) // RequeuePreservingFIFO re-enqueues an entry, preserving both FIFO order // across the supersede→requeue cycle AND coalesce-replace semantics on @@ -49,6 +68,7 @@ type Repository interface { // positions up before inserting. Queue positions stay positive so session // transfer and later queue mutations keep their ordering invariants. RequeuePreservingFIFO(ctx context.Context, msg *QueuedMessage) error + RequeuePreservingFIFOForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage) error // LifecycleGeneration returns the current archive/delete generation for a // task. Lifecycle insertion verifies this generation atomically. @@ -92,19 +112,43 @@ type Repository interface { // SetAutoRun persists the per-session automatic-drain policy independently // of queue contents. SetAutoRun(ctx context.Context, sessionID string, enabled bool) error + SetAutoRunForSession(ctx context.Context, identity QueueSessionIdentity, enabled bool) error + + // GetAutoMergeOverride returns nil while the exact session inherits globally. + GetAutoMergeOverride(ctx context.Context, identity QueueSessionIdentity) (*AutoMergeOverride, error) + + // SetAutoMergeOverride writes an explicit value and advances its session revision. + SetAutoMergeOverride(ctx context.Context, identity QueueSessionIdentity, enabled bool) (AutoMergeOverride, error) // PauseAutoRunIfPending atomically persists OFF only when at least one // visible pending entry remains. Reserved lifecycle rows do not count. PauseAutoRunIfPending(ctx context.Context, sessionID string) (bool, error) + PauseAutoRunIfPendingForSession(ctx context.Context, identity QueueSessionIdentity) (bool, error) // ReserveHeadIfAutoRun reads the policy and reserves the FIFO head in one // per-session transaction. The returned bool is false only when Auto-run is // OFF; nil with true means the enabled queue is empty. ReserveHeadIfAutoRun(ctx context.Context, sessionID string) (*QueuedMessage, bool, error) + ReserveHeadIfAutoRunForSession(ctx context.Context, identity QueueSessionIdentity) (*QueuedMessage, bool, error) + // ReserveHeadForDeliveryIfAutoRunForSession retains any head entry as an + // identity-bound in-flight row until acknowledgement. Passthrough delivery + // uses this so archive/delete can cancel the claim before PTY acceptance. + ReserveHeadForDeliveryIfAutoRunForSession( + ctx context.Context, + identity QueueSessionIdentity, + ) (*QueuedMessage, bool, error) // AcknowledgeByID is an internal dispatch operation that removes a reserved // entry regardless of its server-owned queued_by identity. AcknowledgeByID(ctx context.Context, sessionID, entryID string) error + AcknowledgeByIDForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) error + // ReleaseDeliveryReservationForSession makes an unaccepted retained entry + // visible again without changing its FIFO position. + ReleaseDeliveryReservationForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, + ) error // TakeByID atomically returns and deletes the entry identified by entryID // for sessionID, regardless of its FIFO position. Unlike DeleteByID, it has @@ -113,6 +157,7 @@ type Repository interface { // its own interrupt, not a user-driven "remove queued message" request. // Returns nil, nil if no entry matches (already taken or never existed). TakeByID(ctx context.Context, sessionID, entryID string) (*QueuedMessage, error) + TakeByIDForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) (*QueuedMessage, error) // ClaimSendNow atomically claims the exact ordered source snapshot for an // interrupt-and-replace dispatch. The repository compares every requested @@ -121,6 +166,7 @@ type Repository interface { // lifecycle rows are reserved until AcknowledgeSendNowClaim or // RestoreSendNowClaim. ClaimSendNow(ctx context.Context, sessionID string, expected []QueuedMessage) (*SendNowClaim, error) + ClaimSendNowForSession(ctx context.Context, identity QueueSessionIdentity, expected []QueuedMessage) (*SendNowClaim, error) // RestoreSendNowClaim puts every source back at its original position and // clears durable lifecycle reservations. It must restore the complete claim @@ -142,6 +188,7 @@ type Repository interface { // applies metadata key updates. A nil update value removes that key while // unrelated metadata remains unchanged. UpdateContentAndMetadata(ctx context.Context, sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error + UpdateContentAndMetadataForSession(ctx context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error // MergeIntoAbove folds the entry identified by sourceID into the entry // immediately above it (greatest position strictly below the source) within @@ -156,12 +203,14 @@ type Repository interface { // or a reserved in-flight lifecycle target — returns ErrNoMergeTarget. A // missing source returns ErrEntryNotFound. MergeIntoAbove(ctx context.Context, sessionID, sourceID, queuedBy string) (*QueuedMessage, error) + MergeIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, sourceID, queuedBy string) (*QueuedMessage, error) // AutoMergeIntoAbove attempts to fold the exact source entry into its // immediate predecessor. A compatible merge returns the surviving target // and true. A missing source, missing predecessor, or incompatible pair is a // successful skip and returns false without mutating either row. AutoMergeIntoAbove(ctx context.Context, sessionID, sourceID string) (*QueuedMessage, bool, error) + AutoMergeIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, sourceID string) (*QueuedMessage, bool, error) // AutoMergeCandidateIntoAbove folds a not-yet-admitted candidate message // into the session's tail entry when compatible, without inserting it. Used @@ -170,7 +219,7 @@ type Repository interface { // ErrQueueFull. A missing tail or incompatible pair is a successful skip // and returns false without mutating any row. AutoMergeCandidateIntoAbove(ctx context.Context, candidate *QueuedMessage) (*QueuedMessage, bool, error) - + AutoMergeCandidateIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, candidate *QueuedMessage) (*QueuedMessage, bool, error) // ReorderEntries atomically rewrites the FIFO positions of the session's // pending entries to match orderedIDs. The submitted list must be an exact // permutation of the currently visible pending (not reserved-in-flight) @@ -180,6 +229,7 @@ type Repository interface { // in-flight rows keep their place in the sequence; visible rows are // interleaved in the submitted order and all positions are compacted to // 1..N in one transaction. + ReorderEntriesForSession(ctx context.Context, identity QueueSessionIdentity, orderedIDs []string) error ReorderEntries(ctx context.Context, sessionID string, orderedIDs []string) error // DeleteByID removes a single entry. The session scope (`AND session_id = ?`) @@ -194,21 +244,31 @@ type Repository interface { // of provenance. Durable lifecycle entries already reserved in flight remain. // Returns the exact count removed. DeleteAllBySession(ctx context.Context, sessionID string) (int, error) + // DiscardLifecycleReservation removes a durable in-flight row only when its + // persisted reservation owner matches identity. It deliberately does not + // require identity to remain current: stale workers use it after replacement. + DiscardLifecycleReservation(ctx context.Context, identity QueueSessionIdentity, entryID string) error // TransferSession moves all entries (and any pending move) from oldSessionID // to newSessionID. Used on workflow session switches. TransferSession(ctx context.Context, oldSessionID, newSessionID string) error + // TransferSessionIdentities additionally requires both live incarnations to + // belong to the same task so queue ownership cannot cross task boundaries. + TransferSessionIdentities(ctx context.Context, source, destination QueueSessionIdentity) error // ReplaceSession replaces a session's queued entries and pending move with // the supplied snapshot, preserving queued-message identity fields. ReplaceSession(ctx context.Context, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error + ReplaceSessionForIdentity(ctx context.Context, identity QueueSessionIdentity, entries []QueuedMessage, pendingMove *PendingMove) error // SetPendingMove upserts the deferred workflow move for a session. SetPendingMove(ctx context.Context, sessionID string, move *PendingMove) error + DeleteByIDForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) (*QueueRemovalResult, error) // GetPendingMove returns the deferred move for a session without removing it. // Returns nil, nil if absent. GetPendingMove(ctx context.Context, sessionID string) (*PendingMove, error) + DeleteAllBySessionForIdentity(ctx context.Context, identity QueueSessionIdentity) (*QueueRemovalResult, error) // TakePendingMove returns and removes the deferred move for a session. // Returns nil, nil if absent. diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_auto_merge_override_test.go b/apps/backend/internal/orchestrator/messagequeue/repository_auto_merge_override_test.go new file mode 100644 index 0000000000..bb41413f95 --- /dev/null +++ b/apps/backend/internal/orchestrator/messagequeue/repository_auto_merge_override_test.go @@ -0,0 +1,100 @@ +package messagequeue + +import ( + "context" + "errors" + "testing" +) + +type autoMergeOverrideRepository interface { + GetAutoMergeOverride(context.Context, QueueSessionIdentity) (*AutoMergeOverride, error) + SetAutoMergeOverride(context.Context, QueueSessionIdentity, bool) (AutoMergeOverride, error) +} + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 +func TestMemoryAutoMergeOverridePersistsTriStateAndRejectsTextualReuse(t *testing.T) { + repo := NewMemoryRepository() + policies, ok := repo.(autoMergeOverrideRepository) + if !ok { + t.Fatal("memory repository does not implement session Auto-merge overrides") + } + ctx := context.Background() + first := QueueSessionIdentity{TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation-1"} + replacement := QueueSessionIdentity{TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation-2"} + seedQueueSessionIdentity(t, repo, first) + + inherited, err := policies.GetAutoMergeOverride(ctx, first) + if err != nil { + t.Fatalf("get inherited override: %v", err) + } + if inherited != nil { + t.Fatalf("inherited override = %+v, want nil", inherited) + } + written, err := policies.SetAutoMergeOverride(ctx, first, false) + if err != nil { + t.Fatalf("write explicit OFF: %v", err) + } + if written.Enabled || written.Revision != 1 { + t.Fatalf("explicit OFF = %+v, want false revision 1", written) + } + written, err = policies.SetAutoMergeOverride(ctx, first, true) + if err != nil { + t.Fatalf("write explicit ON: %v", err) + } + if !written.Enabled || written.Revision != 2 { + t.Fatalf("explicit ON = %+v, want true revision 2", written) + } + + inherited, err = policies.GetAutoMergeOverride(ctx, replacement) + if !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("replacement identity error = %v, want ErrSessionIdentityMismatch", err) + } + if inherited != nil { + t.Fatalf("replacement override = %+v, want nil", inherited) + } +} + +func TestSQLiteAutoMergeOverrideReadRejectsReplacedSessionIdentity(t *testing.T) { + repo := newTestSQLiteRepo(t) + ctx := context.Background() + first := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation-1", + } + replacement := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation-2", + } + seedQueueSessionIdentity(t, repo, first) + if _, err := repo.SetAutoMergeOverride(ctx, first, true); err != nil { + t.Fatalf("set first override: %v", err) + } + seedQueueSessionIdentity(t, repo, replacement) + + override, err := repo.GetAutoMergeOverride(ctx, first) + if !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("stale identity error = %v, want ErrSessionIdentityMismatch", err) + } + if override != nil { + t.Fatalf("stale identity override = %+v, want nil", override) + } +} + +func TestSQLiteAutoMergeOverrideClaimsExistingUnboundSessionState(t *testing.T) { + repo := newTestSQLiteRepo(t) + ctx := context.Background() + const sessionID = "session" + if err := repo.SetAutoRun(ctx, sessionID, false); err != nil { + t.Fatalf("create unbound queue session state: %v", err) + } + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: sessionID, SessionIncarnationID: "incarnation", + } + seedQueueSessionIdentity(t, repo, identity) + written, err := repo.SetAutoMergeOverride(ctx, identity, true) + if err != nil { + t.Fatalf("claim queue session state for override: %v", err) + } + if !written.Enabled || written.Revision != 1 { + t.Fatalf("written override = %+v, want ON revision 1", written) + } +} diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_auto_run_test.go b/apps/backend/internal/orchestrator/messagequeue/repository_auto_run_test.go index 18300da998..e89831ae1b 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_auto_run_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_auto_run_test.go @@ -116,6 +116,49 @@ func TestRepositories_ReserveHeadIfAutoRunDistinguishesPausedAndEmpty(t *testing } } +func TestRepositories_AutoRunPolicyDoesNotLeakAcrossSessionIncarnations(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + repo := factory.new(t) + ctx := context.Background() + first := QueueSessionIdentity{ + TaskID: "task-1", + SessionID: "session-1", + SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repo, first) + require.NoError(t, repo.SetAutoRunForSession(ctx, first, false)) + firstSnapshot, err := repo.Snapshot(ctx, first) + require.NoError(t, err) + assert.False(t, firstSnapshot.AutoRun) + + replacement := first + replacement.SessionIncarnationID = "incarnation-2" + seedQueueSessionIdentity(t, repo, replacement) + replacementSnapshot, err := repo.Snapshot(ctx, replacement) + require.NoError(t, err) + assert.True(t, replacementSnapshot.AutoRun) + reserved, enabled, err := repo.ReserveHeadIfAutoRunForSession(ctx, replacement) + require.NoError(t, err) + assert.Nil(t, reserved) + assert.True(t, enabled) + entry := &QueuedMessage{ + SessionID: replacement.SessionID, + TaskID: replacement.TaskID, + Content: "queued", + QueuedBy: QueuedByUser, + } + require.NoError(t, repo.InsertForSession(ctx, replacement, entry, 10)) + paused, err := repo.PauseAutoRunIfPendingForSession(ctx, replacement) + require.NoError(t, err) + assert.True(t, paused) + replacementSnapshot, err = repo.Snapshot(ctx, replacement) + require.NoError(t, err) + assert.False(t, replacementSnapshot.AutoRun) + }) + } +} + func TestRepositories_ClaimSendNowResumesOnlyAcceptedClaim(t *testing.T) { for _, factory := range autoRunRepositoryFactories { t.Run(factory.name, func(t *testing.T) { diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_identity_boundary_test.go b/apps/backend/internal/orchestrator/messagequeue/repository_identity_boundary_test.go new file mode 100644 index 0000000000..0d600304ba --- /dev/null +++ b/apps/backend/internal/orchestrator/messagequeue/repository_identity_boundary_test.go @@ -0,0 +1,368 @@ +package messagequeue + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/stretchr/testify/require" +) + +func TestServiceExactTargetedOperationsRejectReplacementSession(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + queue := NewService(repository, DefaultMaxPerSession, logger.Default()) + first := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, first) + + queued, _, err := queue.QueueMessageWithCoalesceKeyForSession( + ctx, first, "original", "", QueuedByAgent, false, nil, + map[string]interface{}{MetadataCoalesceKey: "automation"}, "automation", true, + ) + require.NoError(t, err) + + replacement := first + replacement.SessionIncarnationID = "incarnation-2" + seedQueueSessionIdentity(t, repository, replacement) + + _, _, err = queue.TakeQueuedEntryForSession(ctx, first, queued.ID) + require.ErrorIs(t, err, ErrSessionIdentityMismatch) + _, _, err = queue.QueueMessageWithCoalesceKeyForSession( + ctx, first, "stale replacement", "", QueuedByAgent, false, nil, + nil, "automation", true, + ) + require.ErrorIs(t, err, ErrSessionIdentityMismatch) + }) + } +} + +func TestRepositoryRetainedDeliveryReservationLifecycle(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, identity) + require.NoError(t, repository.InsertForSession(ctx, identity, &QueuedMessage{ + ID: "queued-1", TaskID: identity.TaskID, SessionID: identity.SessionID, Content: "prompt", + }, DefaultMaxPerSession)) + + reserved, autoRun, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.True(t, autoRun) + require.NotNil(t, reserved) + require.False(t, reserved.IsReservedInFlight()) + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.True(t, stored[0].IsReservedInFlight()) + + require.NoError(t, repository.ReleaseDeliveryReservationForSession(ctx, identity, reserved.ID)) + stored, err = repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.False(t, stored[0].IsReservedInFlight()) + + reserved, autoRun, err = repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.True(t, autoRun) + require.NoError(t, repository.AcknowledgeByIDForSession(ctx, identity, reserved.ID)) + stored, err = repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Empty(t, stored) + }) + } +} + +func TestRepositoryLifecycleCoalescingPreservesInFlightReservation(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, identity) + message := func(id, content string) *QueuedMessage { + return &QueuedMessage{ + ID: id, TaskID: identity.TaskID, SessionID: identity.SessionID, + Content: content, QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{ + MetadataLifecycleDurable: true, + MetadataLifecycleGeneration: float64(0), + MetadataCoalesceKey: "lifecycle:event", + }, + } + } + _, replaced, err := repository.InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx, identity, message("reserved", "first"), "lifecycle:event", 1, true, + ) + require.NoError(t, err) + require.False(t, replaced) + reserved, _, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.Equal(t, "reserved", reserved.ID) + + successor, replaced, err := repository.InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx, identity, message("successor", "second"), "lifecycle:event", 1, true, + ) + require.NoError(t, err) + require.False(t, replaced) + require.Equal(t, "successor", successor.ID) + + successor, replaced, err = repository.InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx, identity, message("replacement", "third"), "lifecycle:event", 1, true, + ) + require.NoError(t, err) + require.True(t, replaced) + require.Equal(t, "successor", successor.ID) + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 2) + require.Equal(t, "first", stored[0].Content) + require.True(t, stored[0].IsReservedInFlight()) + require.Equal(t, "third", stored[1].Content) + require.False(t, stored[1].IsReservedInFlight()) + + require.NoError(t, repository.RequeuePreservingFIFOForSession(ctx, identity, reserved)) + stored, err = repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.Equal(t, "successor", stored[0].ID) + require.Equal(t, "third", stored[0].Content) + require.False(t, stored[0].IsReservedInFlight()) + }) + } +} + +func TestRepositoryCoalescingReleasePrefersPendingSuccessor(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, identity) + message := func(id, content string) *QueuedMessage { + return &QueuedMessage{ + ID: id, TaskID: identity.TaskID, SessionID: identity.SessionID, + Content: content, QueuedBy: QueuedByAgent, + Metadata: map[string]interface{}{MetadataCoalesceKey: "automation"}, + } + } + _, replaced, err := repository.InsertOrReplaceByCoalesceKeyForSession( + ctx, identity, message("reserved", "first"), "automation", 1, true, + ) + require.NoError(t, err) + require.False(t, replaced) + reserved, _, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + + _, replaced, err = repository.InsertOrReplaceByCoalesceKeyForSession( + ctx, identity, message("successor", "second"), "automation", 1, true, + ) + require.NoError(t, err) + require.False(t, replaced) + + require.NoError(t, repository.ReleaseDeliveryReservationForSession(ctx, identity, reserved.ID)) + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.Equal(t, "successor", stored[0].ID) + require.False(t, stored[0].IsReservedInFlight()) + }) + } +} + +func TestRepositoryRetainedOrdinaryReservationRejectsReplacementSession(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + first := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, first) + require.NoError(t, repository.InsertForSession(ctx, first, &QueuedMessage{ + ID: "queued-1", TaskID: first.TaskID, SessionID: first.SessionID, Content: "stale", + }, DefaultMaxPerSession)) + reserved, _, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, first) + require.NoError(t, err) + require.NotNil(t, reserved) + + replacement := first + replacement.SessionIncarnationID = "incarnation-2" + seedQueueSessionIdentity(t, repository, replacement) + reserved, _, err = repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, replacement) + require.NoError(t, err) + require.Nil(t, reserved) + stored, err := repository.ListBySession(ctx, replacement.SessionID) + require.NoError(t, err) + require.Empty(t, stored) + }) + } +} + +func TestMemoryRepositoryAuthorityReplacementClearsSessionState(t *testing.T) { + ctx := context.Background() + current := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + repository := NewMemoryRepositoryWithAuthority( + func(context.Context, string, string) (QueueSessionIdentity, error) { + return current, nil + }, + ).(*memoryRepository) + first, err := repository.ResolveSessionIdentity(ctx, current.TaskID, current.SessionID) + require.NoError(t, err) + require.NoError(t, repository.InsertForSession(ctx, first, &QueuedMessage{ + ID: "queued-1", TaskID: first.TaskID, SessionID: first.SessionID, Content: "stale", + }, DefaultMaxPerSession)) + repository.mu.Lock() + repository.pendingMoves[first.SessionID] = &PendingMove{ + MoveID: "move-1", TaskID: first.TaskID, SessionIncarnationID: first.SessionIncarnationID, + } + repository.mu.Unlock() + + current = QueueSessionIdentity{ + TaskID: "task-2", SessionID: "session-1", SessionIncarnationID: "incarnation-2", + } + replacement, err := repository.ResolveSessionIdentity(ctx, current.TaskID, current.SessionID) + require.NoError(t, err) + snapshot, err := repository.Snapshot(ctx, replacement) + require.NoError(t, err) + require.Empty(t, snapshot.Entries) + require.Nil(t, snapshot.PendingMove) +} + +func TestMemoryRepositoryPurgeTaskClearsPendingMoves(t *testing.T) { + repository := NewMemoryRepository().(*memoryRepository) + repository.mu.Lock() + repository.pendingMoves["session-1"] = &PendingMove{ + MoveID: "move-1", TaskID: "task-1", SessionIncarnationID: "incarnation-1", + } + repository.mu.Unlock() + + _, err := repository.PurgeTask(context.Background(), "task-1") + require.NoError(t, err) + repository.mu.Lock() + defer repository.mu.Unlock() + require.NotContains(t, repository.pendingMoves, "session-1") +} + +func TestRepositoryRetainedDeliveryReservationRejectsUserMutation(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, identity) + require.NoError(t, repository.InsertForSession(ctx, identity, &QueuedMessage{ + ID: "queued-1", TaskID: identity.TaskID, SessionID: identity.SessionID, + Content: "original", QueuedBy: QueuedByUser, + }, DefaultMaxPerSession)) + reserved, _, err := repository.ReserveHeadForDeliveryIfAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.NotNil(t, reserved) + + err = repository.UpdateContentAndMetadataForSession( + ctx, identity, reserved.ID, "edited", nil, nil, QueuedByUser, + ) + require.ErrorIs(t, err, ErrEntryNotFound) + _, err = repository.DeleteByIDForSession(ctx, identity, reserved.ID) + require.ErrorIs(t, err, ErrEntryNotFound) + + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Len(t, stored, 1) + require.Equal(t, "original", stored[0].Content) + require.True(t, stored[0].IsReservedInFlight()) + }) + } +} + +func TestRepositoryIdentityRestoreRejectsForeignTaskEntries(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repository, identity) + + err := repository.ReplaceSessionForIdentity(ctx, identity, []QueuedMessage{{ + ID: "foreign", TaskID: "task-2", SessionID: identity.SessionID, Content: "foreign", + }}, nil) + require.ErrorIs(t, err, ErrSessionIdentityMismatch) + + stored, err := repository.ListBySession(ctx, identity.SessionID) + require.NoError(t, err) + require.Empty(t, stored) + }) + } +} + +func TestRepositoriesTransferSessionIdentitiesRejectCrossTaskStateMove(t *testing.T) { + for _, factory := range autoRunRepositoryFactories { + t.Run(factory.name, func(t *testing.T) { + ctx := context.Background() + repository := factory.new(t) + source := QueueSessionIdentity{ + TaskID: "task-source", SessionID: "session-source", SessionIncarnationID: "incarnation-source", + } + destination := QueueSessionIdentity{ + TaskID: "task-destination", SessionID: "session-destination", SessionIncarnationID: "incarnation-destination", + } + seedQueueSessionIdentity(t, repository, source) + seedQueueSessionIdentity(t, repository, destination) + + require.NoError(t, repository.InsertForSession(ctx, source, &QueuedMessage{ + ID: "source-entry", TaskID: source.TaskID, SessionID: source.SessionID, Content: "source", + }, DefaultMaxPerSession)) + require.NoError(t, repository.InsertForSession(ctx, destination, &QueuedMessage{ + ID: "destination-entry", TaskID: destination.TaskID, SessionID: destination.SessionID, Content: "destination", + }, DefaultMaxPerSession)) + require.NoError(t, repository.SetPendingMove(ctx, source.SessionID, &PendingMove{ + MoveID: "move-source", + SessionIncarnationID: source.SessionIncarnationID, + TaskID: source.TaskID, + WorkflowID: "workflow-source", + WorkflowStepID: "step-source", + })) + require.NoError(t, repository.SetAutoRunForSession(ctx, source, false)) + _, err := repository.SetAutoMergeOverride(ctx, source, true) + require.NoError(t, err) + _, err = repository.SetAutoMergeOverride(ctx, destination, false) + require.NoError(t, err) + + sourceBefore, err := repository.Snapshot(ctx, source) + require.NoError(t, err) + destinationBefore, err := repository.Snapshot(ctx, destination) + require.NoError(t, err) + sourceBefore.StatusGeneration = 0 + destinationBefore.StatusGeneration = 0 + + err = repository.TransferSessionIdentities(ctx, source, destination) + require.ErrorIs(t, err, ErrSessionIdentityMismatch) + + sourceAfter, err := repository.Snapshot(ctx, source) + require.NoError(t, err) + destinationAfter, err := repository.Snapshot(ctx, destination) + require.NoError(t, err) + sourceAfter.StatusGeneration = 0 + destinationAfter.StatusGeneration = 0 + require.Equal(t, sourceBefore, sourceAfter) + require.Equal(t, destinationBefore, destinationAfter) + }) + } +} diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_memory.go b/apps/backend/internal/orchestrator/messagequeue/repository_memory.go index 0bc738c471..b271373e94 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_memory.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_memory.go @@ -13,25 +13,153 @@ import ( // memoryRepository is an in-memory Repository implementation used in tests and // any deployment that explicitly opts into ephemeral queueing. type memoryRepository struct { - mu sync.Mutex - entries map[string][]*QueuedMessage // sessionID -> ordered list (head = index 0) - nextPosition map[string]int64 // sessionID -> monotonic counter - pendingMoves map[string]*PendingMove - generation map[string]int64 - autoRun map[string]bool + mu sync.Mutex + entries map[string][]*QueuedMessage // sessionID -> ordered list (head = index 0) + nextPosition map[string]int64 // sessionID -> monotonic counter + pendingMoves map[string]*PendingMove + generation map[string]int64 + sendNowGeneration map[string]int64 + autoRun map[string]bool + autoRunIncarnation map[string]string + autoMergeOverrides map[QueueSessionIdentity]AutoMergeOverride + identities map[string]QueueSessionIdentity + statusGeneration map[string]int64 + authority func(context.Context, string, string) (QueueSessionIdentity, error) } // NewMemoryRepository returns an in-memory Repository. Suitable for tests. func NewMemoryRepository() Repository { return &memoryRepository{ - entries: make(map[string][]*QueuedMessage), - nextPosition: make(map[string]int64), - pendingMoves: make(map[string]*PendingMove), - generation: make(map[string]int64), - autoRun: make(map[string]bool), + entries: make(map[string][]*QueuedMessage), + nextPosition: make(map[string]int64), + pendingMoves: make(map[string]*PendingMove), + generation: make(map[string]int64), + sendNowGeneration: make(map[string]int64), + autoRun: make(map[string]bool), + autoRunIncarnation: make(map[string]string), + autoMergeOverrides: make(map[QueueSessionIdentity]AutoMergeOverride), + statusGeneration: make(map[string]int64), + identities: make(map[string]QueueSessionIdentity), } } +// NewMemoryRepositoryWithAuthority returns an in-memory queue whose immutable +// identities come from the supplied task-session authority. +func NewMemoryRepositoryWithAuthority( + authority func(context.Context, string, string) (QueueSessionIdentity, error), +) Repository { + repo := NewMemoryRepository().(*memoryRepository) + repo.authority = authority + return repo +} + +func (r *memoryRepository) clearSessionStateLocked(sessionID string) { + delete(r.entries, sessionID) + delete(r.nextPosition, sessionID) + delete(r.pendingMoves, sessionID) + delete(r.sendNowGeneration, sessionID) + delete(r.autoRun, sessionID) + delete(r.autoRunIncarnation, sessionID) + delete(r.statusGeneration, sessionID) + delete(r.identities, sessionID) + for identity := range r.autoMergeOverrides { + if identity.SessionID == sessionID { + delete(r.autoMergeOverrides, identity) + } + } +} + +func (r *memoryRepository) bindIdentityLocked(identity QueueSessionIdentity) error { + if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" { + return ErrSessionIdentityMismatch + } + if current, ok := r.identities[identity.SessionID]; ok { + if current != identity { + return ErrSessionIdentityMismatch + } + return nil + } + if identity.SessionIncarnationID != "memory:"+identity.SessionID { + return ErrSessionIdentityMismatch + } + for _, entry := range r.entries[identity.SessionID] { + if entry.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + } + r.identities[identity.SessionID] = identity + return nil +} + +func (r *memoryRepository) ResolveSessionIdentity(ctx context.Context, taskID, sessionID string) (QueueSessionIdentity, error) { + if taskID == "" || sessionID == "" { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + if r.authority != nil { + identity, err := r.authority(ctx, taskID, sessionID) + if err != nil { + return QueueSessionIdentity{}, err + } + if identity.TaskID != taskID || identity.SessionID != sessionID || identity.SessionIncarnationID == "" { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + r.mu.Lock() + defer r.mu.Unlock() + if existing, ok := r.identities[sessionID]; ok && existing != identity { + r.clearSessionStateLocked(sessionID) + } + r.identities[sessionID] = identity + return identity, nil + } + r.mu.Lock() + defer r.mu.Unlock() + if identity, ok := r.identities[sessionID]; ok { + if identity.TaskID != taskID { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + return identity, nil + } + identity := QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: "memory:" + sessionID, + } + r.identities[sessionID] = identity + return identity, nil +} + +func (r *memoryRepository) Snapshot( + _ context.Context, + identity QueueSessionIdentity, +) (RepositorySnapshot, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return RepositorySnapshot{}, err + } + entries := make([]QueuedMessage, len(r.entries[identity.SessionID])) + for index, entry := range r.entries[identity.SessionID] { + entries[index] = *entry + } + autoRun := r.autoRunForIdentityLocked(identity) + var override *AutoMergeOverride + if stored, ok := r.autoMergeOverrides[identity]; ok { + copy := stored + override = © + } + var pendingMove *PendingMove + if stored := r.pendingMoves[identity.SessionID]; stored != nil { + copy := *stored + pendingMove = © + } + r.statusGeneration[identity.SessionID]++ + return RepositorySnapshot{ + Entries: entries, + PendingMove: pendingMove, + AutoRun: autoRun, + AutoMergeOverride: override, + StatusGeneration: r.statusGeneration[identity.SessionID], + }, nil +} + // LifecycleGeneration returns the current archive/delete generation for a task. func (r *memoryRepository) LifecycleGeneration(_ context.Context, taskID string) (int64, error) { r.mu.Lock() @@ -60,6 +188,11 @@ func (r *memoryRepository) PurgeTask(_ context.Context, taskID string) (int, err } r.entries[sessionID] = kept } + for sessionID, move := range r.pendingMoves { + if move != nil && move.TaskID == taskID { + delete(r.pendingMoves, sessionID) + } + } r.generation[taskID]++ return removed, nil } @@ -96,10 +229,77 @@ func (r *memoryRepository) Insert(_ context.Context, msg *QueuedMessage, maxPerS return r.insertLocked(msg, maxPerSession) } +func (r *memoryRepository) InsertForSession(_ context.Context, identity QueueSessionIdentity, msg *QueuedMessage, maxPerSession int) error { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.insertLocked(msg, maxPerSession) +} + +func (r *memoryRepository) InsertForSessionWithClaim(_ context.Context, identity QueueSessionIdentity, msg *QueuedMessage, _ QueueAttachmentClaim, maxPerSession int) error { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.insertLocked(msg, maxPerSession) +} + +func (r *memoryRepository) InsertForSessionWithPolicy( + _ context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + _ *QueueAttachmentClaim, + maxPerSession int, + policy AutoMergePolicy, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + if err := r.validateAutoMergePolicyLocked(identity, policy); err != nil { + return err + } + return r.insertLocked(msg, maxPerSession) +} + // Restore reinserts a previously dequeued entry at its original FIFO position. func (r *memoryRepository) Restore(_ context.Context, msg *QueuedMessage, maxPerSession int) error { r.mu.Lock() defer r.mu.Unlock() + return r.restoreLocked(msg, maxPerSession) +} + +func (r *memoryRepository) RestoreForSession( + _ context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + maxPerSession int, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.restoreLocked(msg, maxPerSession) +} + +func (r *memoryRepository) restoreLocked(msg *QueuedMessage, maxPerSession int) error { list := r.entries[msg.SessionID] if maxPerSession > 0 && len(list) >= maxPerSession { return ErrQueueFull @@ -122,6 +322,67 @@ func (r *memoryRepository) Restore(_ context.Context, msg *QueuedMessage, maxPer return nil } +func (r *memoryRepository) GetAutoMergeOverride( + _ context.Context, + identity QueueSessionIdentity, +) (*AutoMergeOverride, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.validateIdentityLocked(identity); err != nil { + return nil, err + } + override, ok := r.autoMergeOverrides[identity] + if !ok { + return nil, nil + } + return &override, nil +} + +func (r *memoryRepository) SetAutoMergeOverride( + _ context.Context, + identity QueueSessionIdentity, + enabled bool, +) (AutoMergeOverride, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return AutoMergeOverride{}, err + } + override := r.autoMergeOverrides[identity] + override.Enabled = enabled + override.Revision++ + r.autoMergeOverrides[identity] = override + return override, nil +} + +func (r *memoryRepository) validateIdentityLocked(identity QueueSessionIdentity) error { + if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" { + return ErrSessionIdentityMismatch + } + current, ok := r.identities[identity.SessionID] + if !ok || current != identity { + return ErrSessionIdentityMismatch + } + return nil +} + +func (r *memoryRepository) validateAutoMergePolicyLocked(identity QueueSessionIdentity, policy AutoMergePolicy) error { + override, exists := r.autoMergeOverrides[identity] + switch policy.Source { + case AutoMergeSourceGlobal: + if exists { + return ErrAutoMergePolicyChanged + } + case AutoMergeSourceSession: + if !exists || override.Enabled != policy.Enabled || override.Revision != policy.Revision { + return ErrAutoMergePolicyChanged + } + default: + return ErrAutoMergePolicyChanged + } + return nil +} + // insertLocked performs the actual insert. Caller must already hold r.mu. func (r *memoryRepository) insertLocked(msg *QueuedMessage, maxPerSession int) error { list := r.entries[msg.SessionID] @@ -160,25 +421,45 @@ func (r *memoryRepository) insertLocked(msg *QueuedMessage, maxPerSession int) e func (r *memoryRepository) RequeuePreservingFIFO(_ context.Context, msg *QueuedMessage) error { r.mu.Lock() defer r.mu.Unlock() + return r.requeuePreservingFIFOLocked(msg, "") +} + +func (r *memoryRepository) RequeuePreservingFIFOForSession( + _ context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.requeuePreservingFIFOLocked(msg, identity.SessionIncarnationID) +} + +func (r *memoryRepository) requeuePreservingFIFOLocked( + msg *QueuedMessage, + expectedIncarnationID string, +) error { + if msg.IsReservedLifecycleDelivery() { + return r.releaseLifecycleReservationForRetryLocked(msg, expectedIncarnationID) + } list := r.entries[msg.SessionID] coalesceKey := metadataString(msg.Metadata, MetadataCoalesceKey) - // Coalesce-replace: only when caller supplied a coalesce key. This - // matches the original RequeueMessage's branching (coalesceKey != "" - // takes the coalesce-replace path; empty coalesceKey takes the - // tail-append path). A bare requeue of a message with no coalesce - // key must NOT collapse onto an unrelated same-sender entry. + + // Coalesce-replace only when the caller supplied a key. Reserved rows + // belong to an active delivery and cannot be replacement targets. if coalesceKey != "" { for _, existing := range list { - if existing.QueuedBy != msg.QueuedBy { + if existing.IsReservedInFlight() || existing.QueuedBy != msg.QueuedBy { continue } if metadataString(existing.Metadata, MetadataCoalesceKey) != coalesceKey { continue } - // Coalesce hit: replace in place. The retry keeps the existing - // entry's position, which by construction is the most head-of- - // queue position for this coalesce key — supersede→requeue of - // the same content stays FIFO at the same slot. existing.TaskID = msg.TaskID existing.Content = msg.Content existing.Model = msg.Model @@ -210,6 +491,53 @@ func (r *memoryRepository) RequeuePreservingFIFO(_ context.Context, msg *QueuedM return nil } +func (r *memoryRepository) releaseLifecycleReservationForRetryLocked( + msg *QueuedMessage, + expectedIncarnationID string, +) error { + for index, existing := range r.entries[msg.SessionID] { + if existing.ID != msg.ID || !existing.IsReservedInFlight() { + continue + } + if expectedIncarnationID != "" && + lifecycleReservationIncarnation(existing.Metadata) != expectedIncarnationID { + return ErrEntryNotFound + } + coalesceKey := metadataString(existing.Metadata, MetadataCoalesceKey) + if r.hasPendingCoalescedSuccessorLocked( + msg.SessionID, + index, + existing.QueuedBy, + coalesceKey, + ) { + r.removeEntryLocked(msg.SessionID, index) + return nil + } + existing.Metadata = clearReservedMetadata(existing.Metadata) + return nil + } + return ErrEntryNotFound +} + +func (r *memoryRepository) hasPendingCoalescedSuccessorLocked( + sessionID string, + excludedIndex int, + queuedBy, coalesceKey string, +) bool { + if coalesceKey == "" { + return false + } + for index, successor := range r.entries[sessionID] { + if index == excludedIndex || successor.IsReservedInFlight() || successor.QueuedBy != queuedBy { + continue + } + if metadataString(successor.Metadata, MetadataCoalesceKey) == coalesceKey { + return true + } + } + return false +} + func (r *memoryRepository) nextRequeuePositionLocked(sessionID string, list []*QueuedMessage) int64 { if len(list) == 0 { r.nextPosition[sessionID] = 1 @@ -243,7 +571,19 @@ func (r *memoryRepository) nextRequeuePositionLocked(sessionID string, list []*Q func (r *memoryRepository) AppendOrInsertTail(_ context.Context, sessionID, taskID, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { r.mu.Lock() defer r.mu.Unlock() + return r.appendOrInsertTailLocked(sessionID, taskID, content, model, queuedBy, planMode, attachments, metadata, maxPerSession) +} +func (r *memoryRepository) AppendOrInsertTailForSession(_ context.Context, identity QueueSessionIdentity, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.appendOrInsertTailLocked(identity.SessionID, identity.TaskID, content, model, queuedBy, planMode, attachments, metadata, maxPerSession) +} + +func (r *memoryRepository) appendOrInsertTailLocked(sessionID, taskID, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { list := r.entries[sessionID] if len(list) > 0 { tail := list[len(list)-1] @@ -274,7 +614,35 @@ func (r *memoryRepository) AppendOrInsertTail(_ context.Context, sessionID, task func (r *memoryRepository) InsertOrReplaceByCoalesceKey(_ context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) { r.mu.Lock() defer r.mu.Unlock() + return r.insertOrReplaceByCoalesceKeyLocked(msg, coalesceKey, maxPerSession, allowInsert) +} +func (r *memoryRepository) InsertOrReplaceByCoalesceKeyForSession( + _ context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.insertOrReplaceByCoalesceKeyLocked(msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *memoryRepository) insertOrReplaceByCoalesceKeyLocked( + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + reservedMatch := false for _, existing := range r.entries[msg.SessionID] { if existing.QueuedBy != msg.QueuedBy { continue @@ -282,39 +650,8 @@ func (r *memoryRepository) InsertOrReplaceByCoalesceKey(_ context.Context, msg * if metadataString(existing.Metadata, MetadataCoalesceKey) != coalesceKey { continue } - if msg.QueuedAt.IsZero() { - msg.QueuedAt = time.Now().UTC() - } - existing.TaskID = msg.TaskID - existing.Content = msg.Content - existing.Model = msg.Model - existing.PlanMode = msg.PlanMode - existing.Attachments = msg.Attachments - existing.Metadata = msg.Metadata - existing.QueuedAt = msg.QueuedAt - out := *existing - return &out, true, nil - } - if !allowInsert { - return nil, false, ErrEntryNotFound - } - if err := r.insertLocked(msg, maxPerSession); err != nil { - return nil, false, err - } - return msg, false, nil -} - -// The memory queue has no task store. Tests which need an archive race wrap -// this method with their deterministic active-task guard. -func (r *memoryRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) { - r.mu.Lock() - defer r.mu.Unlock() - expected, ok := lifecycleGenerationFromMetadata(msg.Metadata) - if !ok || expected != r.generation[msg.TaskID] { - return nil, false, ErrLifecycleCancelled - } - for _, existing := range r.entries[msg.SessionID] { - if existing.QueuedBy != msg.QueuedBy || metadataString(existing.Metadata, MetadataCoalesceKey) != coalesceKey { + if existing.IsReservedInFlight() { + reservedMatch = true continue } if msg.QueuedAt.IsZero() { @@ -333,6 +670,83 @@ func (r *memoryRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Con if !allowInsert { return nil, false, ErrEntryNotFound } + if reservedMatch { + maxPerSession = 0 + } + if err := r.insertLocked(msg, maxPerSession); err != nil { + return nil, false, err + } + return msg, false, nil +} +func (r *memoryRepository) InsertOrReplaceLifecycleByCoalesceKey( + _ context.Context, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.insertOrReplaceLifecycleLocked(msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *memoryRepository) InsertOrReplaceLifecycleByCoalesceKeyForSession( + _ context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.insertOrReplaceLifecycleLocked(msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *memoryRepository) insertOrReplaceLifecycleLocked( + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + expected, ok := lifecycleGenerationFromMetadata(msg.Metadata) + if !ok || expected != r.generation[msg.TaskID] { + return nil, false, ErrLifecycleCancelled + } + reservedMatch := false + for _, existing := range r.entries[msg.SessionID] { + if existing.QueuedBy != msg.QueuedBy || metadataString(existing.Metadata, MetadataCoalesceKey) != coalesceKey { + continue + } + if existing.IsReservedInFlight() { + reservedMatch = true + continue + } + if msg.QueuedAt.IsZero() { + msg.QueuedAt = time.Now().UTC() + } + existing.TaskID = msg.TaskID + existing.Content = msg.Content + existing.Model = msg.Model + existing.PlanMode = msg.PlanMode + existing.Attachments = msg.Attachments + existing.Metadata = msg.Metadata + existing.QueuedAt = msg.QueuedAt + out := *existing + return &out, true, nil + } + if !allowInsert { + return nil, false, ErrEntryNotFound + } + if reservedMatch { + maxPerSession = 0 + } if err := r.insertLocked(msg, maxPerSession); err != nil { return nil, false, err } @@ -404,31 +818,48 @@ func (r *memoryRepository) TakeHead(_ context.Context, sessionID string) (*Queue func (r *memoryRepository) ReserveHead(_ context.Context, sessionID string) (*QueuedMessage, error) { r.mu.Lock() defer r.mu.Unlock() - return r.reserveHeadLocked(sessionID), nil + return r.reserveHeadLocked(sessionID, nil, false), nil } -func (r *memoryRepository) reserveHeadLocked(sessionID string) *QueuedMessage { +func (r *memoryRepository) reserveHeadLocked( + sessionID string, + identity *QueueSessionIdentity, + retainOrdinary bool, +) *QueuedMessage { list := r.entries[sessionID] if len(list) == 0 { return nil } head := list[0] out := *head - if head.IsDurableLifecycle() { - // Mirror the SQLite reservation: the stored row is flagged in flight so - // queue status stops listing it, while the returned copy keeps the - // unmarked metadata a requeue would write back. + if head.IsDurableLifecycle() || retainOrdinary { + reservationIncarnation := lifecycleReservationIncarnation(head.Metadata) + if identity != nil && reservationIncarnation != "" && + reservationIncarnation != identity.SessionIncarnationID { + r.removeEntryLocked(sessionID, 0) + return r.reserveHeadLocked(sessionID, identity, retainOrdinary) + } out.Metadata = clearReservedMetadata(head.Metadata) - out.reservedLifecycleDelivery = true - head.Metadata = markReservedMetadata(out.Metadata) + out.reservedLifecycleDelivery = head.IsDurableLifecycle() + if identity != nil { + out.reservationIdentity = *identity + head.Metadata = markReservedMetadataForIncarnation(out.Metadata, identity.SessionIncarnationID) + } else { + head.Metadata = markReservedMetadata(out.Metadata) + } return &out } - r.entries[sessionID] = list[1:] + r.removeEntryLocked(sessionID, 0) + return &out +} + +func (r *memoryRepository) removeEntryLocked(sessionID string, index int) { + list := r.entries[sessionID] + r.entries[sessionID] = append(list[:index], list[index+1:]...) if len(r.entries[sessionID]) == 0 { delete(r.entries, sessionID) delete(r.nextPosition, sessionID) } - return &out } // GetAutoRun returns true when no explicit policy exists. @@ -442,12 +873,57 @@ func (r *memoryRepository) autoRunLocked(sessionID string) bool { enabled, exists := r.autoRun[sessionID] return !exists || enabled } +func (r *memoryRepository) autoRunForIdentityLocked(identity QueueSessionIdentity) bool { + enabled, exists := r.autoRun[identity.SessionID] + if !exists { + return true + } + incarnationID := r.autoRunIncarnation[identity.SessionID] + if incarnationID == "" { + r.autoRunIncarnation[identity.SessionID] = identity.SessionIncarnationID + return enabled + } + if incarnationID != identity.SessionIncarnationID { + return true + } + return enabled +} + +func identityPointer(identity QueueSessionIdentity) *QueueSessionIdentity { + if identity.SessionIncarnationID == "" { + return nil + } + return &identity +} + +func (r *memoryRepository) setAutoRunLocked( + identity *QueueSessionIdentity, + sessionID string, + enabled bool, +) { + r.autoRun[sessionID] = enabled + if identity == nil { + r.autoRunIncarnation[sessionID] = "" + return + } + r.autoRunIncarnation[sessionID] = identity.SessionIncarnationID +} // SetAutoRun persists the per-session automatic-drain policy. func (r *memoryRepository) SetAutoRun(_ context.Context, sessionID string, enabled bool) error { r.mu.Lock() defer r.mu.Unlock() - r.autoRun[sessionID] = enabled + r.setAutoRunLocked(nil, sessionID, enabled) + return nil +} + +func (r *memoryRepository) SetAutoRunForSession(_ context.Context, identity QueueSessionIdentity, enabled bool) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + r.setAutoRunLocked(&identity, identity.SessionID, enabled) return nil } @@ -455,14 +931,33 @@ func (r *memoryRepository) SetAutoRun(_ context.Context, sessionID string, enabl func (r *memoryRepository) PauseAutoRunIfPending(_ context.Context, sessionID string) (bool, error) { r.mu.Lock() defer r.mu.Unlock() + return r.pauseAutoRunIfPendingLocked(nil, sessionID), nil +} + +func (r *memoryRepository) PauseAutoRunIfPendingForSession( + _ context.Context, + identity QueueSessionIdentity, +) (bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return false, err + } + return r.pauseAutoRunIfPendingLocked(&identity, identity.SessionID), nil +} + +func (r *memoryRepository) pauseAutoRunIfPendingLocked( + identity *QueueSessionIdentity, + sessionID string, +) bool { for _, msg := range r.entries[sessionID] { if msg.IsReservedInFlight() { continue } - r.autoRun[sessionID] = false - return true, nil + r.setAutoRunLocked(identity, sessionID, false) + return true } - return false, nil + return false } // ReserveHeadIfAutoRun atomically checks policy and reserves the FIFO head. @@ -472,7 +967,34 @@ func (r *memoryRepository) ReserveHeadIfAutoRun(_ context.Context, sessionID str if !r.autoRunLocked(sessionID) { return nil, false, nil } - return r.reserveHeadLocked(sessionID), true, nil + return r.reserveHeadLocked(sessionID, nil, false), true, nil +} + +func (r *memoryRepository) ReserveHeadIfAutoRunForSession(_ context.Context, identity QueueSessionIdentity) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, true, err + } + if !r.autoRunForIdentityLocked(identity) { + return nil, false, nil + } + return r.reserveHeadLocked(identity.SessionID, &identity, false), true, nil +} + +func (r *memoryRepository) ReserveHeadForDeliveryIfAutoRunForSession( + _ context.Context, + identity QueueSessionIdentity, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, true, err + } + if !r.autoRunForIdentityLocked(identity) { + return nil, false, nil + } + return r.reserveHeadLocked(identity.SessionID, &identity, true), true, nil } // AcknowledgeByID removes a reserved durable entry after executor acceptance. @@ -494,18 +1016,112 @@ func (r *memoryRepository) AcknowledgeByID(_ context.Context, sessionID, entryID return ErrEntryNotFound } +func (r *memoryRepository) AcknowledgeByIDForSession( + _ context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + for index, msg := range r.entries[identity.SessionID] { + if msg.ID != entryID { + continue + } + if !msg.IsReservedInFlight() || + lifecycleReservationIncarnation(msg.Metadata) != identity.SessionIncarnationID { + return ErrEntryNotFound + } + r.removeEntryLocked(identity.SessionID, index) + return nil + } + return ErrEntryNotFound +} + +func (r *memoryRepository) ReleaseDeliveryReservationForSession( + _ context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + for index, msg := range r.entries[identity.SessionID] { + if msg.ID != entryID { + continue + } + if !msg.IsReservedInFlight() || + lifecycleReservationIncarnation(msg.Metadata) != identity.SessionIncarnationID { + return ErrEntryNotFound + } + coalesceKey := metadataString(msg.Metadata, MetadataCoalesceKey) + if r.hasPendingCoalescedSuccessorLocked( + identity.SessionID, + index, + msg.QueuedBy, + coalesceKey, + ) { + r.removeEntryLocked(identity.SessionID, index) + return nil + } + msg.Metadata = clearReservedMetadata(msg.Metadata) + return nil + } + return ErrEntryNotFound +} +func (r *memoryRepository) DiscardLifecycleReservation( + _ context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + r.mu.Lock() + defer r.mu.Unlock() + for index, msg := range r.entries[identity.SessionID] { + if msg.ID != entryID { + continue + } + if !msg.IsReservedInFlight() || + lifecycleReservationIncarnation(msg.Metadata) != identity.SessionIncarnationID { + return ErrEntryNotFound + } + r.removeEntryLocked(identity.SessionID, index) + return nil + } + return ErrEntryNotFound +} + // TakeByID atomically returns and deletes the entry identified by entryID, // regardless of its FIFO position. Unlike DeleteByID, it has no // QueuedByAgent guard — see the Repository interface doc comment. func (r *memoryRepository) TakeByID(_ context.Context, sessionID, entryID string) (*QueuedMessage, error) { r.mu.Lock() defer r.mu.Unlock() + return r.takeByIDLocked(sessionID, entryID), nil +} +func (r *memoryRepository) TakeByIDForSession( + _ context.Context, + identity QueueSessionIdentity, + entryID string, +) (*QueuedMessage, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, err + } + return r.takeByIDLocked(identity.SessionID, entryID), nil +} + +func (r *memoryRepository) takeByIDLocked(sessionID, entryID string) *QueuedMessage { list, ok := r.entries[sessionID] if !ok { - return nil, nil + return nil } - for i, m := range list { - if m.ID != entryID { + for i, message := range list { + if message.ID != entryID { continue } r.entries[sessionID] = append(list[:i], list[i+1:]...) @@ -513,16 +1129,33 @@ func (r *memoryRepository) TakeByID(_ context.Context, sessionID, entryID string delete(r.entries, sessionID) delete(r.nextPosition, sessionID) } - out := *m - return &out, nil + out := *message + return &out } - return nil, nil + return nil } // ClaimSendNow atomically claims the exact ordered source snapshot for a send-now dispatch. func (r *memoryRepository) ClaimSendNow(_ context.Context, sessionID string, expected []QueuedMessage) (*SendNowClaim, error) { r.mu.Lock() defer r.mu.Unlock() + return r.claimSendNowLocked(r.identities[sessionID], sessionID, expected) +} + +func (r *memoryRepository) ClaimSendNowForSession(_ context.Context, identity QueueSessionIdentity, expected []QueuedMessage) (*SendNowClaim, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, err + } + return r.claimSendNowLocked(identity, identity.SessionID, expected) +} + +func (r *memoryRepository) claimSendNowLocked( + identity QueueSessionIdentity, + sessionID string, + expected []QueuedMessage, +) (*SendNowClaim, error) { if len(expected) == 0 { return nil, ErrSendNowEmpty } @@ -551,6 +1184,7 @@ func (r *memoryRepository) ClaimSendNow(_ context.Context, sessionID string, exp } sources := cloneSendNowSources(selected) + bindSendNowLifecycleReservations(sources, identity) envelope, err := BuildSendNowEnvelope(sources) if err != nil { return nil, err @@ -569,7 +1203,10 @@ func (r *memoryRepository) ClaimSendNow(_ context.Context, sessionID string, exp continue } if entry.IsDurableLifecycle() { - entry.Metadata = markReservedMetadata(entry.Metadata) + entry.Metadata = markReservedMetadataForIncarnation( + entry.Metadata, + identity.SessionIncarnationID, + ) remaining = append(remaining, entry) } } @@ -579,19 +1216,56 @@ func (r *memoryRepository) ClaimSendNow(_ context.Context, sessionID string, exp } else { r.entries[sessionID] = remaining } - r.autoRun[sessionID] = true - return &SendNowClaim{Sources: sources, Dispatch: *envelope, SourceGenerations: generations}, nil + r.setAutoRunLocked(identityPointer(identity), sessionID, true) + r.sendNowGeneration[sessionID]++ + return &SendNowClaim{ + Identity: identity, + OperationGeneration: r.sendNowGeneration[sessionID], + Sources: sources, + Dispatch: *envelope, + SourceGenerations: generations, + }, nil +} + +func (r *memoryRepository) validateSendNowClaimAuthority( + ctx context.Context, + claim *SendNowClaim, +) error { + if claim.Identity.SessionIncarnationID == "" || r.authority == nil { + return nil + } + current, err := r.authority(ctx, claim.Identity.TaskID, claim.Identity.SessionID) + if err != nil || current != claim.Identity { + return ErrSessionIdentityMismatch + } + return nil +} + +func (r *memoryRepository) validateSendNowClaimLocked(claim *SendNowClaim) (string, error) { + sessionID := claim.Sources[0].SessionID + if claim.Identity.SessionIncarnationID != "" && r.identities[sessionID] != claim.Identity { + return "", ErrSessionIdentityMismatch + } + if claim.OperationGeneration > 0 && r.sendNowGeneration[sessionID] != claim.OperationGeneration { + return "", ErrSendNowClaimChanged + } + return sessionID, nil } // RestoreSendNowClaim puts every claimed source back at its original position. -func (r *memoryRepository) RestoreSendNowClaim(_ context.Context, claim *SendNowClaim) error { +func (r *memoryRepository) RestoreSendNowClaim(ctx context.Context, claim *SendNowClaim) error { if claim == nil || len(claim.Sources) == 0 { return ErrSendNowEmpty } + if err := r.validateSendNowClaimAuthority(ctx, claim); err != nil { + return err + } r.mu.Lock() defer r.mu.Unlock() - - sessionID := claim.Sources[0].SessionID + sessionID, err := r.validateSendNowClaimLocked(claim) + if err != nil { + return err + } list := r.entries[sessionID] existing := make(map[string]*QueuedMessage, len(list)) for _, entry := range list { @@ -652,13 +1326,19 @@ func validateMemorySendNowRestore( } // AcknowledgeSendNowClaim removes every durable source after the replacement prompt is accepted. -func (r *memoryRepository) AcknowledgeSendNowClaim(_ context.Context, claim *SendNowClaim) error { +func (r *memoryRepository) AcknowledgeSendNowClaim(ctx context.Context, claim *SendNowClaim) error { if claim == nil || len(claim.Sources) == 0 { return ErrSendNowEmpty } + if err := r.validateSendNowClaimAuthority(ctx, claim); err != nil { + return err + } r.mu.Lock() defer r.mu.Unlock() - sessionID := claim.Sources[0].SessionID + sessionID, err := r.validateSendNowClaimLocked(claim) + if err != nil { + return err + } requested := make(map[string]struct{}) for _, source := range claim.Sources { if source.SessionID != sessionID { @@ -718,6 +1398,14 @@ func cloneQueuedMessage(entry *QueuedMessage) *QueuedMessage { return &clone } +func cloneQueuedMessages(entries []*QueuedMessage) []QueuedMessage { + clones := make([]QueuedMessage, 0, len(entries)) + for _, entry := range entries { + clones = append(clones, *cloneQueuedMessage(entry)) + } + return clones +} + // sameQueuedMessageContent reports whether two entries carry identical content, attachments, and metadata. func sameQueuedMessageContent(left, right *QueuedMessage) bool { if left == nil || right == nil { @@ -741,6 +1429,28 @@ func (r *memoryRepository) UpdateContent(ctx context.Context, sessionID, entryID func (r *memoryRepository) UpdateContentAndMetadata(_ context.Context, sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { r.mu.Lock() defer r.mu.Unlock() + return r.updateContentAndMetadataLocked(sessionID, entryID, content, attachments, metadataUpdates, queuedBy) +} + +func (r *memoryRepository) UpdateContentAndMetadataForSession(_ context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.updateContentAndMetadataLocked(identity.SessionID, entryID, content, attachments, metadataUpdates, queuedBy) +} + +func (r *memoryRepository) UpdateContentAndMetadataForSessionWithClaim(_ context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string, _ QueueAttachmentClaim) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.updateContentAndMetadataLocked(identity.SessionID, entryID, content, attachments, metadataUpdates, queuedBy) +} + +func (r *memoryRepository) updateContentAndMetadataLocked(sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { if queuedBy == "" || IsReservedQueuedBy(queuedBy) { return ErrEntryNotFound } @@ -752,10 +1462,7 @@ func (r *memoryRepository) UpdateContentAndMetadata(_ context.Context, sessionID if m.ID != entryID { continue } - if queuedBy != "" && m.QueuedBy != queuedBy { - return ErrEntryNotFound - } - if IsReservedQueuedBy(m.QueuedBy) { + if m.QueuedBy != queuedBy || m.IsReservedInFlight() { return ErrEntryNotFound } m.Content = content @@ -774,6 +1481,19 @@ func (r *memoryRepository) UpdateContentAndMetadata(_ context.Context, sessionID func (r *memoryRepository) MergeIntoAbove(_ context.Context, sessionID, sourceID, queuedBy string) (*QueuedMessage, error) { r.mu.Lock() defer r.mu.Unlock() + return r.mergeIntoAboveLocked(sessionID, sourceID, queuedBy) +} + +func (r *memoryRepository) MergeIntoAboveForSession(_ context.Context, identity QueueSessionIdentity, sourceID, queuedBy string) (*QueuedMessage, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, err + } + return r.mergeIntoAboveLocked(identity.SessionID, sourceID, queuedBy) +} + +func (r *memoryRepository) mergeIntoAboveLocked(sessionID, sourceID, queuedBy string) (*QueuedMessage, error) { list, ok := r.entries[sessionID] if !ok { return nil, ErrEntryNotFound @@ -822,10 +1542,8 @@ func (r *memoryRepository) MergeIntoAbove(_ context.Context, sessionID, sourceID target.QueuedAt = latestQueuedAt(target.QueuedAt, source.QueuedAt) r.entries[sessionID] = append(list[:sourceIndex], list[sourceIndex+1:]...) if len(r.entries[sessionID]) == 0 { - delete(r.entries, sessionID) delete(r.nextPosition, sessionID) } - merged := *target return &merged, nil } @@ -835,7 +1553,36 @@ func (r *memoryRepository) MergeIntoAbove(_ context.Context, sessionID, sourceID func (r *memoryRepository) AutoMergeIntoAbove(_ context.Context, sessionID, sourceID string) (*QueuedMessage, bool, error) { r.mu.Lock() defer r.mu.Unlock() + return r.autoMergeIntoAboveLocked(sessionID, sourceID) +} +func (r *memoryRepository) AutoMergeIntoAboveForSession(_ context.Context, identity QueueSessionIdentity, sourceID string) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.autoMergeIntoAboveLocked(identity.SessionID, sourceID) +} + +func (r *memoryRepository) AutoMergeIntoAboveForSessionWithPolicy( + _ context.Context, + identity QueueSessionIdentity, + sourceID string, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + if err := r.validateAutoMergePolicyLocked(identity, policy); err != nil { + return nil, false, err + } + return r.autoMergeIntoAboveLocked(identity.SessionID, sourceID) +} + +func (r *memoryRepository) autoMergeIntoAboveLocked(sessionID, sourceID string) (*QueuedMessage, bool, error) { list := r.entries[sessionID] sourceIndex := -1 for index, message := range list { @@ -875,7 +1622,55 @@ func (r *memoryRepository) AutoMergeIntoAbove(_ context.Context, sessionID, sour func (r *memoryRepository) AutoMergeCandidateIntoAbove(_ context.Context, candidate *QueuedMessage) (*QueuedMessage, bool, error) { r.mu.Lock() defer r.mu.Unlock() + return r.autoMergeCandidateIntoAboveLocked(candidate) +} +func (r *memoryRepository) AutoMergeCandidateIntoAboveForSession(_ context.Context, identity QueueSessionIdentity, candidate *QueuedMessage) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.autoMergeCandidateIntoAboveLocked(candidate) +} + +func (r *memoryRepository) AutoMergeCandidateIntoAboveForSessionWithClaim(_ context.Context, identity QueueSessionIdentity, candidate *QueuedMessage, _ QueueAttachmentClaim) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + return r.autoMergeCandidateIntoAboveLocked(candidate) +} + +func (r *memoryRepository) AutoMergeCandidateIntoAboveForSessionWithPolicy( + _ context.Context, + identity QueueSessionIdentity, + candidate *QueuedMessage, + _ *QueueAttachmentClaim, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + r.mu.Lock() + defer r.mu.Unlock() + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(identity); err != nil { + return nil, false, err + } + if err := r.validateAutoMergePolicyLocked(identity, policy); err != nil { + return nil, false, err + } + return r.autoMergeCandidateIntoAboveLocked(candidate) +} + +func (r *memoryRepository) autoMergeCandidateIntoAboveLocked(candidate *QueuedMessage) (*QueuedMessage, bool, error) { list := r.entries[candidate.SessionID] var target *QueuedMessage for _, message := range list { @@ -907,6 +1702,19 @@ func (r *memoryRepository) AutoMergeCandidateIntoAbove(_ context.Context, candid func (r *memoryRepository) ReorderEntries(_ context.Context, sessionID string, orderedIDs []string) error { r.mu.Lock() defer r.mu.Unlock() + return r.reorderEntriesLocked(sessionID, orderedIDs) +} + +func (r *memoryRepository) ReorderEntriesForSession(_ context.Context, identity QueueSessionIdentity, orderedIDs []string) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.reorderEntriesLocked(identity.SessionID, orderedIDs) +} + +func (r *memoryRepository) reorderEntriesLocked(sessionID string, orderedIDs []string) error { stored := r.entries[sessionID] // The stored slice is not guaranteed position-sorted after ReplaceSession; @@ -950,6 +1758,46 @@ func (r *memoryRepository) ReorderEntries(_ context.Context, sessionID string, o func (r *memoryRepository) DeleteByID(_ context.Context, sessionID, entryID string) error { r.mu.Lock() defer r.mu.Unlock() + return r.deleteByIDLocked(sessionID, entryID) +} + +func (r *memoryRepository) DeleteByIDForSession(_ context.Context, identity QueueSessionIdentity, entryID string) (*QueueRemovalResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, err + } + return r.deleteByIDForSessionLocked(identity.SessionID, entryID) +} + +func (r *memoryRepository) deleteByIDForSessionLocked(sessionID, entryID string) (*QueueRemovalResult, error) { + list, ok := r.entries[sessionID] + if !ok { + return nil, ErrEntryNotFound + } + for i, message := range list { + if message.ID != entryID { + continue + } + if message.IsReservedInFlight() { + return nil, ErrEntryNotFound + } + removed := cloneQueuedMessage(message) + r.entries[sessionID] = append(list[:i], list[i+1:]...) + retained := cloneQueuedMessages(r.entries[sessionID]) + if len(r.entries[sessionID]) == 0 { + delete(r.entries, sessionID) + delete(r.nextPosition, sessionID) + } + return &QueueRemovalResult{ + Removed: []QueuedMessage{*removed}, + Retained: retained, + }, nil + } + return nil, ErrEntryNotFound +} + +func (r *memoryRepository) deleteByIDLocked(sessionID, entryID string) error { list, ok := r.entries[sessionID] if !ok { return ErrEntryNotFound @@ -975,6 +1823,41 @@ func (r *memoryRepository) DeleteByID(_ context.Context, sessionID, entryID stri func (r *memoryRepository) DeleteAllBySession(_ context.Context, sessionID string) (int, error) { r.mu.Lock() defer r.mu.Unlock() + return r.deleteAllBySessionLocked(sessionID) +} + +func (r *memoryRepository) DeleteAllBySessionForIdentity(_ context.Context, identity QueueSessionIdentity) (*QueueRemovalResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.bindIdentityLocked(identity); err != nil { + return nil, err + } + return r.deleteAllBySessionForIdentityLocked(identity.SessionID), nil +} + +func (r *memoryRepository) deleteAllBySessionForIdentityLocked(sessionID string) *QueueRemovalResult { + list := r.entries[sessionID] + kept := list[:0] + result := &QueueRemovalResult{} + for _, message := range list { + cloned := cloneQueuedMessage(message) + if message.IsReservedInFlight() { + kept = append(kept, message) + result.Retained = append(result.Retained, *cloned) + continue + } + result.Removed = append(result.Removed, *cloned) + } + if len(kept) == 0 { + delete(r.entries, sessionID) + delete(r.nextPosition, sessionID) + } else { + r.entries[sessionID] = kept + } + return result +} + +func (r *memoryRepository) deleteAllBySessionLocked(sessionID string) (int, error) { list := r.entries[sessionID] kept := list[:0] removed := 0 @@ -994,14 +1877,59 @@ func (r *memoryRepository) DeleteAllBySession(_ context.Context, sessionID strin return removed, nil } +func (r *memoryRepository) PurgeDeletedSession( + _ context.Context, + identity QueueSessionIdentity, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if current, exists := r.identities[identity.SessionID]; exists && current != identity { + return nil + } + r.clearSessionStateLocked(identity.SessionID) + return nil +} + // TransferSession moves all entries (and any pending move) from one session to another. func (r *memoryRepository) TransferSession(_ context.Context, oldSessionID, newSessionID string) error { r.mu.Lock() defer r.mu.Unlock() + return r.transferSessionLocked(nil, nil, oldSessionID, newSessionID) +} + +func (r *memoryRepository) TransferSessionIdentities(_ context.Context, source, destination QueueSessionIdentity) error { + r.mu.Lock() + defer r.mu.Unlock() + if source.TaskID != destination.TaskID { + return ErrSessionIdentityMismatch + } + if err := r.bindIdentityLocked(source); err != nil { + return err + } + if err := r.bindIdentityLocked(destination); err != nil { + return err + } + return r.transferSessionLocked(&source, &destination, source.SessionID, destination.SessionID) +} + +func (r *memoryRepository) transferSessionLocked( + source, destination *QueueSessionIdentity, + oldSessionID, newSessionID string, +) error { if oldSessionID == newSessionID { return nil } - destinationAutoRun := r.autoRunLocked(oldSessionID) && r.autoRunLocked(newSessionID) + for _, entry := range r.entries[oldSessionID] { + if entry.IsReservedInFlight() { + return ErrQueueChanged + } + } + sourceAutoRun := r.autoRunLocked(oldSessionID) + destinationAutoRun := r.autoRunLocked(newSessionID) + if source != nil && destination != nil { + sourceAutoRun = r.autoRunForIdentityLocked(*source) + destinationAutoRun = r.autoRunForIdentityLocked(*destination) + } if list, ok := r.entries[oldSessionID]; ok { // Mirror the SQLite repo: shift transferred positions past the // destination's max so source entries always sort *after* anything @@ -1031,11 +1959,17 @@ func (r *memoryRepository) TransferSession(_ context.Context, oldSessionID, newS delete(r.nextPosition, oldSessionID) } if move, ok := r.pendingMoves[oldSessionID]; ok { + if destination, exists := r.identities[newSessionID]; exists { + move.SessionIncarnationID = destination.SessionIncarnationID + move.TaskID = destination.TaskID + } r.pendingMoves[newSessionID] = move delete(r.pendingMoves, oldSessionID) } - r.autoRun[newSessionID] = destinationAutoRun + r.setAutoRunLocked(destination, newSessionID, sourceAutoRun && destinationAutoRun) delete(r.autoRun, oldSessionID) + delete(r.autoRunIncarnation, oldSessionID) + delete(r.identities, oldSessionID) return nil } @@ -1043,6 +1977,22 @@ func (r *memoryRepository) TransferSession(_ context.Context, oldSessionID, newS func (r *memoryRepository) ReplaceSession(_ context.Context, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { r.mu.Lock() defer r.mu.Unlock() + return r.replaceSessionLocked(sessionID, entries, pendingMove) +} + +func (r *memoryRepository) ReplaceSessionForIdentity(_ context.Context, identity QueueSessionIdentity, entries []QueuedMessage, pendingMove *PendingMove) error { + r.mu.Lock() + defer r.mu.Unlock() + if err := validateReplacementSnapshot(&identity, entries, pendingMove); err != nil { + return err + } + if err := r.bindIdentityLocked(identity); err != nil { + return err + } + return r.replaceSessionLocked(identity.SessionID, entries, pendingMove) +} + +func (r *memoryRepository) replaceSessionLocked(sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { if len(entries) == 0 { delete(r.entries, sessionID) delete(r.nextPosition, sessionID) @@ -1073,6 +2023,32 @@ func (r *memoryRepository) ReplaceSession(_ context.Context, sessionID string, e func (r *memoryRepository) SetPendingMove(_ context.Context, sessionID string, move *PendingMove) error { r.mu.Lock() defer r.mu.Unlock() + if move == nil || move.TaskID == "" || sessionID == "" { + return ErrSessionIdentityMismatch + } + if move.SessionIncarnationID == "" { + if identity, ok := r.identities[sessionID]; ok { + if identity.TaskID != move.TaskID { + return ErrSessionIdentityMismatch + } + move.SessionIncarnationID = identity.SessionIncarnationID + } else { + move.SessionIncarnationID = "memory:" + sessionID + r.identities[sessionID] = QueueSessionIdentity{ + TaskID: move.TaskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + } + } + } + if _, ok := r.identities[sessionID]; !ok { + r.identities[sessionID] = QueueSessionIdentity{ + TaskID: move.TaskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + } + } + if err := r.bindIdentityLocked(QueueSessionIdentity{ + TaskID: move.TaskID, SessionID: sessionID, SessionIncarnationID: move.SessionIncarnationID, + }); err != nil { + return err + } if move.QueuedAt.IsZero() { move.QueuedAt = time.Now().UTC() } diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_send_now_test.go b/apps/backend/internal/orchestrator/messagequeue/repository_send_now_test.go index 87a5ac3c6d..7f1d8fa7c9 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_send_now_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_send_now_test.go @@ -83,6 +83,230 @@ func TestSendNowClaimIsExactAtomicAndRestorable(t *testing.T) { } } +func TestIdentityBoundSendNowClaimRejectsRecreatedSession(t *testing.T) { + tests := []struct { + name string + new func(*testing.T) Repository + }{ + {name: "memory", new: func(*testing.T) Repository { return NewMemoryRepository() }}, + {name: "sqlite", new: newTestSQLiteRepo}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := tt.new(t) + ctx := context.Background() + first := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + replacement := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-2", + } + seedQueueSessionIdentity(t, repo, first) + source := &QueuedMessage{ + SessionID: first.SessionID, TaskID: first.TaskID, Content: "old prompt", QueuedBy: QueuedByUser, + } + if err := repo.InsertForSession(ctx, first, source, 0); err != nil { + t.Fatalf("insert old source: %v", err) + } + claim, err := repo.ClaimSendNowForSession(ctx, first, []QueuedMessage{*source}) + if err != nil { + t.Fatalf("claim old source: %v", err) + } + seedQueueSessionIdentity(t, repo, replacement) + + if err := repo.RestoreSendNowClaim(ctx, claim); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("restore old claim error = %v, want ErrSessionIdentityMismatch", err) + } + if err := repo.AcknowledgeSendNowClaim(ctx, claim); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("acknowledge old claim error = %v, want ErrSessionIdentityMismatch", err) + } + entries, err := repo.ListBySession(ctx, replacement.SessionID) + if err != nil { + t.Fatalf("list replacement queue: %v", err) + } + if len(entries) != 0 { + t.Fatalf("old claim mutated replacement queue: %+v", entries) + } + }) + } +} +func TestIdentityBoundSendNowClaimPersistsDurableReservationIncarnation(t *testing.T) { + tests := []struct { + name string + new func(*testing.T) Repository + }{ + {name: "memory", new: func(*testing.T) Repository { return NewMemoryRepository() }}, + {name: "sqlite", new: newTestSQLiteRepo}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := tt.new(t) + ctx := context.Background() + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repo, identity) + source := &QueuedMessage{ + SessionID: identity.SessionID, + TaskID: identity.TaskID, + Content: "durable", + QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{MetadataLifecycleDurable: true}, + } + if err := repo.InsertForSession(ctx, identity, source, 0); err != nil { + t.Fatalf("insert durable source: %v", err) + } + if _, err := repo.ClaimSendNowForSession(ctx, identity, []QueuedMessage{*source}); err != nil { + t.Fatalf("claim durable source: %v", err) + } + entries, err := repo.ListBySession(ctx, identity.SessionID) + if err != nil { + t.Fatalf("list reserved source: %v", err) + } + if len(entries) != 1 { + t.Fatalf("reserved entries = %d, want 1", len(entries)) + } + if got := lifecycleReservationIncarnation(entries[0].Metadata); got != identity.SessionIncarnationID { + t.Fatalf("reservation incarnation = %q, want %q", got, identity.SessionIncarnationID) + } + }) + } +} + +func TestIdentityBoundPostClaimMutationsRejectRecreatedSession(t *testing.T) { + tests := []struct { + name string + new func(*testing.T) Repository + }{ + {name: "memory", new: func(*testing.T) Repository { return NewMemoryRepository() }}, + {name: "sqlite", new: newTestSQLiteRepo}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := tt.new(t) + ctx := context.Background() + first := QueueSessionIdentity{SessionID: "session-1", TaskID: "task-1", SessionIncarnationID: "incarnation-1"} + replacement := first + replacement.SessionIncarnationID = "incarnation-2" + seedQueueSessionIdentity(t, repo, first) + + reserved := &QueuedMessage{ + SessionID: first.SessionID, + TaskID: first.TaskID, + Content: "old lifecycle prompt", + QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{ + MetadataLifecycleDurable: true, + MetadataLifecycleGeneration: int64(0), + MetadataCoalesceKey: "old-lifecycle", + }, + } + if err := repo.InsertForSession(ctx, first, reserved, 0); err != nil { + t.Fatalf("insert reserved source: %v", err) + } + claimed, enabled, err := repo.ReserveHeadIfAutoRunForSession(ctx, first) + if err != nil { + t.Fatalf("reserve source: %v", err) + } + if !enabled { + t.Fatal("expected Auto-run enabled") + } + if claimed == nil { + t.Fatal("expected reserved source") + } + seedQueueSessionIdentity(t, repo, replacement) + + if err := repo.AcknowledgeByIDForSession(ctx, first, claimed.ID); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("acknowledge error = %v, want ErrSessionIdentityMismatch", err) + } + ordinary := &QueuedMessage{ + ID: "restored-old-entry", + SessionID: first.SessionID, + TaskID: first.TaskID, + Content: "old ordinary prompt", + QueuedBy: QueuedByUser, + } + if err := repo.RestoreForSession(ctx, first, ordinary, 0); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("restore error = %v, want ErrSessionIdentityMismatch", err) + } + if err := repo.RequeuePreservingFIFOForSession(ctx, first, ordinary); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("FIFO requeue error = %v, want ErrSessionIdentityMismatch", err) + } + retry := &QueuedMessage{ + SessionID: first.SessionID, + TaskID: first.TaskID, + Content: "old lifecycle retry", + QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{ + MetadataLifecycleGeneration: int64(0), + }, + } + if _, _, err := repo.InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx, first, retry, "old-lifecycle", 0, true, + ); !errors.Is(err, ErrSessionIdentityMismatch) { + t.Fatalf("lifecycle requeue error = %v, want ErrSessionIdentityMismatch", err) + } + + entries, err := repo.ListBySession(ctx, replacement.SessionID) + if err != nil { + t.Fatalf("list replacement queue: %v", err) + } + if len(entries) != 1 || entries[0].ID != claimed.ID || !entries[0].IsReservedInFlight() { + t.Fatalf("post-claim mutation changed replacement queue: %+v", entries) + } + if got := lifecycleReservationIncarnation(entries[0].Metadata); got != first.SessionIncarnationID { + t.Fatalf("reservation incarnation = %q, want %q", got, first.SessionIncarnationID) + } + }) + } +} + +func TestSendNowClaimGenerationRejectsSupersededRestore(t *testing.T) { + tests := []struct { + name string + new func(*testing.T) Repository + }{ + {name: "memory", new: func(*testing.T) Repository { return NewMemoryRepository() }}, + {name: "sqlite", new: newTestSQLiteRepo}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := tt.new(t) + ctx := context.Background() + identity := QueueSessionIdentity{ + TaskID: "task-1", SessionID: "session-1", SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repo, identity) + source := &QueuedMessage{ + SessionID: identity.SessionID, TaskID: identity.TaskID, Content: "prompt", QueuedBy: QueuedByUser, + } + if err := repo.InsertForSession(ctx, identity, source, 0); err != nil { + t.Fatalf("insert source: %v", err) + } + first, err := repo.ClaimSendNowForSession(ctx, identity, []QueuedMessage{*source}) + if err != nil { + t.Fatalf("first claim: %v", err) + } + if err := repo.RestoreSendNowClaim(ctx, first); err != nil { + t.Fatalf("restore first claim: %v", err) + } + second, err := repo.ClaimSendNowForSession(ctx, identity, []QueuedMessage{*source}) + if err != nil { + t.Fatalf("second claim: %v", err) + } + + if err := repo.RestoreSendNowClaim(ctx, first); !errors.Is(err, ErrSendNowClaimChanged) { + t.Fatalf("superseded restore error = %v, want ErrSendNowClaimChanged", err) + } + if err := repo.RestoreSendNowClaim(ctx, second); err != nil { + t.Fatalf("restore current claim: %v", err) + } + }) + } +} + func TestSendNowClaimRejectsMissingOrReservedSelectionWithoutMutation(t *testing.T) { tests := []struct { name string diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go index edf65902e0..f298c1ca5f 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go @@ -13,6 +13,7 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" internaldb "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" ) // sqliteRepository persists queued messages and pending moves. @@ -23,12 +24,11 @@ type sqliteRepository struct { mu sync.Mutex sessionLocks map[string]*sync.Mutex - // tasksTablePresent is whether the owning tasks table exists, resolved at - // construction. The queue repository's isolated tests create only queue - // tables; production databases always have the task schema. It is checked - // OUTSIDE any transaction because a failed statement on PostgreSQL aborts - // the whole transaction — the guard must never issue its UPDATE against a - // missing table inside a tx. + // tasksTablePresent records whether both owning task and session tables + // exist. Production always supplies them; queue-only repository tests use + // the legacy textual methods, while every identity-bound method fails + // closed when session authority is unavailable. Detection happens outside + // transactions because a missing-table statement aborts PostgreSQL txs. tasksTablePresent bool } @@ -42,17 +42,55 @@ func NewSQLiteRepository(writer, reader *sqlx.DB) (Repository, error) { var present bool var err error if writer.DriverName() == "pgx" { - err = writer.Get(&present, `SELECT to_regclass('tasks') IS NOT NULL`) + err = writer.Get(&present, `SELECT to_regclass('tasks') IS NOT NULL AND to_regclass('task_sessions') IS NOT NULL`) } else { - err = writer.Get(&present, `SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'tasks')`) + err = writer.Get(&present, `SELECT + EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'tasks') + AND EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_sessions')`) } if err != nil { return nil, fmt.Errorf("messagequeue: resolve tasks table presence: %w", err) } r.tasksTablePresent = present + if present { + if err := r.migratePendingMoveIdentities(); err != nil { + return nil, fmt.Errorf("messagequeue: migrate pending move identities: %w", err) + } + } return r, nil } +func (r *sqliteRepository) sharedTablePresent(name string) (bool, error) { + present, err := internaldb.TableExists(r.db, name) + if err != nil { + return false, fmt.Errorf("resolve shared table %s presence: %w", name, err) + } + return present, nil +} + +func (r *sqliteRepository) migratePendingMoveIdentities() error { + if _, err := r.db.Exec(r.db.Rebind(` + UPDATE pending_moves + SET session_incarnation_id = ( + SELECT queue_incarnation_id FROM task_sessions + WHERE task_sessions.id = pending_moves.session_id + AND task_sessions.task_id = pending_moves.task_id + ) + WHERE session_incarnation_id = '' + AND EXISTS ( + SELECT 1 FROM task_sessions + WHERE task_sessions.id = pending_moves.session_id + AND task_sessions.task_id = pending_moves.task_id + ) + `)); err != nil { + return fmt.Errorf("backfill pending move session incarnations: %w", err) + } + if _, err := r.db.Exec(`DELETE FROM pending_moves WHERE session_incarnation_id = ''`); err != nil { + return fmt.Errorf("delete unmatched legacy pending moves: %w", err) + } + return nil +} + // withSessionLock serializes the queue mutations of one session so a concurrent // merge and drain cannot interleave between their reads and writes. It is // per-session (not global), so unrelated sessions proceed in parallel. The @@ -98,6 +136,88 @@ func (r *sqliteRepository) guardActiveTaskTx(ctx context.Context, tx *sqlx.Tx, t return nil } +func (r *sqliteRepository) validateSessionIdentityTx( + ctx context.Context, + tx *sqlx.Tx, + identity QueueSessionIdentity, +) error { + if !r.tasksTablePresent { + return ErrSessionIdentityMismatch + } + var incarnationID string + query := ` + SELECT queue_incarnation_id + FROM task_sessions + WHERE id = ? AND task_id = ?` + if r.db.DriverName() == "pgx" { + query += ` FOR UPDATE` + } + err := tx.GetContext( + ctx, + &incarnationID, + r.db.Rebind(query), + identity.SessionID, + identity.TaskID, + ) + if errors.Is(err, sql.ErrNoRows) { + return ErrSessionIdentityMismatch + } + if err != nil { + return fmt.Errorf("validate queue session identity: %w", err) + } + if incarnationID != identity.SessionIncarnationID { + return ErrSessionIdentityMismatch + } + return nil +} + +func (r *sqliteRepository) guardOptionalActiveTaskTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, +) error { + if identity == nil { + return nil + } + return r.guardActiveTaskTx(ctx, tx, identity.TaskID) +} + +func (r *sqliteRepository) validateOptionalSessionIdentityTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, +) error { + if identity == nil { + return nil + } + return r.validateSessionIdentityTx(ctx, tx, *identity) +} + +func (r *sqliteRepository) ResolveSessionIdentity(ctx context.Context, taskID, sessionID string) (QueueSessionIdentity, error) { + if taskID == "" || sessionID == "" { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + if !r.tasksTablePresent { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + var incarnationID string + err := r.ro.GetContext(ctx, &incarnationID, r.db.Rebind(` + SELECT queue_incarnation_id FROM task_sessions WHERE id = ? AND task_id = ? + `), sessionID, taskID) + if errors.Is(err, sql.ErrNoRows) { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + if err != nil { + return QueueSessionIdentity{}, fmt.Errorf("resolve queue session identity: %w", err) + } + if incarnationID == "" { + return QueueSessionIdentity{}, ErrSessionIdentityMismatch + } + return QueueSessionIdentity{ + TaskID: taskID, SessionID: sessionID, SessionIncarnationID: incarnationID, + }, nil +} + // lockSessionTxIn takes the per-session cross-process lock inside an existing // transaction (see lockSessionTx). It is the shared core used by the // repository methods and by PurgeTaskInTransaction, which runs inside the task @@ -123,6 +243,7 @@ func lockSessionTxIn(ctx context.Context, tx *sqlx.Tx, db *sqlx.DB, sessionID st } return nil } + func (r *sqliteRepository) withSessionLock(sessionID string) func() { r.mu.Lock() lock := r.sessionLocks[sessionID] @@ -162,6 +283,7 @@ func (r *sqliteRepository) initSchema() error { CREATE TABLE IF NOT EXISTS pending_moves ( id TEXT PRIMARY KEY, move_id TEXT NOT NULL DEFAULT '', + session_incarnation_id TEXT NOT NULL DEFAULT '', session_id TEXT NOT NULL UNIQUE, task_id TEXT NOT NULL DEFAULT '', workflow_id TEXT NOT NULL DEFAULT '', @@ -182,8 +304,13 @@ func (r *sqliteRepository) initSchema() error { ); CREATE TABLE IF NOT EXISTS queue_session_state ( - session_id TEXT PRIMARY KEY, - auto_run INTEGER NOT NULL DEFAULT 1 + session_id TEXT PRIMARY KEY, + session_incarnation_id TEXT NOT NULL DEFAULT '', + auto_run INTEGER NOT NULL DEFAULT 1, + auto_merge_override INTEGER, + auto_merge_revision BIGINT NOT NULL DEFAULT 0, + send_now_generation BIGINT NOT NULL DEFAULT 0, + status_generation BIGINT NOT NULL DEFAULT 0 ); `) if err != nil { @@ -201,11 +328,67 @@ func (r *sqliteRepository) initSchema() error { if _, alterErr := r.db.Exec(`ALTER TABLE pending_moves ADD COLUMN move_id TEXT NOT NULL DEFAULT ''`); alterErr != nil && !internaldb.IsDuplicateColumnError(alterErr) { return alterErr } + if _, alterErr := r.db.Exec(`ALTER TABLE pending_moves ADD COLUMN session_incarnation_id TEXT NOT NULL DEFAULT ''`); alterErr != nil && !internaldb.IsDuplicateColumnError(alterErr) { + return alterErr + } + for _, migration := range []struct { + name string + sql string + }{ + {"session incarnation", `ALTER TABLE queue_session_state ADD COLUMN session_incarnation_id TEXT NOT NULL DEFAULT ''`}, + {"Auto-merge override", `ALTER TABLE queue_session_state ADD COLUMN auto_merge_override INTEGER`}, + {"Auto-merge revision", `ALTER TABLE queue_session_state ADD COLUMN auto_merge_revision BIGINT NOT NULL DEFAULT 0`}, + {"Send Now generation", `ALTER TABLE queue_session_state ADD COLUMN send_now_generation BIGINT NOT NULL DEFAULT 0`}, + {"Status generation", `ALTER TABLE queue_session_state ADD COLUMN status_generation BIGINT NOT NULL DEFAULT 0`}, + } { + if _, alterErr := r.db.Exec(migration.sql); alterErr != nil && !internaldb.IsDuplicateColumnError(alterErr) { + return fmt.Errorf("add queue state %s: %w", migration.name, alterErr) + } + } return nil } // Insert appends a new entry at the tail of the session's FIFO queue. func (r *sqliteRepository) Insert(ctx context.Context, msg *QueuedMessage, maxPerSession int) error { + return r.insert(ctx, nil, msg, nil, maxPerSession, nil) +} + +func (r *sqliteRepository) InsertForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, maxPerSession int) error { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + return r.insert(ctx, &identity, msg, nil, maxPerSession, nil) +} + +func (r *sqliteRepository) InsertForSessionWithClaim(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, claim QueueAttachmentClaim, maxPerSession int) error { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + return r.insert(ctx, &identity, msg, &claim, maxPerSession, nil) +} + +func (r *sqliteRepository) InsertForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + claim *QueueAttachmentClaim, + maxPerSession int, + policy AutoMergePolicy, +) error { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + return r.insert(ctx, &identity, msg, claim, maxPerSession, &policy) +} + +func (r *sqliteRepository) insert( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, + claim *QueueAttachmentClaim, + maxPerSession int, + policy *AutoMergePolicy, +) error { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return fmt.Errorf("begin insert tx: %w", err) @@ -217,15 +400,18 @@ func (r *sqliteRepository) Insert(ctx context.Context, msg *QueuedMessage, maxPe if err := r.lockSessionTx(ctx, tx, msg.SessionID); err != nil { return err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } + if err := r.validateAutoMergePolicyTx(ctx, tx, identity, policy); err != nil { + return err + } - if maxPerSession > 0 { - var count int - if err := tx.GetContext(ctx, &count, r.db.Rebind(`SELECT COUNT(*) FROM queued_messages WHERE session_id = ?`), msg.SessionID); err != nil { - return fmt.Errorf("count: %w", err) - } - if count >= maxPerSession { - return ErrQueueFull - } + if err := r.ensureQueueCapacityTx(ctx, tx, msg.SessionID, maxPerSession); err != nil { + return err + } + if err := claimOptionalMessageAttachmentsTx(ctx, tx, identity, claim, msg.TaskID, msg.SessionID); err != nil { + return err } var maxPos sql.NullInt64 @@ -262,6 +448,192 @@ func (r *sqliteRepository) Insert(ctx context.Context, msg *QueuedMessage, maxPe return tx.Commit() } +func (r *sqliteRepository) validateAutoMergePolicyTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + policy *AutoMergePolicy, +) error { + if identity == nil || policy == nil { + return nil + } + var enabled sql.NullInt64 + var revision int64 + err := tx.QueryRowxContext(ctx, tx.Rebind(` + SELECT auto_merge_override, auto_merge_revision + FROM queue_session_state + WHERE session_id = ? AND session_incarnation_id = ? + `), identity.SessionID, identity.SessionIncarnationID).Scan(&enabled, &revision) + if errors.Is(err, sql.ErrNoRows) { + if policy.Source == AutoMergeSourceGlobal { + return nil + } + return ErrAutoMergePolicyChanged + } + if err != nil { + return fmt.Errorf("validate queue Auto-merge policy: %w", err) + } + switch policy.Source { + case AutoMergeSourceGlobal: + if enabled.Valid { + return ErrAutoMergePolicyChanged + } + case AutoMergeSourceSession: + if !enabled.Valid || enabled.Int64 != int64(boolToInt(policy.Enabled)) || revision != policy.Revision { + return ErrAutoMergePolicyChanged + } + default: + return ErrAutoMergePolicyChanged + } + return nil +} + +func claimStagedMessageAttachmentTx( + ctx context.Context, + tx *sqlx.Tx, + claim QueueAttachmentClaim, + taskID, sessionID, attachmentID string, + attachment models.TaskMessageAttachment, + now time.Time, +) (int64, error) { + if !attachment.ExpiresAt.IsZero() && !attachment.ExpiresAt.After(now) { + return 0, models.ErrAttachmentClaimConflict + } + if attachment.SizeBytes < 0 || attachment.SizeBytes > models.MaxMessageAttachmentBytes { + return 0, models.ErrAttachmentTooLarge + } + result, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE task_message_attachments + SET task_id = ?, session_id = ?, state = ?, updated_at = ? + WHERE id = ? AND owner_id = ? AND workspace_id = ? AND state = ? + `), taskID, sessionID, models.AttachmentStateClaimed, now, attachmentID, + claim.OwnerID, claim.WorkspaceID, models.AttachmentStateStaged) + if err != nil { + return 0, fmt.Errorf("claim attachment for queued message: %w", err) + } + affected, err := result.RowsAffected() + if err != nil || affected != 1 { + return 0, models.ErrAttachmentClaimConflict + } + return attachment.SizeBytes, nil +} + +func validateClaimedMessageAttachment( + attachment models.TaskMessageAttachment, + taskID, sessionID string, +) error { + if attachment.TaskID != taskID || + (attachment.SessionID != "" && attachment.SessionID != sessionID) { + return models.ErrAttachmentClaimConflict + } + return nil +} + +func claimMessageAttachmentTx( + ctx context.Context, + tx *sqlx.Tx, + claim QueueAttachmentClaim, + taskID, sessionID, attachmentID string, + now time.Time, +) (int64, error) { + var attachment models.TaskMessageAttachment + if err := tx.GetContext(ctx, &attachment, tx.Rebind(` + SELECT id, owner_id, workspace_id, task_id, session_id, state, size_bytes, expires_at + FROM task_message_attachments WHERE id = ? + `), attachmentID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, models.ErrAttachmentNotFound + } + return 0, fmt.Errorf("load attachment for queue claim: %w", err) + } + if attachment.OwnerID != claim.OwnerID || attachment.WorkspaceID != claim.WorkspaceID { + return 0, models.ErrAttachmentClaimConflict + } + switch attachment.State { + case models.AttachmentStateStaged: + return claimStagedMessageAttachmentTx( + ctx, tx, claim, taskID, sessionID, attachmentID, attachment, now, + ) + case models.AttachmentStateClaimed: + return 0, validateClaimedMessageAttachment(attachment, taskID, sessionID) + default: + return 0, models.ErrAttachmentClaimConflict + } +} + +func claimMessageAttachmentsTx(ctx context.Context, tx *sqlx.Tx, claim QueueAttachmentClaim, taskID, sessionID string) error { + if len(claim.IDs) == 0 { + return nil + } + if claim.OwnerID == "" || claim.WorkspaceID == "" || len(claim.IDs) > models.MaxMessageAttachmentCount { + return models.ErrAttachmentClaimConflict + } + seen := make(map[string]struct{}, len(claim.IDs)) + var totalSize int64 + now := time.Now().UTC() + for _, id := range claim.IDs { + if id == "" { + return models.ErrAttachmentClaimConflict + } + if _, duplicate := seen[id]; duplicate { + return models.ErrAttachmentClaimConflict + } + seen[id] = struct{}{} + size, err := claimMessageAttachmentTx(ctx, tx, claim, taskID, sessionID, id, now) + if err != nil { + return err + } + totalSize += size + if totalSize > models.MaxMessageAttachmentBytes { + return models.ErrAttachmentTotalTooLarge + } + } + return nil +} + +func claimOptionalMessageAttachmentsTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + claim *QueueAttachmentClaim, + taskID, sessionID string, +) error { + if claim == nil || len(claim.IDs) == 0 { + return nil + } + if identity == nil { + return ErrSessionIdentityMismatch + } + if taskID == "" { + taskID = identity.TaskID + } + return claimMessageAttachmentsTx(ctx, tx, *claim, taskID, sessionID) +} + +func (r *sqliteRepository) ensureQueueCapacityTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, + maxPerSession int, +) error { + if maxPerSession <= 0 { + return nil + } + var count int + if err := tx.GetContext( + ctx, + &count, + r.db.Rebind(`SELECT COUNT(*) FROM queued_messages WHERE session_id = ?`), + sessionID, + ); err != nil { + return fmt.Errorf("count: %w", err) + } + if count >= maxPerSession { + return ErrQueueFull + } + return nil +} + // RequeuePreservingFIFO re-enqueues an entry, preserving both FIFO order // across the supersede→requeue cycle AND coalesce-replace semantics on // the original retry. Used by Service.requeueMessage when a queued @@ -280,6 +652,25 @@ func (r *sqliteRepository) Insert(ctx context.Context, msg *QueuedMessage, maxPe // insert. This keeps positions positive for TransferSession and future queue // mutations while preserving the current order. func (r *sqliteRepository) RequeuePreservingFIFO(ctx context.Context, msg *QueuedMessage) error { + return r.requeuePreservingFIFO(ctx, nil, msg) +} + +func (r *sqliteRepository) RequeuePreservingFIFOForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, +) error { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + return r.requeuePreservingFIFO(ctx, &identity, msg) +} + +func (r *sqliteRepository) requeuePreservingFIFO( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, +) error { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return fmt.Errorf("begin requeue-fifo tx: %w", err) @@ -291,6 +682,14 @@ func (r *sqliteRepository) RequeuePreservingFIFO(ctx context.Context, msg *Queue if err := r.lockSessionTx(ctx, tx, msg.SessionID); err != nil { return err } + if identity != nil { + if err := r.validateSessionIdentityTx(ctx, tx, *identity); err != nil { + return err + } + } + if msg.IsReservedLifecycleDelivery() { + return r.releaseLifecycleReservationForRetryTx(ctx, tx, identity, msg) + } // Coalesce-replace: only when caller supplied a coalesce key. // Matches the original RequeueMessage's branching (coalesceKey != @@ -300,7 +699,7 @@ func (r *sqliteRepository) RequeuePreservingFIFO(ctx context.Context, msg *Queue coalesceKey := metadataString(msg.Metadata, MetadataCoalesceKey) var existingID string if coalesceKey != "" { - existing, findErr := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) + existing, _, findErr := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) if findErr != nil { return findErr } @@ -313,6 +712,98 @@ func (r *sqliteRepository) RequeuePreservingFIFO(ctx context.Context, msg *Queue } return r.applyHeadInsertTx(ctx, tx, msg) } +func (r *sqliteRepository) releaseLifecycleReservationForRetryTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + msg *QueuedMessage, +) error { + var metadataJSON string + if err := tx.GetContext(ctx, &metadataJSON, r.db.Rebind(` + SELECT metadata_json FROM queued_messages WHERE id = ? AND session_id = ? + `), msg.ID, msg.SessionID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrEntryNotFound + } + return fmt.Errorf("read lifecycle reservation for retry: %w", err) + } + metadata := make(map[string]interface{}) + if metadataJSON != "" && metadataJSON != "{}" { + if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { + return fmt.Errorf("unmarshal lifecycle reservation for retry: %w", err) + } + } + reserved, _ := metadata[MetadataLifecycleReserved].(bool) + if !reserved { + return ErrEntryNotFound + } + if identity != nil && + lifecycleReservationIncarnation(metadata) != identity.SessionIncarnationID { + return ErrEntryNotFound + } + + successor, err := r.findPendingCoalescedSuccessor( + ctx, + tx, + msg.SessionID, + msg.QueuedBy, + metadata, + ) + if err != nil { + return err + } + if err := r.releaseReservedQueueRowTx( + ctx, + tx, + msg.SessionID, + msg.ID, + metadataJSON, + metadata, + successor != nil, + ); err != nil { + return fmt.Errorf("release lifecycle reservation for retry: %w", err) + } + return tx.Commit() +} + +func (r *sqliteRepository) releaseReservedQueueRowTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID, entryID, metadataJSON string, + metadata map[string]interface{}, + discard bool, +) error { + var ( + result sql.Result + err error + ) + if discard { + result, err = tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM queued_messages + WHERE id = ? AND session_id = ? AND metadata_json = ? + `), entryID, sessionID, metadataJSON) + } else { + releasedJSON, marshalErr := marshalMetadata(clearReservedMetadata(metadata)) + if marshalErr != nil { + return marshalErr + } + result, err = tx.ExecContext(ctx, r.db.Rebind(` + UPDATE queued_messages SET metadata_json = ? + WHERE id = ? AND session_id = ? AND metadata_json = ? + `), releasedJSON, entryID, sessionID, metadataJSON) + } + if err != nil { + return err + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrEntryNotFound + } + return nil +} // applyCoalesceReplaceTx UPDATEs the existing entry in place, // preserving its position and ID. The retry stays at the most head-of- @@ -416,6 +907,27 @@ func (r *sqliteRepository) applyHeadInsertTx(ctx context.Context, tx *sqlx.Tx, m // Restore reinserts a previously dequeued entry at its original FIFO position. func (r *sqliteRepository) Restore(ctx context.Context, msg *QueuedMessage, maxPerSession int) error { + return r.restore(ctx, nil, msg, maxPerSession) +} + +func (r *sqliteRepository) RestoreForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + maxPerSession int, +) error { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + return r.restore(ctx, &identity, msg, maxPerSession) +} + +func (r *sqliteRepository) restore( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, + maxPerSession int, +) error { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return fmt.Errorf("begin restore tx: %w", err) @@ -427,6 +939,11 @@ func (r *sqliteRepository) Restore(ctx context.Context, msg *QueuedMessage, maxP if err := r.lockSessionTx(ctx, tx, msg.SessionID); err != nil { return err } + if identity != nil { + if err := r.validateSessionIdentityTx(ctx, tx, *identity); err != nil { + return err + } + } if maxPerSession > 0 { var count int @@ -463,8 +980,38 @@ func (r *sqliteRepository) Restore(ctx context.Context, msg *QueuedMessage, maxP return tx.Commit() } +func (r *sqliteRepository) appendMatchingTailTx( + ctx context.Context, + tx *sqlx.Tx, + tail *QueuedMessage, + queuedBy, content string, +) (bool, error) { + if tail == nil || tail.QueuedBy != queuedBy { + return false, nil + } + newContent := tail.Content + "\n\n---\n\n" + content + if _, err := tx.ExecContext( + ctx, + r.db.Rebind(`UPDATE queued_messages SET content = ? WHERE id = ?`), + newContent, + tail.ID, + ); err != nil { + return false, fmt.Errorf("append update: %w", err) + } + tail.Content = newContent + return true, nil +} + // AppendOrInsertTail concatenates onto the tail entry when its owner matches, otherwise inserts a new entry. func (r *sqliteRepository) AppendOrInsertTail(ctx context.Context, sessionID, taskID, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { + return r.appendOrInsertTail(ctx, nil, sessionID, taskID, content, model, queuedBy, planMode, attachments, metadata, maxPerSession) +} + +func (r *sqliteRepository) AppendOrInsertTailForSession(ctx context.Context, identity QueueSessionIdentity, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { + return r.appendOrInsertTail(ctx, &identity, identity.SessionID, identity.TaskID, content, model, queuedBy, planMode, attachments, metadata, maxPerSession) +} + +func (r *sqliteRepository) appendOrInsertTail(ctx context.Context, identity *QueueSessionIdentity, sessionID, taskID, content, model, queuedBy string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, bool, error) { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return nil, false, fmt.Errorf("begin append tx: %w", err) @@ -476,17 +1023,19 @@ func (r *sqliteRepository) AppendOrInsertTail(ctx context.Context, sessionID, ta if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, false, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, false, err + } tail, err := r.scanTail(ctx, tx, sessionID) if err != nil { return nil, false, err } - if tail != nil && tail.QueuedBy == queuedBy { - newContent := tail.Content + "\n\n---\n\n" + content - if _, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE queued_messages SET content = ? WHERE id = ?`), newContent, tail.ID); err != nil { - return nil, false, fmt.Errorf("append update: %w", err) - } - tail.Content = newContent + appended, err := r.appendMatchingTailTx(ctx, tx, tail, queuedBy, content) + if err != nil { + return nil, false, err + } + if appended { if err := tx.Commit(); err != nil { return nil, false, err } @@ -494,14 +1043,8 @@ func (r *sqliteRepository) AppendOrInsertTail(ctx context.Context, sessionID, ta } // No matching tail — insert a fresh entry while still inside the same tx. - if maxPerSession > 0 { - var count int - if err := tx.GetContext(ctx, &count, r.db.Rebind(`SELECT COUNT(*) FROM queued_messages WHERE session_id = ?`), sessionID); err != nil { - return nil, false, fmt.Errorf("count: %w", err) - } - if count >= maxPerSession { - return nil, false, ErrQueueFull - } + if err := r.ensureQueueCapacityTx(ctx, tx, sessionID, maxPerSession); err != nil { + return nil, false, err } var maxPos sql.NullInt64 if err := tx.GetContext(ctx, &maxPos, r.db.Rebind(`SELECT MAX(position) FROM queued_messages WHERE session_id = ?`), sessionID); err != nil { @@ -547,6 +1090,31 @@ func (r *sqliteRepository) AppendOrInsertTail(ctx context.Context, sessionID, ta // InsertOrReplaceByCoalesceKey replaces an entry with the same session/queued_by/coalesce key, or inserts when allowInsert is set. func (r *sqliteRepository) InsertOrReplaceByCoalesceKey(ctx context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) { + return r.insertOrReplaceByCoalesceKey(ctx, nil, msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *sqliteRepository) InsertOrReplaceByCoalesceKeyForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + return r.insertOrReplaceByCoalesceKey(ctx, &identity, msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *sqliteRepository) insertOrReplaceByCoalesceKey( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return nil, false, fmt.Errorf("begin coalesce tx: %w", err) @@ -558,8 +1126,11 @@ func (r *sqliteRepository) InsertOrReplaceByCoalesceKey(ctx context.Context, msg if err := r.lockSessionTx(ctx, tx, msg.SessionID); err != nil { return nil, false, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, false, err + } - existing, err := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) + existing, reservedMatch, err := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) if err != nil { return nil, false, err } @@ -576,6 +1147,7 @@ func (r *sqliteRepository) InsertOrReplaceByCoalesceKey(ctx context.Context, msg if !allowInsert { return nil, false, ErrEntryNotFound } + maxPerSession = coalescedInsertCapacity(maxPerSession, reservedMatch) if err := r.insertCoalesced(ctx, tx, msg, maxPerSession); err != nil { return nil, false, err } @@ -585,12 +1157,44 @@ func (r *sqliteRepository) InsertOrReplaceByCoalesceKey(ctx context.Context, msg return msg, false, nil } +func coalescedInsertCapacity(maxPerSession int, reservedMatch bool) int { + if reservedMatch { + return 0 + } + return maxPerSession +} + // InsertOrReplaceLifecycleByCoalesceKey serializes lifecycle acceptance with // task archival. The no-op UPDATE takes SQLite's writer lock only when the // referenced task is still active; archive either commits first (this returns // ErrTaskInactive) or queues first (the archive then follows normal // cancellation semantics). func (r *sqliteRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Context, msg *QueuedMessage, coalesceKey string, maxPerSession int, allowInsert bool) (*QueuedMessage, bool, error) { + return r.insertOrReplaceLifecycleByCoalesceKey(ctx, nil, msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *sqliteRepository) InsertOrReplaceLifecycleByCoalesceKeyForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { + if msg == nil || msg.SessionID != identity.SessionID || msg.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + return r.insertOrReplaceLifecycleByCoalesceKey(ctx, &identity, msg, coalesceKey, maxPerSession, allowInsert) +} + +func (r *sqliteRepository) insertOrReplaceLifecycleByCoalesceKey( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, + coalesceKey string, + maxPerSession int, + allowInsert bool, +) (*QueuedMessage, bool, error) { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return nil, false, fmt.Errorf("begin lifecycle coalesce tx: %w", err) @@ -619,19 +1223,14 @@ func (r *sqliteRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Con if err := r.lockSessionTx(ctx, tx, msg.SessionID); err != nil { return nil, false, err } - expectedGeneration, ok := lifecycleGenerationFromMetadata(msg.Metadata) - if !ok { - return nil, false, ErrLifecycleCancelled - } - generation, err := lifecycleGenerationInTx(ctx, tx, r.db, msg.TaskID) - if err != nil { + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { return nil, false, err } - if expectedGeneration != generation { - return nil, false, ErrLifecycleCancelled + if err := r.validateLifecycleGenerationTx(ctx, tx, msg); err != nil { + return nil, false, err } - existing, err := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) + existing, reservedMatch, err := r.findCoalesced(ctx, tx, msg.SessionID, msg.QueuedBy, coalesceKey) if err != nil { return nil, false, err } @@ -648,6 +1247,7 @@ func (r *sqliteRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Con if !allowInsert { return nil, false, ErrEntryNotFound } + maxPerSession = coalescedInsertCapacity(maxPerSession, reservedMatch) if err := r.insertCoalesced(ctx, tx, msg, maxPerSession); err != nil { return nil, false, err } @@ -657,6 +1257,25 @@ func (r *sqliteRepository) InsertOrReplaceLifecycleByCoalesceKey(ctx context.Con return msg, false, nil } +func (r *sqliteRepository) validateLifecycleGenerationTx( + ctx context.Context, + tx *sqlx.Tx, + msg *QueuedMessage, +) error { + expected, ok := lifecycleGenerationFromMetadata(msg.Metadata) + if !ok { + return ErrLifecycleCancelled + } + generation, err := lifecycleGenerationInTx(ctx, tx, r.db, msg.TaskID) + if err != nil { + return err + } + if expected != generation { + return ErrLifecycleCancelled + } + return nil +} + // LifecycleGeneration returns the current archive/delete generation for a task. func (r *sqliteRepository) LifecycleGeneration(ctx context.Context, taskID string) (int64, error) { var generation int64 @@ -741,6 +1360,42 @@ func purgeQueueRowSessions(ctx context.Context, tx *sqlx.Tx, db *sqlx.DB, taskID // reaches across schemas, and a failed cross-schema query would abort the // caller's transaction on PostgreSQL. The standalone PurgeTask passes nil and // locks only the sessions that currently hold queue rows. +// DeleteSessionInTransaction removes queue state only when the authoritative +// task-session row still matches identity. The caller must lock the task row first. +func DeleteSessionInTransaction(ctx context.Context, tx *sqlx.Tx, db *sqlx.DB, identity QueueSessionIdentity) error { + if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" { + return ErrSessionIdentityMismatch + } + if err := lockSessionTxIn(ctx, tx, db, identity.SessionID); err != nil { + return err + } + var incarnationID string + query := `SELECT queue_incarnation_id FROM task_sessions WHERE id = ? AND task_id = ?` + if db.DriverName() == "pgx" { + query += ` FOR UPDATE` + } + err := tx.GetContext(ctx, &incarnationID, db.Rebind(query), identity.SessionID, identity.TaskID) + if errors.Is(err, sql.ErrNoRows) { + return ErrSessionIdentityMismatch + } + if err != nil { + return fmt.Errorf("validate deleted queue session identity: %w", err) + } + if incarnationID != identity.SessionIncarnationID { + return ErrSessionIdentityMismatch + } + for _, statement := range []string{ + `DELETE FROM queued_messages WHERE session_id = ?`, + `DELETE FROM pending_moves WHERE session_id = ?`, + `DELETE FROM queue_session_state WHERE session_id = ?`, + } { + if _, err := tx.ExecContext(ctx, db.Rebind(statement), identity.SessionID); err != nil { + return fmt.Errorf("delete queue session state: %w", err) + } + } + return nil +} + func PurgeTaskInTransaction(ctx context.Context, tx *sqlx.Tx, db *sqlx.DB, taskID string, taskSessions []string) (int, error) { // Task lifecycle operations can run before the optional queue repository has // been initialized. PostgreSQL aborts a transaction on a missing-table @@ -798,6 +1453,13 @@ func PurgeTaskInTransaction(ctx context.Context, tx *sqlx.Tx, db *sqlx.DB, taskI if err != nil { return 0, fmt.Errorf("purge queued task entries rows affected: %w", err) } + if _, err := tx.ExecContext( + ctx, + db.Rebind(`DELETE FROM pending_moves WHERE task_id = ?`), + taskID, + ); err != nil { + return 0, fmt.Errorf("purge pending task moves: %w", err) + } if _, err := tx.ExecContext(ctx, db.Rebind(` INSERT INTO lifecycle_queue_generations (task_id, generation) VALUES (?, 1) ON CONFLICT(task_id) DO UPDATE SET generation = lifecycle_queue_generations.generation + 1 @@ -927,6 +1589,169 @@ func (r *sqliteRepository) ListBySession(ctx context.Context, sessionID string) return out, rows.Err() } +func (r *sqliteRepository) Snapshot( + ctx context.Context, + identity QueueSessionIdentity, +) (RepositorySnapshot, error) { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return RepositorySnapshot{}, fmt.Errorf("begin queue snapshot tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.guardActiveTaskTx(ctx, tx, identity.TaskID); err != nil { + return RepositorySnapshot{}, err + } + if err := r.lockSessionTx(ctx, tx, identity.SessionID); err != nil { + return RepositorySnapshot{}, err + } + if err := r.validateSessionIdentityTx(ctx, tx, identity); err != nil { + return RepositorySnapshot{}, err + } + + entries, err := r.listBySessionTx(ctx, tx, identity.SessionID) + if err != nil { + return RepositorySnapshot{}, err + } + pendingMove, err := r.getPendingMoveTx(ctx, tx, identity.SessionID) + if err != nil { + return RepositorySnapshot{}, err + } + snapshot := RepositorySnapshot{Entries: entries, PendingMove: pendingMove, AutoRun: true} + snapshot.AutoRun, err = r.getAutoRunTx(ctx, tx, &identity, identity.SessionID) + if err != nil { + return RepositorySnapshot{}, fmt.Errorf("get queue snapshot Auto-run: %w", err) + } + snapshot.StatusGeneration, err = r.nextStatusGenerationTx(ctx, tx, identity) + if err != nil { + return RepositorySnapshot{}, err + } + + var ( + enabled sql.NullInt64 + revision int64 + ) + err = tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT auto_merge_override, auto_merge_revision + FROM queue_session_state + WHERE session_id = ? AND session_incarnation_id = ? + `), identity.SessionID, identity.SessionIncarnationID).Scan(&enabled, &revision) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + snapshot.AutoMergeError = fmt.Errorf("get queue snapshot Auto-merge override: %w", err) + if commitErr := tx.Commit(); commitErr != nil { + return RepositorySnapshot{}, commitErr + } + return snapshot, nil + } + if err == nil && enabled.Valid { + snapshot.AutoMergeOverride = &AutoMergeOverride{ + Enabled: enabled.Int64 != 0, Revision: revision, + } + } + if err := tx.Commit(); err != nil { + return RepositorySnapshot{}, err + } + return snapshot, nil +} + +func (r *sqliteRepository) nextStatusGenerationTx( + ctx context.Context, + tx *sqlx.Tx, + identity QueueSessionIdentity, +) (int64, error) { + var generation int64 + err := tx.QueryRowxContext(ctx, r.db.Rebind(` + INSERT INTO queue_session_state ( + session_id, session_incarnation_id, status_generation + ) VALUES (?, ?, 1) + ON CONFLICT(session_id) DO UPDATE SET + auto_run = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.auto_run + ELSE 1 + END, + auto_merge_override = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.auto_merge_override + ELSE NULL + END, + auto_merge_revision = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.auto_merge_revision + ELSE 0 + END, + send_now_generation = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.send_now_generation + ELSE 0 + END, + status_generation = queue_session_state.status_generation + 1, + session_incarnation_id = excluded.session_incarnation_id + RETURNING status_generation + `), identity.SessionID, identity.SessionIncarnationID).Scan(&generation) + if err != nil { + return 0, fmt.Errorf("advance queue status generation: %w", err) + } + return generation, nil +} + +func (r *sqliteRepository) getPendingMoveTx(ctx context.Context, tx *sqlx.Tx, sessionID string) (*PendingMove, error) { + var ( + moveID, sessionIncarnationID, taskID, workflowID, workflowStepID string + position int + queuedAt time.Time + actor, senderSessionID string + ) + err := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT move_id, session_incarnation_id, task_id, workflow_id, workflow_step_id, + step_position, queued_at, actor, sender_session_id + FROM pending_moves WHERE session_id = ? + `), sessionID).Scan( + &moveID, &sessionIncarnationID, &taskID, &workflowID, &workflowStepID, + &position, &queuedAt, &actor, &senderSessionID, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read pending move snapshot: %w", err) + } + return &PendingMove{ + MoveID: moveID, SessionIncarnationID: sessionIncarnationID, TaskID: taskID, + WorkflowID: workflowID, WorkflowStepID: workflowStepID, Position: position, + QueuedAt: queuedAt, Actor: actor, SenderSessionID: senderSessionID, + }, nil +} + +func (r *sqliteRepository) listBySessionTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, +) ([]QueuedMessage, error) { + rows, err := tx.QueryxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE session_id = ? + ORDER BY position ASC + `), sessionID) + if err != nil { + return nil, fmt.Errorf("list queued snapshot: %w", err) + } + defer func() { _ = rows.Close() }() + var entries []QueuedMessage + for rows.Next() { + message, scanErr := scanQueuedRow(rows) + if scanErr != nil { + return nil, scanErr + } + entries = append(entries, *message) + } + return entries, rows.Err() +} + // ListDurableLifecycleEntries returns durable lifecycle rows in stable FIFO // order across sessions. Startup recovery uses this view to remove queue rows // left by a crash between queue admission and attempt persistence. @@ -1064,7 +1889,7 @@ func (r *sqliteRepository) TakeHead(ctx context.Context, sessionID string) (*Que // ReserveHead returns the lowest-position entry, deleting ordinary rows and reserving durable lifecycle rows. func (r *sqliteRepository) ReserveHead(ctx context.Context, sessionID string) (*QueuedMessage, error) { - msg, _, err := r.reserveHead(ctx, sessionID, false) + msg, _, err := r.reserveHead(ctx, nil, sessionID, false, false) return msg, err } @@ -1085,6 +1910,14 @@ func (r *sqliteRepository) GetAutoRun(ctx context.Context, sessionID string) (bo // SetAutoRun persists policy while holding the queue's per-session lock. func (r *sqliteRepository) SetAutoRun(ctx context.Context, sessionID string, enabled bool) error { + return r.setAutoRun(ctx, nil, sessionID, enabled) +} + +func (r *sqliteRepository) SetAutoRunForSession(ctx context.Context, identity QueueSessionIdentity, enabled bool) error { + return r.setAutoRun(ctx, &identity, identity.SessionID, enabled) +} + +func (r *sqliteRepository) setAutoRun(ctx context.Context, identity *QueueSessionIdentity, sessionID string, enabled bool) error { unlock := r.withSessionLock(sessionID) defer unlock() @@ -1093,17 +1926,127 @@ func (r *sqliteRepository) SetAutoRun(ctx context.Context, sessionID string, ena return fmt.Errorf("begin set auto-run tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return err } - if err := r.setAutoRunTx(ctx, tx, sessionID, enabled); err != nil { + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } + if err := r.setAutoRunTx(ctx, tx, identity, sessionID, enabled); err != nil { return err } return tx.Commit() } +func (r *sqliteRepository) GetAutoMergeOverride( + ctx context.Context, + identity QueueSessionIdentity, +) (*AutoMergeOverride, error) { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin get Auto-merge override: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.validateSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, err + } + + var ( + enabled sql.NullInt64 + revision int64 + ) + err = tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT auto_merge_override, auto_merge_revision + FROM queue_session_state + WHERE session_id = ? AND session_incarnation_id = ? + `), identity.SessionID, identity.SessionIncarnationID).Scan(&enabled, &revision) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get queue Auto-merge override: %w", err) + } + if !enabled.Valid { + return nil, nil + } + return &AutoMergeOverride{Enabled: enabled.Int64 != 0, Revision: revision}, nil +} + +func (r *sqliteRepository) SetAutoMergeOverride( + ctx context.Context, + identity QueueSessionIdentity, + enabled bool, +) (AutoMergeOverride, error) { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return AutoMergeOverride{}, fmt.Errorf("begin set Auto-merge override: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.guardActiveTaskTx(ctx, tx, identity.TaskID); err != nil { + return AutoMergeOverride{}, err + } + if err := r.lockSessionTx(ctx, tx, identity.SessionID); err != nil { + return AutoMergeOverride{}, err + } + if err := r.validateSessionIdentityTx(ctx, tx, identity); err != nil { + return AutoMergeOverride{}, err + } + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + INSERT INTO queue_session_state ( + session_id, session_incarnation_id, auto_merge_override, auto_merge_revision + ) VALUES (?, ?, ?, 1) + ON CONFLICT(session_id) DO UPDATE SET + session_incarnation_id = excluded.session_incarnation_id, + auto_merge_override = excluded.auto_merge_override, + auto_merge_revision = queue_session_state.auto_merge_revision + 1 + WHERE queue_session_state.session_incarnation_id = '' + OR queue_session_state.session_incarnation_id = excluded.session_incarnation_id + `), identity.SessionID, identity.SessionIncarnationID, boolToInt(enabled)); err != nil { + return AutoMergeOverride{}, fmt.Errorf("set queue Auto-merge override: %w", err) + } + var result AutoMergeOverride + var storedEnabled int + if err := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT auto_merge_override, auto_merge_revision + FROM queue_session_state + WHERE session_id = ? AND session_incarnation_id = ? + `), identity.SessionID, identity.SessionIncarnationID).Scan(&storedEnabled, &result.Revision); errors.Is(err, sql.ErrNoRows) { + return AutoMergeOverride{}, ErrSessionIdentityMismatch + } else if err != nil { + return AutoMergeOverride{}, fmt.Errorf("read written queue Auto-merge override: %w", err) + } + result.Enabled = storedEnabled != 0 + if err := tx.Commit(); err != nil { + return AutoMergeOverride{}, err + } + return result, nil +} + // PauseAutoRunIfPending persists OFF only when a visible pending row exists. func (r *sqliteRepository) PauseAutoRunIfPending(ctx context.Context, sessionID string) (bool, error) { + return r.pauseAutoRunIfPending(ctx, nil, sessionID) +} + +func (r *sqliteRepository) PauseAutoRunIfPendingForSession( + ctx context.Context, + identity QueueSessionIdentity, +) (bool, error) { + return r.pauseAutoRunIfPending(ctx, &identity, identity.SessionID) +} + +func (r *sqliteRepository) pauseAutoRunIfPending( + ctx context.Context, + identity *QueueSessionIdentity, + sessionID string, +) (bool, error) { unlock := r.withSessionLock(sessionID) defer unlock() tx, err := r.db.BeginTxx(ctx, nil) @@ -1111,9 +2054,15 @@ func (r *sqliteRepository) PauseAutoRunIfPending(ctx context.Context, sessionID return false, fmt.Errorf("begin conditional auto-run pause tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return false, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return false, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return false, err + } rows, err := tx.QueryxContext(ctx, r.db.Rebind(` SELECT metadata_json FROM queued_messages WHERE session_id = ? `), sessionID) @@ -1147,7 +2096,7 @@ func (r *sqliteRepository) PauseAutoRunIfPending(ctx context.Context, sessionID if !hasPending { return false, nil } - if err := r.setAutoRunTx(ctx, tx, sessionID, false); err != nil { + if err := r.setAutoRunTx(ctx, tx, identity, sessionID, false); err != nil { return false, err } if err := tx.Commit(); err != nil { @@ -1156,36 +2105,123 @@ func (r *sqliteRepository) PauseAutoRunIfPending(ctx context.Context, sessionID return true, nil } -func (r *sqliteRepository) setAutoRunTx(ctx context.Context, tx *sqlx.Tx, sessionID string, enabled bool) error { +func (r *sqliteRepository) setAutoRunTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + sessionID string, + enabled bool, +) error { + if identity == nil { + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + INSERT INTO queue_session_state (session_id, auto_run) VALUES (?, ?) + ON CONFLICT(session_id) DO UPDATE SET auto_run = excluded.auto_run + `), sessionID, boolToInt(enabled)); err != nil { + return fmt.Errorf("set queue auto-run: %w", err) + } + return nil + } if _, err := tx.ExecContext(ctx, r.db.Rebind(` - INSERT INTO queue_session_state (session_id, auto_run) VALUES (?, ?) - ON CONFLICT(session_id) DO UPDATE SET auto_run = excluded.auto_run - `), sessionID, boolToInt(enabled)); err != nil { + INSERT INTO queue_session_state (session_id, session_incarnation_id, auto_run) + VALUES (?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + auto_run = excluded.auto_run, + auto_merge_override = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.auto_merge_override + ELSE NULL + END, + auto_merge_revision = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.auto_merge_revision + ELSE 0 + END, + send_now_generation = CASE + WHEN queue_session_state.session_incarnation_id = excluded.session_incarnation_id + THEN queue_session_state.send_now_generation + ELSE 0 + END, + session_incarnation_id = excluded.session_incarnation_id + `), sessionID, identity.SessionIncarnationID, boolToInt(enabled)); err != nil { return fmt.Errorf("set queue auto-run: %w", err) } return nil } -func (r *sqliteRepository) getAutoRunTx(ctx context.Context, tx *sqlx.Tx, sessionID string) (bool, error) { - var enabled int - err := tx.GetContext(ctx, &enabled, r.db.Rebind(` - SELECT auto_run FROM queue_session_state WHERE session_id = ? - `), sessionID) +func (r *sqliteRepository) getAutoRunTx( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + sessionID string, +) (bool, error) { + if identity == nil { + var enabled int + err := tx.GetContext(ctx, &enabled, r.db.Rebind(` + SELECT auto_run FROM queue_session_state WHERE session_id = ? + `), sessionID) + if errors.Is(err, sql.ErrNoRows) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("get queue auto-run in transaction: %w", err) + } + return enabled != 0, nil + } + + var ( + enabled int + storedIncarnationID string + ) + err := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT auto_run, session_incarnation_id + FROM queue_session_state + WHERE session_id = ? + `), sessionID).Scan(&enabled, &storedIncarnationID) if errors.Is(err, sql.ErrNoRows) { return true, nil } if err != nil { return false, fmt.Errorf("get queue auto-run in transaction: %w", err) } + if storedIncarnationID == "" { + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE queue_session_state + SET session_incarnation_id = ? + WHERE session_id = ? AND session_incarnation_id = '' + `), identity.SessionIncarnationID, sessionID); err != nil { + return false, fmt.Errorf("bind queue auto-run incarnation: %w", err) + } + return enabled != 0, nil + } + if storedIncarnationID != identity.SessionIncarnationID { + return true, nil + } return enabled != 0, nil } // ReserveHeadIfAutoRun reads policy and reserves the FIFO head under one lock. func (r *sqliteRepository) ReserveHeadIfAutoRun(ctx context.Context, sessionID string) (*QueuedMessage, bool, error) { - return r.reserveHead(ctx, sessionID, true) + return r.reserveHead(ctx, nil, sessionID, true, false) } -func (r *sqliteRepository) reserveHead(ctx context.Context, sessionID string, requireAutoRun bool) (*QueuedMessage, bool, error) { +func (r *sqliteRepository) ReserveHeadIfAutoRunForSession(ctx context.Context, identity QueueSessionIdentity) (*QueuedMessage, bool, error) { + return r.reserveHead(ctx, &identity, identity.SessionID, true, false) +} + +func (r *sqliteRepository) ReserveHeadForDeliveryIfAutoRunForSession( + ctx context.Context, + identity QueueSessionIdentity, +) (*QueuedMessage, bool, error) { + return r.reserveHead(ctx, &identity, identity.SessionID, true, true) +} + +func (r *sqliteRepository) reserveHead( + ctx context.Context, + identity *QueueSessionIdentity, + sessionID string, + requireAutoRun bool, + retainOrdinary bool, +) (*QueuedMessage, bool, error) { unlock := r.withSessionLock(sessionID) defer unlock() @@ -1194,11 +2230,17 @@ func (r *sqliteRepository) reserveHead(ctx context.Context, sessionID string, re return nil, true, fmt.Errorf("begin reserve tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return nil, true, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, true, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, true, err + } if requireAutoRun { - enabled, err := r.getAutoRunTx(ctx, tx, sessionID) + enabled, err := r.getAutoRunTx(ctx, tx, identity, sessionID) if err != nil { return nil, true, err } @@ -1207,40 +2249,115 @@ func (r *sqliteRepository) reserveHead(ctx context.Context, sessionID string, re } } - row := tx.QueryRowxContext(ctx, r.db.Rebind(` - SELECT id, session_id, task_id, position, content, model, plan_mode, - attachments_json, metadata_json, queued_at, queued_by - FROM queued_messages - WHERE session_id = ? - ORDER BY position ASC - LIMIT 1 - `), sessionID) - msg, storedMetadataJSON, err := scanQueuedRowWithMetadataJSON(row) - if errors.Is(err, sql.ErrNoRows) { - return nil, true, nil - } - if err != nil { - return nil, true, fmt.Errorf("reserve head: %w", err) - } - if msg.IsDurableLifecycle() { - reserved, err := r.reserveLifecycleHead(ctx, tx, msg, storedMetadataJSON) - return reserved, true, err - } - reserved, err := r.reserveOrdinaryHead(ctx, tx, msg) - return reserved, true, err + return r.reserveHeadTx(ctx, tx, identity, sessionID, retainOrdinary) } -func (r *sqliteRepository) reserveLifecycleHead( +func (r *sqliteRepository) reserveHeadTx( ctx context.Context, tx *sqlx.Tx, + identity *QueueSessionIdentity, + sessionID string, + retainOrdinary bool, +) (*QueuedMessage, bool, error) { + discardedStaleReservation := false + for { + row := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE session_id = ? + ORDER BY position ASC + LIMIT 1 + `), sessionID) + msg, storedMetadataJSON, err := scanQueuedRowWithMetadataJSON(row) + if errors.Is(err, sql.ErrNoRows) { + if discardedStaleReservation { + if err := tx.Commit(); err != nil { + return nil, true, err + } + } + return nil, true, nil + } + if err != nil { + return nil, true, fmt.Errorf("reserve head: %w", err) + } + discarded, err := r.discardStaleReservationHead( + ctx, + tx, + identity, + msg, + storedMetadataJSON, + ) + if err != nil { + return nil, true, err + } + if discarded { + discardedStaleReservation = true + continue + } + retainedLifecycle := msg.IsDurableLifecycle() + if retainedLifecycle || retainOrdinary { + reserved, err := r.reserveRetainedHead( + ctx, tx, identity, msg, storedMetadataJSON, retainedLifecycle, + ) + return reserved, true, err + } + reserved, err := r.reserveOrdinaryHead(ctx, tx, msg) + return reserved, true, err + } +} +func (r *sqliteRepository) discardStaleReservationHead( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, msg *QueuedMessage, storedMetadataJSON string, +) (bool, error) { + if identity == nil || !msg.IsReservedInFlight() { + return false, nil + } + reservationIncarnation := lifecycleReservationIncarnation(msg.Metadata) + if reservationIncarnation == "" || + reservationIncarnation == identity.SessionIncarnationID { + return false, nil + } + result, err := tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM queued_messages + WHERE id = ? AND session_id = ? AND metadata_json = ? + `), msg.ID, msg.SessionID, storedMetadataJSON) + if err != nil { + return false, fmt.Errorf("discard stale delivery reservation: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return false, err + } + if affected == 0 { + return false, ErrQueueChanged + } + return true, nil +} + +func (r *sqliteRepository) reserveRetainedHead( + ctx context.Context, + tx *sqlx.Tx, + identity *QueueSessionIdentity, + msg *QueuedMessage, + storedMetadataJSON string, + lifecycle bool, ) (*QueuedMessage, error) { // Keep the row for crash recovery but stop reporting it as pending. - // Strip a marker persisted by an interrupted prior process from the + // Strip metadata persisted by an interrupted prior process from the // returned copy so a failed retry becomes visible again. msg.Metadata = clearReservedMetadata(msg.Metadata) - reservedJSON, err := marshalMetadata(markReservedMetadata(msg.Metadata)) + reservedMetadata := markReservedMetadata(msg.Metadata) + if identity != nil { + reservedMetadata = markReservedMetadataForIncarnation( + msg.Metadata, + identity.SessionIncarnationID, + ) + } + reservedJSON, err := marshalMetadata(reservedMetadata) if err != nil { return nil, err } @@ -1249,11 +2366,11 @@ func (r *sqliteRepository) reserveLifecycleHead( WHERE id = ? AND session_id = ? AND metadata_json = ? `), reservedJSON, msg.ID, msg.SessionID, storedMetadataJSON) if err != nil { - return nil, fmt.Errorf("mark lifecycle reservation in flight: %w", err) + return nil, fmt.Errorf("mark delivery reservation in flight: %w", err) } affected, err := res.RowsAffected() if err != nil { - return nil, fmt.Errorf("mark lifecycle reservation rows affected: %w", err) + return nil, fmt.Errorf("mark delivery reservation rows affected: %w", err) } if affected == 0 { return nil, nil @@ -1261,7 +2378,10 @@ func (r *sqliteRepository) reserveLifecycleHead( if err := tx.Commit(); err != nil { return nil, err } - msg.reservedLifecycleDelivery = true + msg.reservedLifecycleDelivery = lifecycle + if identity != nil { + msg.reservationIdentity = *identity + } return msg, nil } @@ -1324,6 +2444,171 @@ func (r *sqliteRepository) AcknowledgeByID(ctx context.Context, sessionID, entry return nil } +func (r *sqliteRepository) AcknowledgeByIDForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin identity-bound acknowledge tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.lockSessionTx(ctx, tx, identity.SessionID); err != nil { + return err + } + if err := r.validateSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } + var metadataJSON string + err = tx.GetContext(ctx, &metadataJSON, r.db.Rebind(` + SELECT metadata_json FROM queued_messages WHERE id = ? AND session_id = ? + `), entryID, identity.SessionID) + if errors.Is(err, sql.ErrNoRows) { + return ErrEntryNotFound + } + if err != nil { + return fmt.Errorf("read identity-bound lifecycle reservation: %w", err) + } + reservationIncarnation, reserved, err := lifecycleReservationFromMetadataJSON(metadataJSON) + if err != nil { + return err + } + if !reserved || reservationIncarnation != identity.SessionIncarnationID { + return ErrEntryNotFound + } + result, err := tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM queued_messages + WHERE id = ? AND session_id = ? AND metadata_json = ? + `), entryID, identity.SessionID, metadataJSON) + if err != nil { + return fmt.Errorf("acknowledge identity-bound queued message: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrEntryNotFound + } + return tx.Commit() +} + +func (r *sqliteRepository) ReleaseDeliveryReservationForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin release delivery reservation tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.lockSessionTx(ctx, tx, identity.SessionID); err != nil { + return err + } + if err := r.validateSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } + var metadataJSON, queuedBy string + err = tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT metadata_json, queued_by FROM queued_messages WHERE id = ? AND session_id = ? + `), entryID, identity.SessionID).Scan(&metadataJSON, &queuedBy) + if errors.Is(err, sql.ErrNoRows) { + return ErrEntryNotFound + } + if err != nil { + return fmt.Errorf("read delivery reservation: %w", err) + } + metadata := make(map[string]interface{}) + if metadataJSON != "" && metadataJSON != "{}" { + if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { + return fmt.Errorf("unmarshal delivery reservation metadata: %w", err) + } + } + if reserved, _ := metadata[MetadataLifecycleReserved].(bool); !reserved || + lifecycleReservationIncarnation(metadata) != identity.SessionIncarnationID { + return ErrEntryNotFound + } + successor, err := r.findPendingCoalescedSuccessor( + ctx, + tx, + identity.SessionID, + queuedBy, + metadata, + ) + if err != nil { + return err + } + if err := r.releaseReservedQueueRowTx( + ctx, + tx, + identity.SessionID, + entryID, + metadataJSON, + metadata, + successor != nil, + ); err != nil { + return fmt.Errorf("release delivery reservation: %w", err) + } + return tx.Commit() +} +func (r *sqliteRepository) DiscardLifecycleReservation( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + unlock := r.withSessionLock(identity.SessionID) + defer unlock() + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin discard lifecycle reservation tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.lockSessionTx(ctx, tx, identity.SessionID); err != nil { + return err + } + var metadataJSON string + err = tx.GetContext(ctx, &metadataJSON, r.db.Rebind(` + SELECT metadata_json FROM queued_messages WHERE id = ? AND session_id = ? + `), entryID, identity.SessionID) + if errors.Is(err, sql.ErrNoRows) { + return ErrEntryNotFound + } + if err != nil { + return fmt.Errorf("read lifecycle reservation: %w", err) + } + reservationIncarnation, reserved, err := lifecycleReservationFromMetadataJSON(metadataJSON) + if err != nil { + return err + } + if !reserved || reservationIncarnation != identity.SessionIncarnationID { + return ErrEntryNotFound + } + result, err := tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM queued_messages + WHERE id = ? AND session_id = ? AND metadata_json = ? + `), entryID, identity.SessionID, metadataJSON) + if err != nil { + return fmt.Errorf("discard lifecycle reservation: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrEntryNotFound + } + return tx.Commit() +} + // TakeByID atomically returns and deletes the entry identified by entryID, // regardless of its FIFO position. Mirrors TakeHead's race handling: if a // concurrent take already removed the row between the SELECT and DELETE, @@ -1334,6 +2619,22 @@ func (r *sqliteRepository) AcknowledgeByID(ctx context.Context, sessionID, entry // the SELECT and DELETE can't have it removed out from under the new // session — same session-scope invariant DeleteByID/UpdateContent enforce. func (r *sqliteRepository) TakeByID(ctx context.Context, sessionID, entryID string) (*QueuedMessage, error) { + return r.takeByID(ctx, nil, sessionID, entryID) +} + +func (r *sqliteRepository) TakeByIDForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) (*QueuedMessage, error) { + return r.takeByID(ctx, &identity, identity.SessionID, entryID) +} + +func (r *sqliteRepository) takeByID( + ctx context.Context, + identity *QueueSessionIdentity, + sessionID, entryID string, +) (*QueuedMessage, error) { unlock := r.withSessionLock(sessionID) defer unlock() @@ -1345,6 +2646,9 @@ func (r *sqliteRepository) TakeByID(ctx context.Context, sessionID, entryID stri if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, err + } row := tx.QueryRowxContext(ctx, r.db.Rebind(` SELECT id, session_id, task_id, position, content, model, plan_mode, @@ -1376,8 +2680,43 @@ func (r *sqliteRepository) TakeByID(ctx context.Context, sessionID, entryID stri return msg, nil } +func lifecycleGenerationsForSourcesTx( + ctx context.Context, + tx *sqlx.Tx, + db *sqlx.DB, + sources []QueuedMessage, +) (map[string]int64, error) { + generations := make(map[string]int64) + for _, source := range sources { + if source.TaskID == "" { + continue + } + generation, err := lifecycleGenerationInTx(ctx, tx, db, source.TaskID) + if err != nil { + return nil, err + } + generations[source.TaskID] = generation + } + return generations, nil +} + // ClaimSendNow atomically claims the exact ordered source snapshot for a send-now dispatch. func (r *sqliteRepository) ClaimSendNow(ctx context.Context, sessionID string, expected []QueuedMessage) (*SendNowClaim, error) { + return r.claimSendNow(ctx, nil, sessionID, expected) +} + +func (r *sqliteRepository) ClaimSendNowForSession(ctx context.Context, identity QueueSessionIdentity, expected []QueuedMessage) (*SendNowClaim, error) { + return r.claimSendNow(ctx, &identity, identity.SessionID, expected) +} + +func queueSessionIdentityValue(identity *QueueSessionIdentity) QueueSessionIdentity { + if identity == nil { + return QueueSessionIdentity{} + } + return *identity +} + +func (r *sqliteRepository) claimSendNow(ctx context.Context, identity *QueueSessionIdentity, sessionID string, expected []QueuedMessage) (*SendNowClaim, error) { if len(expected) == 0 { return nil, ErrSendNowEmpty } @@ -1394,9 +2733,15 @@ func (r *sqliteRepository) ClaimSendNow(ctx context.Context, sessionID string, e return nil, fmt.Errorf("begin send-now claim tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return nil, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, err + } ordered, storedByID, err := r.listOrderedStoredSessionEntries(ctx, tx, sessionID) if err != nil { @@ -1406,32 +2751,37 @@ func (r *sqliteRepository) ClaimSendNow(ctx context.Context, sessionID string, e if err != nil { return nil, err } + claimIdentity := queueSessionIdentityValue(identity) + bindSendNowLifecycleReservations(sources, claimIdentity) envelope, err := BuildSendNowEnvelope(sources) if err != nil { return nil, err } - generations := make(map[string]int64) - for _, source := range sources { - if source.TaskID == "" { - continue - } - generation, generationErr := lifecycleGenerationInTx(ctx, tx, r.db, source.TaskID) - if generationErr != nil { - return nil, generationErr - } - generations[source.TaskID] = generation - } - - if err := r.applySQLiteSendNowClaim(ctx, tx, sessionID, sources, storedByID); err != nil { + generations, err := lifecycleGenerationsForSourcesTx(ctx, tx, r.db, sources) + if err != nil { return nil, err } - if err := r.setAutoRunTx(ctx, tx, sessionID, true); err != nil { + + if err := r.applySQLiteSendNowClaim(ctx, tx, identity, sessionID, sources, storedByID); err != nil { + return nil, err + } + if err := r.setAutoRunTx(ctx, tx, identity, sessionID, true); err != nil { + return nil, err + } + operationGeneration, err := incrementSendNowGenerationTx(ctx, tx, r.db, sessionID) + if err != nil { return nil, err } if err := tx.Commit(); err != nil { return nil, err } - return &SendNowClaim{Sources: sources, Dispatch: *envelope, SourceGenerations: generations}, nil + return &SendNowClaim{ + Identity: claimIdentity, + OperationGeneration: operationGeneration, + Sources: sources, + Dispatch: *envelope, + SourceGenerations: generations, + }, nil } // RestoreSendNowClaim puts every claimed source back at its original position. @@ -1520,9 +2870,58 @@ func (r *sqliteRepository) beginSendNowClaimTx( unlock() return nil, "", nil, err } + if claim.Identity.SessionIncarnationID != "" { + if err := r.validateSessionIdentityTx(ctx, tx, claim.Identity); err != nil { + _ = tx.Rollback() + unlock() + return nil, "", nil, err + } + } + if claim.OperationGeneration > 0 { + var operationGeneration int64 + err = tx.GetContext(ctx, &operationGeneration, r.db.Rebind(` + SELECT send_now_generation + FROM queue_session_state + WHERE session_id = ? + `), sessionID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + _ = tx.Rollback() + unlock() + return nil, "", nil, fmt.Errorf("validate send-now generation: %w", err) + } + if errors.Is(err, sql.ErrNoRows) || operationGeneration != claim.OperationGeneration { + _ = tx.Rollback() + unlock() + return nil, "", nil, ErrSendNowClaimChanged + } + } return tx, sessionID, unlock, nil } +func incrementSendNowGenerationTx( + ctx context.Context, + tx *sqlx.Tx, + db *sqlx.DB, + sessionID string, +) (int64, error) { + if _, err := tx.ExecContext(ctx, db.Rebind(` + UPDATE queue_session_state + SET send_now_generation = send_now_generation + 1 + WHERE session_id = ? + `), sessionID); err != nil { + return 0, fmt.Errorf("advance send-now generation: %w", err) + } + var generation int64 + if err := tx.GetContext(ctx, &generation, db.Rebind(` + SELECT send_now_generation + FROM queue_session_state + WHERE session_id = ? + `), sessionID); err != nil { + return 0, fmt.Errorf("read send-now generation: %w", err) + } + return generation, nil +} + type storedQueueEntry struct { message *QueuedMessage raw string @@ -1606,6 +3005,7 @@ func selectSQLiteSendNowSources( func (r *sqliteRepository) applySQLiteSendNowClaim( ctx context.Context, tx *sqlx.Tx, + identity *QueueSessionIdentity, sessionID string, sources []QueuedMessage, storedByID map[string]storedQueueEntry, @@ -1613,7 +3013,7 @@ func (r *sqliteRepository) applySQLiteSendNowClaim( for _, source := range sources { storedEntry := storedByID[source.ID] if source.IsDurableLifecycle() { - if err := r.reserveSQLiteSendNowSource(ctx, tx, sessionID, source, storedEntry); err != nil { + if err := r.reserveSQLiteSendNowSource(ctx, tx, identity, sessionID, source, storedEntry); err != nil { return err } continue @@ -1629,11 +3029,16 @@ func (r *sqliteRepository) applySQLiteSendNowClaim( func (r *sqliteRepository) reserveSQLiteSendNowSource( ctx context.Context, tx *sqlx.Tx, + identity *QueueSessionIdentity, sessionID string, source QueuedMessage, stored storedQueueEntry, ) error { - metadataJSON, err := marshalMetadata(markReservedMetadata(source.Metadata)) + metadata := markReservedMetadata(source.Metadata) + if identity != nil { + metadata = markReservedMetadataForIncarnation(source.Metadata, identity.SessionIncarnationID) + } + metadataJSON, err := marshalMetadata(metadata) if err != nil { return err } @@ -1811,6 +3216,40 @@ func (r *sqliteRepository) acknowledgeSQLiteSendNowSource( return nil } +func updatedQueueMetadataJSON(raw string, updates map[string]interface{}) (string, error) { + var metadata map[string]interface{} + if raw != "" && raw != "{}" { + if err := json.Unmarshal([]byte(raw), &metadata); err != nil { + return "", fmt.Errorf("unmarshal queued metadata: %w", err) + } + } + return marshalMetadata(applyMetadataUpdates(metadata, updates)) +} + +func (r *sqliteRepository) updateQueuedEntryTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID, entryID, content, attachmentsJSON, metadataJSON, queuedBy string, +) error { + result, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE queued_messages + SET content = ?, attachments_json = ?, metadata_json = ? + WHERE id = ? AND session_id = ? AND queued_by = ? AND queued_by NOT IN (?, ?, ?) + `), content, attachmentsJSON, metadataJSON, entryID, sessionID, queuedBy, + QueuedByAgent, QueuedByWorkflow, QueuedByServer) + if err != nil { + return fmt.Errorf("update queued: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrEntryNotFound + } + return nil +} + // UpdateContent replaces the content of an entry owned by queuedBy. func (r *sqliteRepository) UpdateContent(ctx context.Context, sessionID, entryID, content string, attachments []MessageAttachment, queuedBy string) error { return r.UpdateContentAndMetadata(ctx, sessionID, entryID, content, attachments, nil, queuedBy) @@ -1818,6 +3257,18 @@ func (r *sqliteRepository) UpdateContent(ctx context.Context, sessionID, entryID // UpdateContentAndMetadata replaces content and applies metadata updates to an entry owned by queuedBy. func (r *sqliteRepository) UpdateContentAndMetadata(ctx context.Context, sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { + return r.updateContentAndMetadata(ctx, nil, sessionID, entryID, content, attachments, metadataUpdates, queuedBy, nil) +} + +func (r *sqliteRepository) UpdateContentAndMetadataForSession(ctx context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { + return r.updateContentAndMetadata(ctx, &identity, identity.SessionID, entryID, content, attachments, metadataUpdates, queuedBy, nil) +} + +func (r *sqliteRepository) UpdateContentAndMetadataForSessionWithClaim(ctx context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string, claim QueueAttachmentClaim) error { + return r.updateContentAndMetadata(ctx, &identity, identity.SessionID, entryID, content, attachments, metadataUpdates, queuedBy, &claim) +} + +func (r *sqliteRepository) updateContentAndMetadata(ctx context.Context, identity *QueueSessionIdentity, sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string, claim *QueueAttachmentClaim) error { if queuedBy == "" || IsReservedQueuedBy(queuedBy) { return ErrEntryNotFound } @@ -1830,6 +3281,9 @@ func (r *sqliteRepository) UpdateContentAndMetadata(ctx context.Context, session return fmt.Errorf("begin update queued tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return err + } // Content edits serialize with merges/folds on the same session: without // the cross-process session lock a merge's scan-to-write window could // silently overwrite a concurrent edit (the merge's affected-row check @@ -1837,6 +3291,9 @@ func (r *sqliteRepository) UpdateContentAndMetadata(ctx context.Context, session if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } var metadataJSON string query := `SELECT metadata_json FROM queued_messages WHERE id = ? AND session_id = ? AND queued_by = ? AND queued_by NOT IN (?, ?, ?)` @@ -1846,29 +3303,25 @@ func (r *sqliteRepository) UpdateContentAndMetadata(ctx context.Context, session } return fmt.Errorf("read queued metadata: %w", err) } - var metadata map[string]interface{} - if metadataJSON != "" && metadataJSON != "{}" { - if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { - return fmt.Errorf("unmarshal queued metadata: %w", err) - } - } - metadataJSON, err = marshalMetadata(applyMetadataUpdates(metadata, metadataUpdates)) + reserved, err := isReservedMetadataJSON(metadataJSON) if err != nil { return err } - res, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE queued_messages SET content = ?, attachments_json = ?, metadata_json = ? - WHERE id = ? AND session_id = ? AND queued_by = ? AND queued_by NOT IN (?, ?, ?)`), - content, attachmentsJSON, metadataJSON, entryID, sessionID, queuedBy, QueuedByAgent, QueuedByWorkflow, QueuedByServer) - if err != nil { - return fmt.Errorf("update queued: %w", err) - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { + if reserved { return ErrEntryNotFound } + if err := claimOptionalMessageAttachmentsTx(ctx, tx, identity, claim, "", sessionID); err != nil { + return err + } + metadataJSON, err = updatedQueueMetadataJSON(metadataJSON, metadataUpdates) + if err != nil { + return err + } + if err := r.updateQueuedEntryTx( + ctx, tx, sessionID, entryID, content, attachmentsJSON, metadataJSON, queuedBy, + ); err != nil { + return err + } return tx.Commit() } @@ -1880,6 +3333,14 @@ func (r *sqliteRepository) UpdateContentAndMetadata(ctx context.Context, session // ErrEntryNotFound rollback rather than a silent content loss. See // Repository.MergeIntoAbove for the merge rules and error mapping. func (r *sqliteRepository) MergeIntoAbove(ctx context.Context, sessionID, sourceID, queuedBy string) (*QueuedMessage, error) { + return r.mergeIntoAbove(ctx, nil, sessionID, sourceID, queuedBy) +} + +func (r *sqliteRepository) MergeIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, sourceID, queuedBy string) (*QueuedMessage, error) { + return r.mergeIntoAbove(ctx, &identity, identity.SessionID, sourceID, queuedBy) +} + +func (r *sqliteRepository) mergeIntoAbove(ctx context.Context, identity *QueueSessionIdentity, sessionID, sourceID, queuedBy string) (*QueuedMessage, error) { unlock := r.withSessionLock(sessionID) defer unlock() @@ -1888,9 +3349,15 @@ func (r *sqliteRepository) MergeIntoAbove(ctx context.Context, sessionID, source return nil, fmt.Errorf("begin merge tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return nil, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, err + } source, err := readMergeSource(ctx, r, tx, sessionID, sourceID) if err != nil { @@ -1929,6 +3396,28 @@ func (r *sqliteRepository) MergeIntoAbove(ctx context.Context, sessionID, source // predecessor in one transaction. Missing or incompatible candidates are // successful skips and leave storage unchanged. func (r *sqliteRepository) AutoMergeIntoAbove(ctx context.Context, sessionID, sourceID string) (*QueuedMessage, bool, error) { + return r.autoMergeIntoAbove(ctx, nil, sessionID, sourceID, nil) +} + +func (r *sqliteRepository) AutoMergeIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, sourceID string) (*QueuedMessage, bool, error) { + return r.autoMergeIntoAbove(ctx, &identity, identity.SessionID, sourceID, nil) +} + +func (r *sqliteRepository) AutoMergeIntoAboveForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + sourceID string, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + return r.autoMergeIntoAbove(ctx, &identity, identity.SessionID, sourceID, &policy) +} + +func (r *sqliteRepository) autoMergeIntoAbove( + ctx context.Context, + identity *QueueSessionIdentity, + sessionID, sourceID string, + policy *AutoMergePolicy, +) (*QueuedMessage, bool, error) { unlock := r.withSessionLock(sessionID) defer unlock() @@ -1937,9 +3426,18 @@ func (r *sqliteRepository) AutoMergeIntoAbove(ctx context.Context, sessionID, so return nil, false, fmt.Errorf("begin automatic merge tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return nil, false, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return nil, false, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, false, err + } + if err := r.validateAutoMergePolicyTx(ctx, tx, identity, policy); err != nil { + return nil, false, err + } source, err := readMergeSource(ctx, r, tx, sessionID, sourceID) if errors.Is(err, ErrEntryNotFound) { @@ -1981,6 +3479,44 @@ func (r *sqliteRepository) AutoMergeIntoAbove(ctx context.Context, sessionID, so // the fold is the admission. Missing or incompatible tails are successful // skips and leave storage unchanged. func (r *sqliteRepository) AutoMergeCandidateIntoAbove(ctx context.Context, candidate *QueuedMessage) (*QueuedMessage, bool, error) { + return r.autoMergeCandidateIntoAbove(ctx, nil, candidate, nil, nil) +} + +func (r *sqliteRepository) AutoMergeCandidateIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, candidate *QueuedMessage) (*QueuedMessage, bool, error) { + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + return r.autoMergeCandidateIntoAbove(ctx, &identity, candidate, nil, nil) +} + +func (r *sqliteRepository) AutoMergeCandidateIntoAboveForSessionWithClaim(ctx context.Context, identity QueueSessionIdentity, candidate *QueuedMessage, claim QueueAttachmentClaim) (*QueuedMessage, bool, error) { + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + return r.autoMergeCandidateIntoAbove(ctx, &identity, candidate, &claim, nil) +} + +func (r *sqliteRepository) AutoMergeCandidateIntoAboveForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + candidate *QueuedMessage, + claim *QueueAttachmentClaim, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + if candidate == nil || candidate.SessionID != identity.SessionID || candidate.TaskID != identity.TaskID { + return nil, false, ErrSessionIdentityMismatch + } + return r.autoMergeCandidateIntoAbove(ctx, &identity, candidate, claim, &policy) +} + +//nolint:cyclop,funlen // the transaction's guards intentionally surround the one durable fold. +func (r *sqliteRepository) autoMergeCandidateIntoAbove( + ctx context.Context, + identity *QueueSessionIdentity, + candidate *QueuedMessage, + claim *QueueAttachmentClaim, + policy *AutoMergePolicy, +) (*QueuedMessage, bool, error) { unlock := r.withSessionLock(candidate.SessionID) defer unlock() @@ -2000,6 +3536,12 @@ func (r *sqliteRepository) AutoMergeCandidateIntoAbove(ctx context.Context, cand if err := r.lockSessionTx(ctx, tx, candidate.SessionID); err != nil { return nil, false, err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, false, err + } + if err := r.validateAutoMergePolicyTx(ctx, tx, identity, policy); err != nil { + return nil, false, err + } target, storedContent, storedAttachmentsJSON, storedMetadataJSON, err := r.scanTailWithRawJSON(ctx, tx, candidate.SessionID) if err != nil { @@ -2018,6 +3560,11 @@ func (r *sqliteRepository) AutoMergeCandidateIntoAbove(ctx context.Context, cand if !compatible { return nil, false, nil } + if err := claimOptionalMessageAttachmentsTx( + ctx, tx, identity, claim, candidate.TaskID, candidate.SessionID, + ); err != nil { + return nil, false, err + } attachmentsJSON, err := marshalAttachments(values.attachments) if err != nil { return nil, false, err @@ -2070,6 +3617,37 @@ func (r *sqliteRepository) AutoMergeCandidateIntoAbove(ctx context.Context, cand return target, true, nil } +// readReorderEntriesTx loads the ordered session snapshot under the caller's lock. +func (r *sqliteRepository) readReorderEntriesTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, +) ([]*QueuedMessage, error) { + rows, err := tx.QueryxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE session_id = ? + ORDER BY position ASC + `), sessionID) + if err != nil { + return nil, fmt.Errorf("read reorder rows: %w", err) + } + stored := make([]*QueuedMessage, 0, 4) + for rows.Next() { + msg, scanErr := scanQueuedRow(rows) + if scanErr != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan reorder row: %w", scanErr) + } + stored = append(stored, msg) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close reorder rows: %w", err) + } + return stored, nil +} + // ReorderEntries atomically rewrites the FIFO positions of the session's // visible pending entries to match orderedIDs. Reserved in-flight lifecycle // rows keep their place in the sequence; visible rows are interleaved in the @@ -2084,6 +3662,14 @@ func (r *sqliteRepository) AutoMergeCandidateIntoAbove(ctx context.Context, cand // A row that vanished mid-transaction (a drain racing the lock) fails the same // precondition and rolls back. func (r *sqliteRepository) ReorderEntries(ctx context.Context, sessionID string, orderedIDs []string) error { + return r.reorderEntries(ctx, nil, sessionID, orderedIDs) +} + +func (r *sqliteRepository) ReorderEntriesForSession(ctx context.Context, identity QueueSessionIdentity, orderedIDs []string) error { + return r.reorderEntries(ctx, &identity, identity.SessionID, orderedIDs) +} + +func (r *sqliteRepository) reorderEntries(ctx context.Context, identity *QueueSessionIdentity, sessionID string, orderedIDs []string) error { unlock := r.withSessionLock(sessionID) defer unlock() @@ -2092,31 +3678,19 @@ func (r *sqliteRepository) ReorderEntries(ctx context.Context, sessionID string, return fmt.Errorf("begin reorder tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return err } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } - rows, err := tx.QueryxContext(ctx, r.db.Rebind(` - SELECT id, session_id, task_id, position, content, model, plan_mode, - attachments_json, metadata_json, queued_at, queued_by - FROM queued_messages - WHERE session_id = ? - ORDER BY position ASC - `), sessionID) + stored, err := r.readReorderEntriesTx(ctx, tx, sessionID) if err != nil { - return fmt.Errorf("read reorder rows: %w", err) - } - stored := make([]*QueuedMessage, 0, 4) - for rows.Next() { - msg, scanErr := scanQueuedRow(rows) - if scanErr != nil { - _ = rows.Close() - return fmt.Errorf("scan reorder row: %w", scanErr) - } - stored = append(stored, msg) - } - if err := rows.Close(); err != nil { - return fmt.Errorf("close reorder rows: %w", err) + return err } visible, _ := splitVisibleAndReserved(stored) @@ -2271,34 +3845,52 @@ func applyMergeWrites(ctx context.Context, r *sqliteRepository, tx *sqlx.Tx, tar // DeleteByID removes a single pending entry scoped to its session. func (r *sqliteRepository) DeleteByID(ctx context.Context, sessionID, entryID string) error { + _, err := r.deleteByID(ctx, nil, sessionID, entryID) + return err +} + +func (r *sqliteRepository) DeleteByIDForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) (*QueueRemovalResult, error) { + return r.deleteByID(ctx, &identity, identity.SessionID, entryID) +} + +func (r *sqliteRepository) deleteByID(ctx context.Context, identity *QueueSessionIdentity, sessionID, entryID string) (*QueueRemovalResult, error) { unlock := r.withSessionLock(sessionID) defer unlock() tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return fmt.Errorf("begin delete queued tx: %w", err) + return nil, fmt.Errorf("begin delete queued tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return nil, err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { - return err + return nil, err + } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return nil, err } - var metadataJSON string - err = tx.GetContext(ctx, &metadataJSON, r.db.Rebind(` - SELECT metadata_json FROM queued_messages WHERE id = ? AND session_id = ? + row := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE id = ? AND session_id = ? `), entryID, sessionID) + removed, metadataJSON, err := scanQueuedRowWithMetadataJSON(row) if errors.Is(err, sql.ErrNoRows) { - return ErrEntryNotFound + return nil, ErrEntryNotFound } if err != nil { - return fmt.Errorf("read queued cancellation candidate: %w", err) + return nil, fmt.Errorf("read queued cancellation candidate: %w", err) } reserved, err := isReservedMetadataJSON(metadataJSON) if err != nil { - return err + return nil, err } if reserved { - return ErrEntryNotFound + return nil, ErrEntryNotFound } res, err := tx.ExecContext(ctx, r.db.Rebind(` @@ -2306,70 +3898,148 @@ func (r *sqliteRepository) DeleteByID(ctx context.Context, sessionID, entryID st WHERE id = ? AND session_id = ? AND metadata_json = ? `), entryID, sessionID, metadataJSON) if err != nil { - return fmt.Errorf("delete queued: %w", err) + return nil, fmt.Errorf("delete queued: %w", err) } n, err := res.RowsAffected() if err != nil { - return err + return nil, err } if n == 0 { - return ErrEntryNotFound + return nil, ErrEntryNotFound } - return tx.Commit() + retained, err := listQueuedMessagesBySessionTx(ctx, r, tx, sessionID) + if err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, err + } + return &QueueRemovalResult{ + Removed: []QueuedMessage{*removed}, + Retained: retained, + }, nil +} + +func listQueuedMessagesBySessionTx( + ctx context.Context, + r *sqliteRepository, + tx *sqlx.Tx, + sessionID string, +) ([]QueuedMessage, error) { + rows, err := tx.QueryxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE session_id = ? + ORDER BY position ASC + `), sessionID) + if err != nil { + return nil, fmt.Errorf("list retained queue entries: %w", err) + } + defer func() { _ = rows.Close() }() + retained := make([]QueuedMessage, 0) + for rows.Next() { + message, scanErr := scanQueuedRow(rows) + if scanErr != nil { + return nil, scanErr + } + retained = append(retained, *message) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate retained queue entries: %w", err) + } + return retained, nil } // DeleteAllBySession removes every pending entry for a session, keeping reserved in-flight rows. func (r *sqliteRepository) DeleteAllBySession(ctx context.Context, sessionID string) (int, error) { + result, err := r.deleteAllBySession(ctx, nil, sessionID) + if err != nil { + return 0, err + } + return len(result.Removed), nil +} + +func (r *sqliteRepository) DeleteAllBySessionForIdentity(ctx context.Context, identity QueueSessionIdentity) (*QueueRemovalResult, error) { + return r.deleteAllBySession(ctx, &identity, identity.SessionID) +} + +func (r *sqliteRepository) deleteAllBySession(ctx context.Context, identity *QueueSessionIdentity, sessionID string) (*QueueRemovalResult, error) { unlock := r.withSessionLock(sessionID) defer unlock() tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return 0, fmt.Errorf("begin delete all queued tx: %w", err) + return nil, fmt.Errorf("begin delete all queued tx: %w", err) } defer func() { _ = tx.Rollback() }() + if identity != nil { + if err := r.guardActiveTaskTx(ctx, tx, identity.TaskID); err != nil { + return nil, err + } + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { - return 0, err + return nil, err + } + if identity != nil { + if err := r.validateSessionIdentityTx(ctx, tx, *identity); err != nil { + return nil, err + } } - candidates, err := cancellationCandidates(ctx, tx, sessionID) + candidates, err := cancellationCandidates(ctx, r, tx, sessionID) if err != nil { - return 0, err + return nil, err } - removed := 0 + result := &QueueRemovalResult{} for _, candidate := range candidates { + if candidate.reserved { + result.Retained = append(result.Retained, candidate.message) + continue + } res, err := tx.ExecContext(ctx, r.db.Rebind(` DELETE FROM queued_messages WHERE id = ? AND session_id = ? AND metadata_json = ? - `), candidate.id, sessionID, candidate.metadataJSON) + `), candidate.message.ID, sessionID, candidate.metadataJSON) if err != nil { - return 0, fmt.Errorf("delete queued candidate: %w", err) + return nil, fmt.Errorf("delete queued candidate: %w", err) } affected, err := res.RowsAffected() if err != nil { - return 0, fmt.Errorf("delete queued candidate rows affected: %w", err) + return nil, fmt.Errorf("delete queued candidate rows affected: %w", err) } - removed += int(affected) + if affected == 0 { + result.Retained = append(result.Retained, candidate.message) + continue + } + result.Removed = append(result.Removed, candidate.message) } if err := tx.Commit(); err != nil { - return 0, err + return nil, err } - return removed, nil + return result, nil } type cancellationCandidate struct { - id string + message QueuedMessage metadataJSON string + reserved bool } -// cancellationCandidates lists pending entries eligible for cancellation, excluding reserved in-flight rows. +// cancellationCandidates lists every entry so cancellation can return both +// removed pending work and retained in-flight work from one transaction. func cancellationCandidates( ctx context.Context, + r *sqliteRepository, tx *sqlx.Tx, sessionID string, ) ([]cancellationCandidate, error) { - rows, err := tx.QueryxContext(ctx, tx.Rebind(` - SELECT id, metadata_json FROM queued_messages WHERE session_id = ? + rows, err := tx.QueryxContext(ctx, r.db.Rebind(` + SELECT id, session_id, task_id, position, content, model, plan_mode, + attachments_json, metadata_json, queued_at, queued_by + FROM queued_messages + WHERE session_id = ? + ORDER BY position ASC `), sessionID) if err != nil { return nil, fmt.Errorf("list queued cancellation candidates: %w", err) @@ -2378,17 +4048,17 @@ func cancellationCandidates( candidates := make([]cancellationCandidate, 0) for rows.Next() { - var candidate cancellationCandidate - if err := rows.Scan(&candidate.id, &candidate.metadataJSON); err != nil { - return nil, fmt.Errorf("scan queued cancellation candidate: %w", err) + message, metadataJSON, scanErr := scanQueuedRowWithMetadataJSON(rows) + if scanErr != nil { + return nil, fmt.Errorf("scan queued cancellation candidate: %w", scanErr) } - reserved, err := isReservedMetadataJSON(candidate.metadataJSON) + reserved, err := isReservedMetadataJSON(metadataJSON) if err != nil { return nil, err } - if !reserved { - candidates = append(candidates, candidate) - } + candidates = append(candidates, cancellationCandidate{ + message: *message, metadataJSON: metadataJSON, reserved: reserved, + }) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterate queued cancellation candidates: %w", err) @@ -2406,31 +4076,37 @@ func isReservedMetadataJSON(metadataJSON string) (bool, error) { } return (&QueuedMessage{Metadata: metadata}).IsReservedInFlight(), nil } +func lifecycleReservationFromMetadataJSON(metadataJSON string) (string, bool, error) { + metadata := make(map[string]interface{}) + if metadataJSON != "" && metadataJSON != "{}" { + if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { + return "", false, fmt.Errorf("unmarshal lifecycle reservation metadata: %w", err) + } + } + message := &QueuedMessage{Metadata: metadata} + return lifecycleReservationIncarnation(metadata), message.IsReservedInFlight(), nil +} -// TransferSession moves all entries (and any pending move) from one session to another. -func (r *sqliteRepository) TransferSession(ctx context.Context, oldSessionID, newSessionID string) error { - // The transfer moves rows out of the source and into the destination, so - // it must hold BOTH sessions' locks: a concurrent source-side insert - // (holding the source lock) could otherwise commit a row the transfer's - // READ COMMITTED UPDATE missed, orphaning it on the old session. Acquire - // in stable sorted order — in-process and cross-process alike — so - // concurrent transfers in opposite directions cannot deadlock. - first, second := oldSessionID, newSessionID - if first > second { - first, second = second, first +func (r *sqliteRepository) guardTransferTasksTx( + ctx context.Context, + tx *sqlx.Tx, + source, destination *QueueSessionIdentity, +) error { + if source == nil || destination == nil { + return nil } - unlockFirst := r.withSessionLock(first) - defer unlockFirst() - if first != second { - unlockSecond := r.withSessionLock(second) - defer unlockSecond() + if source.TaskID != destination.TaskID { + return ErrSessionIdentityMismatch } + return r.guardActiveTaskTx(ctx, tx, source.TaskID) +} - tx, err := r.db.BeginTxx(ctx, nil) - if err != nil { - return fmt.Errorf("begin transfer tx: %w", err) - } - defer func() { _ = tx.Rollback() }() +func (r *sqliteRepository) lockAndValidateTransferSessionsTx( + ctx context.Context, + tx *sqlx.Tx, + source, destination *QueueSessionIdentity, + first, second string, +) error { if err := r.lockSessionTx(ctx, tx, first); err != nil { return err } @@ -2439,17 +4115,89 @@ func (r *sqliteRepository) TransferSession(ctx context.Context, oldSessionID, ne return err } } + if source == nil || destination == nil { + return nil + } + if err := r.validateSessionIdentityTx(ctx, tx, *source); err != nil { + return err + } + return r.validateSessionIdentityTx(ctx, tx, *destination) +} + +func (r *sqliteRepository) lockTransferSessions(oldSessionID, newSessionID string) ( + first string, + second string, + release func(), +) { + first, second = oldSessionID, newSessionID + if first > second { + first, second = second, first + } + unlockFirst := r.withSessionLock(first) + if first == second { + return first, second, unlockFirst + } + unlockSecond := r.withSessionLock(second) + return first, second, func() { + unlockSecond() + unlockFirst() + } +} + +// TransferSession moves all entries (and any pending move) from one session to another. +func (r *sqliteRepository) TransferSession(ctx context.Context, oldSessionID, newSessionID string) error { + return r.transferSession(ctx, nil, nil, oldSessionID, newSessionID) +} + +func (r *sqliteRepository) TransferSessionIdentities(ctx context.Context, source, destination QueueSessionIdentity) error { + return r.transferSession(ctx, &source, &destination, source.SessionID, destination.SessionID) +} + +//nolint:cyclop // transfer preserves queue, policy, and attachment invariants atomically. +func (r *sqliteRepository) transferSession(ctx context.Context, source, destination *QueueSessionIdentity, oldSessionID, newSessionID string) error { + // The transfer moves rows out of the source and into the destination, so + // it must hold BOTH sessions' locks: a concurrent source-side insert + // (holding the source lock) could otherwise commit a row the transfer's + // READ COMMITTED UPDATE missed, orphaning it on the old session. Acquire + // in stable sorted order — in-process and cross-process alike — so + // concurrent transfers in opposite directions cannot deadlock. + first, second, release := r.lockTransferSessions(oldSessionID, newSessionID) + defer release() + attachmentsTablePresent, err := r.sharedTablePresent("task_message_attachments") + if err != nil { + return err + } + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transfer tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.guardTransferTasksTx(ctx, tx, source, destination); err != nil { + return err + } + if err := r.lockAndValidateTransferSessionsTx(ctx, tx, source, destination, first, second); err != nil { + return err + } if oldSessionID == newSessionID { return tx.Commit() } - sourceAutoRun, err := r.getAutoRunTx(ctx, tx, oldSessionID) + if err := rejectReservedLifecycleTransferTx(ctx, tx, oldSessionID); err != nil { + return err + } + sourceAutoRun, err := r.getAutoRunTx(ctx, tx, source, oldSessionID) if err != nil { return err } - destinationAutoRun, err := r.getAutoRunTx(ctx, tx, newSessionID) + destinationAutoRun, err := r.getAutoRunTx(ctx, tx, destination, newSessionID) if err != nil { return err } + if attachmentsTablePresent { + if err := r.transferQueuedAttachmentClaimsTx(ctx, tx, oldSessionID, newSessionID); err != nil { + return err + } + } // Shift positions on the destination so transferred entries land at the tail // without colliding on the (session_id, position) implicit ordering. @@ -2470,12 +4218,10 @@ func (r *sqliteRepository) TransferSession(ctx context.Context, oldSessionID, ne `), newSessionID); err != nil { return fmt.Errorf("clear dest pending move: %w", err) } - if _, err := tx.ExecContext(ctx, r.db.Rebind(` - UPDATE pending_moves SET session_id = ? WHERE session_id = ? - `), newSessionID, oldSessionID); err != nil { - return fmt.Errorf("transfer pending move: %w", err) + if err := r.transferPendingMoveTx(ctx, tx, oldSessionID, newSessionID); err != nil { + return err } - if err := r.setAutoRunTx(ctx, tx, newSessionID, sourceAutoRun && destinationAutoRun); err != nil { + if err := r.setAutoRunTx(ctx, tx, destination, newSessionID, sourceAutoRun && destinationAutoRun); err != nil { return err } if _, err := tx.ExecContext(ctx, r.db.Rebind(` @@ -2486,20 +4232,183 @@ func (r *sqliteRepository) TransferSession(ctx context.Context, oldSessionID, ne return tx.Commit() } -// ReplaceSession replaces a session's queue with the supplied snapshot. -func (r *sqliteRepository) ReplaceSession(ctx context.Context, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { - tx, err := r.db.BeginTxx(ctx, nil) +//nolint:cyclop // every malformed or conflicting attachment reference must roll back transfer. +func (r *sqliteRepository) transferQueuedAttachmentClaimsTx( + ctx context.Context, + tx *sqlx.Tx, + oldSessionID, newSessionID string, +) error { + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT task_id, attachments_json + FROM queued_messages + WHERE session_id = ? + `), oldSessionID) if err != nil { - return fmt.Errorf("begin replace session tx: %w", err) + return fmt.Errorf("list queued attachment claims for transfer: %w", err) } - defer func() { _ = tx.Rollback() }() - if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { - return err + attachmentTasks := make(map[string]string) + for rows.Next() { + var taskID, encoded string + if err := rows.Scan(&taskID, &encoded); err != nil { + _ = rows.Close() + return fmt.Errorf("scan queued attachment claims for transfer: %w", err) + } + var attachments []MessageAttachment + if err := json.Unmarshal([]byte(encoded), &attachments); err != nil { + _ = rows.Close() + return fmt.Errorf("decode queued attachment claims for transfer: %w", err) + } + for _, attachment := range attachments { + if attachment.AttachmentID == "" { + continue + } + if existingTaskID, ok := attachmentTasks[attachment.AttachmentID]; ok && existingTaskID != taskID { + _ = rows.Close() + return models.ErrAttachmentClaimConflict + } + attachmentTasks[attachment.AttachmentID] = taskID + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate queued attachment claims for transfer: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close queued attachment claims for transfer: %w", err) + } + for attachmentID, taskID := range attachmentTasks { + var storedTaskID, storedSessionID, state string + err := tx.QueryRowxContext(ctx, tx.Rebind(` + SELECT task_id, session_id, state + FROM task_message_attachments + WHERE id = ? + `), attachmentID).Scan(&storedTaskID, &storedSessionID, &state) + if errors.Is(err, sql.ErrNoRows) { + return models.ErrAttachmentClaimConflict + } + if err != nil { + return fmt.Errorf("load queued attachment claim for transfer: %w", err) + } + if storedTaskID != taskID || state != string(models.AttachmentStateClaimed) { + return models.ErrAttachmentClaimConflict + } + switch storedSessionID { + case "", newSessionID: + continue + case oldSessionID: + if _, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE task_message_attachments + SET session_id = ?, updated_at = ? + WHERE id = ? AND task_id = ? AND session_id = ? AND state = ? + `), newSessionID, time.Now().UTC(), attachmentID, taskID, oldSessionID, models.AttachmentStateClaimed); err != nil { + return fmt.Errorf("transfer queued attachment claim: %w", err) + } + default: + return models.ErrAttachmentClaimConflict + } + } + return nil +} +func rejectReservedLifecycleTransferTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, +) error { + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT metadata_json FROM queued_messages WHERE session_id = ? + `), sessionID) + if err != nil { + return fmt.Errorf("list lifecycle reservations before transfer: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var metadataJSON string + if err := rows.Scan(&metadataJSON); err != nil { + return fmt.Errorf("scan lifecycle reservation before transfer: %w", err) + } + reserved, err := isReservedMetadataJSON(metadataJSON) + if err != nil { + return err + } + if reserved { + return ErrQueueChanged + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate lifecycle reservations before transfer: %w", err) + } + return nil +} + +func (r *sqliteRepository) transferPendingMoveTx( + ctx context.Context, + tx *sqlx.Tx, + oldSessionID, newSessionID string, +) error { + if !r.tasksTablePresent { + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE pending_moves SET session_id = ? WHERE session_id = ? + `), newSessionID, oldSessionID); err != nil { + return fmt.Errorf("transfer pending move: %w", err) + } + return nil } - if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM queued_messages WHERE session_id = ?`), sessionID); err != nil { - return fmt.Errorf("clear queued messages: %w", err) + var destinationIncarnationID string + if err := tx.GetContext(ctx, &destinationIncarnationID, r.db.Rebind(` + SELECT queue_incarnation_id FROM task_sessions WHERE id = ? + `), newSessionID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrSessionIdentityMismatch + } + return fmt.Errorf("load destination session incarnation: %w", err) } + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE pending_moves + SET session_id = ?, session_incarnation_id = ? + WHERE session_id = ? + `), newSessionID, destinationIncarnationID, oldSessionID); err != nil { + return fmt.Errorf("transfer pending move: %w", err) + } + return nil +} + +// ReplaceSession replaces a session's queue with the supplied snapshot. +func (r *sqliteRepository) ReplaceSession(ctx context.Context, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { + return r.replaceSession(ctx, nil, sessionID, entries, pendingMove) +} + +func (r *sqliteRepository) ReplaceSessionForIdentity(ctx context.Context, identity QueueSessionIdentity, entries []QueuedMessage, pendingMove *PendingMove) error { + return r.replaceSession(ctx, &identity, identity.SessionID, entries, pendingMove) +} + +func validateReplacementSnapshot( + identity *QueueSessionIdentity, + entries []QueuedMessage, + pendingMove *PendingMove, +) error { + if identity == nil { + return nil + } + for index := range entries { + if entries[index].SessionID != identity.SessionID || entries[index].TaskID != identity.TaskID { + return ErrSessionIdentityMismatch + } + } + if pendingMove != nil && + (pendingMove.TaskID != identity.TaskID || + pendingMove.SessionIncarnationID != identity.SessionIncarnationID) { + return ErrSessionIdentityMismatch + } + return nil +} + +func (r *sqliteRepository) restoreQueueEntriesTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, + entries []QueuedMessage, +) error { for _, entry := range entries { attachmentsJSON, err := marshalAttachments(entry.Attachments) if err != nil { @@ -2524,31 +4433,89 @@ func (r *sqliteRepository) ReplaceSession(ctx context.Context, sessionID string, return fmt.Errorf("restore queued message: %w", err) } } + return nil +} + +func (r *sqliteRepository) restorePendingMoveTx( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, + pendingMove *PendingMove, +) error { + if pendingMove == nil { + return nil + } + queuedAt := pendingMove.QueuedAt + if queuedAt.IsZero() { + queuedAt = time.Now().UTC() + } + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + INSERT INTO pending_moves ( + id, move_id, session_incarnation_id, session_id, task_id, workflow_id, + workflow_step_id, step_position, queued_at, actor, sender_session_id + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `), + uuid.New().String(), pendingMove.MoveID, pendingMove.SessionIncarnationID, sessionID, + pendingMove.TaskID, pendingMove.WorkflowID, pendingMove.WorkflowStepID, + pendingMove.Position, queuedAt, pendingMove.Actor, pendingMove.SenderSessionID, + ); err != nil { + return fmt.Errorf("restore pending move: %w", err) + } + return nil +} + +func (r *sqliteRepository) replaceSession(ctx context.Context, identity *QueueSessionIdentity, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin replace session tx: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := r.guardOptionalActiveTaskTx(ctx, tx, identity); err != nil { + return err + } + if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { + return err + } + if err := r.validateOptionalSessionIdentityTx(ctx, tx, identity); err != nil { + return err + } + if err := validateReplacementSnapshot(identity, entries, pendingMove); err != nil { + return err + } + + if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM queued_messages WHERE session_id = ?`), sessionID); err != nil { + return fmt.Errorf("clear queued messages: %w", err) + } + if err := r.restoreQueueEntriesTx(ctx, tx, sessionID, entries); err != nil { + return err + } if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM pending_moves WHERE session_id = ?`), sessionID); err != nil { return fmt.Errorf("clear pending move: %w", err) } - if pendingMove != nil { - queuedAt := pendingMove.QueuedAt - if queuedAt.IsZero() { - queuedAt = time.Now().UTC() - } - if _, err := tx.ExecContext(ctx, r.db.Rebind(` - INSERT INTO pending_moves (id, move_id, session_id, task_id, workflow_id, workflow_step_id, step_position, queued_at, actor, sender_session_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), - uuid.New().String(), pendingMove.MoveID, sessionID, pendingMove.TaskID, pendingMove.WorkflowID, - pendingMove.WorkflowStepID, pendingMove.Position, queuedAt, pendingMove.Actor, - pendingMove.SenderSessionID, - ); err != nil { - return fmt.Errorf("restore pending move: %w", err) - } + if err := r.restorePendingMoveTx(ctx, tx, sessionID, pendingMove); err != nil { + return err } return tx.Commit() } // SetPendingMove upserts the deferred workflow move for a session. func (r *sqliteRepository) SetPendingMove(ctx context.Context, sessionID string, move *PendingMove) error { + if move == nil || move.TaskID == "" || sessionID == "" { + return ErrSessionIdentityMismatch + } + if move.SessionIncarnationID == "" { + if r.tasksTablePresent { + identity, err := r.ResolveSessionIdentity(ctx, move.TaskID, sessionID) + if err != nil { + return err + } + move.SessionIncarnationID = identity.SessionIncarnationID + } else { + move.SessionIncarnationID = "legacy:" + sessionID + } + } if move.QueuedAt.IsZero() { move.QueuedAt = time.Now().UTC() } @@ -2561,14 +4528,30 @@ func (r *sqliteRepository) SetPendingMove(ctx context.Context, sessionID string, return fmt.Errorf("begin set pending move tx: %w", err) } defer func() { _ = tx.Rollback() }() + if err := r.guardActiveTaskTx(ctx, tx, move.TaskID); err != nil { + return err + } if err := r.lockSessionTx(ctx, tx, sessionID); err != nil { return err } + if r.tasksTablePresent { + if err := r.validateSessionIdentityTx(ctx, tx, QueueSessionIdentity{ + TaskID: move.TaskID, + SessionID: sessionID, + SessionIncarnationID: move.SessionIncarnationID, + }); err != nil { + return err + } + } if _, err := tx.ExecContext(ctx, r.db.Rebind(` - INSERT INTO pending_moves (id, move_id, session_id, task_id, workflow_id, workflow_step_id, step_position, queued_at, actor, sender_session_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO pending_moves ( + id, move_id, session_incarnation_id, session_id, task_id, workflow_id, + workflow_step_id, step_position, queued_at, actor, sender_session_id + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET task_id = excluded.task_id, + session_incarnation_id = excluded.session_incarnation_id, workflow_id = excluded.workflow_id, workflow_step_id = excluded.workflow_step_id, step_position = excluded.step_position, @@ -2577,7 +4560,9 @@ func (r *sqliteRepository) SetPendingMove(ctx context.Context, sessionID string, sender_session_id = excluded.sender_session_id, move_id = excluded.move_id `), - uuid.New().String(), move.MoveID, sessionID, move.TaskID, move.WorkflowID, move.WorkflowStepID, move.Position, move.QueuedAt, move.Actor, move.SenderSessionID, + uuid.New().String(), move.MoveID, move.SessionIncarnationID, sessionID, move.TaskID, + move.WorkflowID, move.WorkflowStepID, move.Position, move.QueuedAt, move.Actor, + move.SenderSessionID, ); err != nil { return fmt.Errorf("upsert pending move: %w", err) } @@ -2587,29 +4572,34 @@ func (r *sqliteRepository) SetPendingMove(ctx context.Context, sessionID string, // GetPendingMove returns the deferred workflow move for a session, or nil when absent. func (r *sqliteRepository) GetPendingMove(ctx context.Context, sessionID string) (*PendingMove, error) { var ( - moveID, taskID, workflowID, workflowStepID string - position int - queuedAt time.Time - actor, senderSessionID string + moveID, sessionIncarnationID, taskID, workflowID, workflowStepID string + position int + queuedAt time.Time + actor, senderSessionID string ) if err := r.ro.QueryRowxContext(ctx, r.db.Rebind(` - SELECT move_id, task_id, workflow_id, workflow_step_id, step_position, queued_at, actor, sender_session_id + SELECT move_id, session_incarnation_id, task_id, workflow_id, workflow_step_id, + step_position, queued_at, actor, sender_session_id FROM pending_moves WHERE session_id = ? - `), sessionID).Scan(&moveID, &taskID, &workflowID, &workflowStepID, &position, &queuedAt, &actor, &senderSessionID); err != nil { + `), sessionID).Scan( + &moveID, &sessionIncarnationID, &taskID, &workflowID, &workflowStepID, + &position, &queuedAt, &actor, &senderSessionID, + ); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, fmt.Errorf("read pending move: %w", err) } return &PendingMove{ - MoveID: moveID, - TaskID: taskID, - WorkflowID: workflowID, - WorkflowStepID: workflowStepID, - Position: position, - QueuedAt: queuedAt, - Actor: actor, - SenderSessionID: senderSessionID, + MoveID: moveID, + SessionIncarnationID: sessionIncarnationID, + TaskID: taskID, + WorkflowID: workflowID, + WorkflowStepID: workflowStepID, + Position: position, + QueuedAt: queuedAt, + Actor: actor, + SenderSessionID: senderSessionID, }, nil } @@ -2627,15 +4617,19 @@ func (r *sqliteRepository) TakePendingMove(ctx context.Context, sessionID string } var ( - moveID, taskID, workflowID, workflowStepID string - position int - queuedAt time.Time - actor, senderSessionID string + moveID, sessionIncarnationID, taskID, workflowID, workflowStepID string + position int + queuedAt time.Time + actor, senderSessionID string ) if err := tx.QueryRowxContext(ctx, r.db.Rebind(` - SELECT move_id, task_id, workflow_id, workflow_step_id, step_position, queued_at, actor, sender_session_id + SELECT move_id, session_incarnation_id, task_id, workflow_id, workflow_step_id, + step_position, queued_at, actor, sender_session_id FROM pending_moves WHERE session_id = ? - `), sessionID).Scan(&moveID, &taskID, &workflowID, &workflowStepID, &position, &queuedAt, &actor, &senderSessionID); err != nil { + `), sessionID).Scan( + &moveID, &sessionIncarnationID, &taskID, &workflowID, &workflowStepID, + &position, &queuedAt, &actor, &senderSessionID, + ); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -2648,14 +4642,15 @@ func (r *sqliteRepository) TakePendingMove(ctx context.Context, sessionID string return nil, err } return &PendingMove{ - MoveID: moveID, - TaskID: taskID, - WorkflowID: workflowID, - WorkflowStepID: workflowStepID, - Position: position, - QueuedAt: queuedAt, - Actor: actor, - SenderSessionID: senderSessionID, + MoveID: moveID, + SessionIncarnationID: sessionIncarnationID, + TaskID: taskID, + WorkflowID: workflowID, + WorkflowStepID: workflowStepID, + Position: position, + QueuedAt: queuedAt, + Actor: actor, + SenderSessionID: senderSessionID, }, nil } @@ -2705,8 +4700,27 @@ func (r *sqliteRepository) scanTailWithRawJSON(ctx context.Context, tx *sqlx.Tx, return msg, rawContent, rawAttachments, rawMetadata, nil } -// findCoalesced locates the entry matching the session, owner, and coalesce key inside a transaction. -func (r *sqliteRepository) findCoalesced(ctx context.Context, tx *sqlx.Tx, sessionID, queuedBy, coalesceKey string) (*QueuedMessage, error) { +func (r *sqliteRepository) findPendingCoalescedSuccessor( + ctx context.Context, + tx *sqlx.Tx, + sessionID, queuedBy string, + metadata map[string]interface{}, +) (*QueuedMessage, error) { + coalesceKey := metadataString(metadata, MetadataCoalesceKey) + if coalesceKey == "" { + return nil, nil + } + successor, _, err := r.findCoalesced(ctx, tx, sessionID, queuedBy, coalesceKey) + return successor, err +} + +// findCoalesced locates the pending entry matching the session, owner, and +// coalesce key and reports whether an in-flight match was skipped. +func (r *sqliteRepository) findCoalesced( + ctx context.Context, + tx *sqlx.Tx, + sessionID, queuedBy, coalesceKey string, +) (*QueuedMessage, bool, error) { rows, err := tx.QueryxContext(ctx, r.db.Rebind(` SELECT id, session_id, task_id, position, content, model, plan_mode, attachments_json, metadata_json, queued_at, queued_by @@ -2715,22 +4729,28 @@ func (r *sqliteRepository) findCoalesced(ctx context.Context, tx *sqlx.Tx, sessi ORDER BY position ASC `), sessionID, queuedBy) if err != nil { - return nil, fmt.Errorf("scan coalesced queued: %w", err) + return nil, false, fmt.Errorf("scan coalesced queued: %w", err) } defer func() { _ = rows.Close() }() + reservedMatch := false for rows.Next() { msg, err := scanQueuedRow(rows) if err != nil { - return nil, err + return nil, false, err } - if metadataString(msg.Metadata, MetadataCoalesceKey) == coalesceKey { - return msg, nil + if metadataString(msg.Metadata, MetadataCoalesceKey) != coalesceKey { + continue } + if msg.IsReservedInFlight() { + reservedMatch = true + continue + } + return msg, reservedMatch, nil } if err := rows.Err(); err != nil { - return nil, err + return nil, false, err } - return nil, nil + return nil, reservedMatch, nil } // scanQueuedRow scans a single queued_messages row from any sqlx-compatible row source. diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite_test.go b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite_test.go index 6957eebc48..1f585b36fe 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/jmoiron/sqlx" + internaldb "github.com/kandev/kandev/internal/db" _ "github.com/mattn/go-sqlite3" ) @@ -35,6 +36,39 @@ func newTestSQLiteRepo(t *testing.T) Repository { return repo } +func seedQueueSessionIdentity(t *testing.T, repo Repository, identity QueueSessionIdentity) { + t.Helper() + switch typed := repo.(type) { + case *memoryRepository: + typed.mu.Lock() + typed.identities[identity.SessionID] = identity + typed.mu.Unlock() + case *sqliteRepository: + statements := []string{ + `CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, archived_at TIMESTAMP, updated_at TIMESTAMP)`, + `ALTER TABLE task_sessions ADD COLUMN task_id TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE task_sessions ADD COLUMN queue_incarnation_id TEXT NOT NULL DEFAULT ''`, + } + for index, statement := range statements { + if _, err := typed.db.Exec(statement); err != nil && (index == 0 || !internaldb.IsDuplicateColumnError(err)) { + t.Fatalf("prepare queue session authority: %v", err) + } + } + if _, err := typed.db.Exec(`INSERT OR IGNORE INTO tasks (id, updated_at) VALUES (?, CURRENT_TIMESTAMP)`, identity.TaskID); err != nil { + t.Fatalf("seed queue task authority: %v", err) + } + if _, err := typed.db.Exec(` + INSERT INTO task_sessions (id, task_id, queue_incarnation_id) VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET task_id = excluded.task_id, queue_incarnation_id = excluded.queue_incarnation_id + `, identity.SessionID, identity.TaskID, identity.SessionIncarnationID); err != nil { + t.Fatalf("seed queue session authority: %v", err) + } + typed.tasksTablePresent = true + default: + t.Fatalf("unsupported queue repository authority fixture %T", repo) + } +} + // seedLiveSessions inserts stub task_sessions rows so queue counts that join // live sessions treat these session IDs as present. func seedLiveSessions(t *testing.T, repo Repository, sessionIDs ...string) { @@ -217,6 +251,52 @@ func TestSQLiteRepository_ReserveHeadMarksLifecycleRowInFlight(t *testing.T) { } } +func TestSQLiteRepository_ReplacementIncarnationDiscardsStaleLifecycleReservation(t *testing.T) { + repo := newTestSQLiteRepo(t) + ctx := context.Background() + first := QueueSessionIdentity{ + TaskID: "t1", + SessionID: "s1", + SessionIncarnationID: "incarnation-1", + } + seedQueueSessionIdentity(t, repo, first) + msg := &QueuedMessage{ + SessionID: "s1", TaskID: "t1", Content: "pr merged", QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{MetadataLifecycleDurable: true}, + } + if err := repo.InsertForSession(ctx, first, msg, 0); err != nil { + t.Fatalf("insert lifecycle message: %v", err) + } + reserved, enabled, err := repo.ReserveHeadIfAutoRunForSession(ctx, first) + if err != nil || !enabled || reserved == nil { + t.Fatalf("reserve first incarnation: reserved=%+v enabled=%v err=%v", reserved, enabled, err) + } + + replacement := first + replacement.SessionIncarnationID = "incarnation-2" + seedQueueSessionIdentity(t, repo, replacement) + fresh := &QueuedMessage{ + SessionID: "s1", TaskID: "t1", Content: "fresh prompt", QueuedBy: QueuedByUser, + } + if err := repo.InsertForSession(ctx, replacement, fresh, 0); err != nil { + t.Fatalf("insert replacement message: %v", err) + } + next, enabled, err := repo.ReserveHeadIfAutoRunForSession(ctx, replacement) + if err != nil || !enabled { + t.Fatalf("reserve replacement incarnation: enabled=%v err=%v", enabled, err) + } + if next == nil || next.ID != fresh.ID { + t.Fatalf("replacement reservation = %+v, want fresh row %s", next, fresh.ID) + } + entries, err := repo.ListBySession(ctx, first.SessionID) + if err != nil { + t.Fatalf("list after stale reservation discard: %v", err) + } + if len(entries) != 0 { + t.Fatalf("entries after stale reservation discard = %d, want 0", len(entries)) + } +} + func TestSQLiteRepository_ReserveHeadUsesStoredMetadataGuard(t *testing.T) { repo := newTestSQLiteRepo(t) ctx := context.Background() @@ -327,14 +407,11 @@ func TestSQLiteRepository_ReserveAfterRestartReturnsRetryableLifecycleMetadata(t t.Fatal("returned reservation lost process-local reservation evidence") } - retried, _, err := repo.InsertOrReplaceLifecycleByCoalesceKey( - ctx, reserved, "github-pr:repo:1:merged", 0, false, - ) - if err != nil { + if err := repo.RequeuePreservingFIFO(ctx, reserved); err != nil { t.Fatalf("requeue failed delivery: %v", err) } - if retried.IsReservedInFlight() { - t.Fatalf("requeued copy retained transient marker: %+v", retried.Metadata) + if reserved.IsReservedInFlight() { + t.Fatalf("requeued copy retained transient marker: %+v", reserved.Metadata) } entries, err := repo.ListBySession(ctx, "s1") if err != nil { @@ -797,6 +874,55 @@ func TestSQLiteRepository_TransferSession(t *testing.T) { } } +func TestRepositories_TransferRejectsSourceLifecycleReservation(t *testing.T) { + factories := []struct { + name string + new func(*testing.T) Repository + }{ + {name: "memory", new: func(*testing.T) Repository { return NewMemoryRepository() }}, + {name: "sqlite", new: newTestSQLiteRepo}, + } + for _, factory := range factories { + t.Run(factory.name, func(t *testing.T) { + repo := factory.new(t) + ctx := context.Background() + source := QueueSessionIdentity{ + TaskID: "t1", SessionID: "source", SessionIncarnationID: "source-incarnation", + } + destination := QueueSessionIdentity{ + TaskID: "t1", SessionID: "destination", SessionIncarnationID: "destination-incarnation", + } + seedQueueSessionIdentity(t, repo, source) + seedQueueSessionIdentity(t, repo, destination) + entry := &QueuedMessage{ + SessionID: source.SessionID, + TaskID: source.TaskID, + Content: "durable", + QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{MetadataLifecycleDurable: true}, + } + if err := repo.InsertForSession(ctx, source, entry, 0); err != nil { + t.Fatalf("insert durable source: %v", err) + } + if reserved, _, err := repo.ReserveHeadIfAutoRunForSession(ctx, source); err != nil || reserved == nil { + t.Fatalf("reserve durable source: reserved=%+v err=%v", reserved, err) + } + + if err := repo.TransferSessionIdentities(ctx, source, destination); !errors.Is(err, ErrQueueChanged) { + t.Fatalf("transfer error = %v, want ErrQueueChanged", err) + } + sourceEntries, err := repo.ListBySession(ctx, source.SessionID) + if err != nil || len(sourceEntries) != 1 { + t.Fatalf("source entries after rejected transfer = %+v err=%v", sourceEntries, err) + } + destinationEntries, err := repo.ListBySession(ctx, destination.SessionID) + if err != nil || len(destinationEntries) != 0 { + t.Fatalf("destination entries after rejected transfer = %+v err=%v", destinationEntries, err) + } + }) + } +} + func TestSQLiteRepository_ReplaceSessionPreservesQueuedIdentity(t *testing.T) { repo := newTestSQLiteRepo(t) ctx := context.Background() @@ -865,7 +991,16 @@ func TestSQLiteRepository_PendingMove(t *testing.T) { t.Fatalf("expected nil move on empty, got %v err=%v", move, err) } - move := &PendingMove{MoveID: "move-a", TaskID: "t1", WorkflowID: "w1", WorkflowStepID: "step-A", Position: 0, Actor: "agent", SenderSessionID: "sender-s1"} + move := &PendingMove{ + MoveID: "move-a", + SessionIncarnationID: "incarnation-a", + TaskID: "t1", + WorkflowID: "w1", + WorkflowStepID: "step-A", + Position: 0, + Actor: "agent", + SenderSessionID: "sender-s1", + } if err := repo.SetPendingMove(ctx, "s1", move); err != nil { t.Fatalf("set pending: %v", err) } @@ -887,6 +1022,9 @@ func TestSQLiteRepository_PendingMove(t *testing.T) { if got == nil || got.MoveID != "move-b" { t.Errorf("expected move-b move ID after upsert, got %+v", got) } + if got == nil || got.SessionIncarnationID != "incarnation-a" { + t.Errorf("expected incarnation-a after upsert, got %+v", got) + } if got == nil || got.Actor != "agent" { t.Errorf("expected agent actor after upsert, got %+v", got) } diff --git a/apps/backend/internal/orchestrator/messagequeue/send_now.go b/apps/backend/internal/orchestrator/messagequeue/send_now.go index b1b070b71d..6cede26bca 100644 --- a/apps/backend/internal/orchestrator/messagequeue/send_now.go +++ b/apps/backend/internal/orchestrator/messagequeue/send_now.go @@ -46,9 +46,11 @@ var ( // at its original position and every durable lifecycle row can be acknowledged // only after the replacement prompt is accepted. type SendNowClaim struct { - Sources []QueuedMessage `json:"sources"` - Dispatch QueuedMessage `json:"dispatch"` - SourceGenerations map[string]int64 `json:"source_generations,omitempty"` + Identity QueueSessionIdentity `json:"identity"` + OperationGeneration int64 `json:"operation_generation"` + Sources []QueuedMessage `json:"sources"` + Dispatch QueuedMessage `json:"dispatch"` + SourceGenerations map[string]int64 `json:"source_generations,omitempty"` } func sendNowSourceGenerationChanged(claim *SendNowClaim, source QueuedMessage, current int64) bool { @@ -127,6 +129,22 @@ func validateSendNowSnapshot(selected []*QueuedMessage, expected []QueuedMessage return nil } +func bindSendNowLifecycleReservations( + sources []QueuedMessage, + identity QueueSessionIdentity, +) { + if identity.SessionIncarnationID == "" { + return + } + for index := range sources { + if !sources[index].IsDurableLifecycle() { + continue + } + sources[index].reservedLifecycleDelivery = true + sources[index].reservationIdentity = identity + } +} + // BuildSendNowEnvelope validates and combines an exact, FIFO-ordered source // snapshot. It performs no repository mutation, which lets callers validate a // bulk selection before interrupting an active turn. diff --git a/apps/backend/internal/orchestrator/messagequeue/service.go b/apps/backend/internal/orchestrator/messagequeue/service.go index e62774695b..431fc610c9 100644 --- a/apps/backend/internal/orchestrator/messagequeue/service.go +++ b/apps/backend/internal/orchestrator/messagequeue/service.go @@ -17,19 +17,26 @@ import ( "sync/atomic" "time" + "github.com/google/uuid" "github.com/kandev/kandev/internal/common/logger" "go.uber.org/zap" ) +type autoMergePolicyLoader struct { + load func(context.Context) (bool, int64, error) +} + // Service manages queued messages for sessions, backed by Repository. type Service struct { - repo Repository - maxPerSession atomic.Int64 - mergeEnabled atomic.Bool - autoMergeEnabled atomic.Bool - logger *logger.Logger - admissionMu sync.Mutex - admissions map[string]*sessionAdmission + repo Repository + maxPerSession atomic.Int64 + mergeEnabled atomic.Bool + autoMergePolicy atomic.Pointer[AutoMergePolicy] + autoMergeLoader atomic.Pointer[autoMergePolicyLoader] + statusEpoch string + logger *logger.Logger + admissionMu sync.Mutex + admissions map[string]*sessionAdmission } type sessionAdmission struct { @@ -44,18 +51,53 @@ type sessionAdmissionToken struct { sessionID string } +type queueAttachmentRepository interface { + InsertForSessionWithClaim(context.Context, QueueSessionIdentity, *QueuedMessage, QueueAttachmentClaim, int) error + AutoMergeCandidateIntoAboveForSessionWithClaim(context.Context, QueueSessionIdentity, *QueuedMessage, QueueAttachmentClaim) (*QueuedMessage, bool, error) + UpdateContentAndMetadataForSessionWithClaim(context.Context, QueueSessionIdentity, string, string, []MessageAttachment, map[string]interface{}, string, QueueAttachmentClaim) error +} + +type deletedSessionRepository interface { + PurgeDeletedSession(context.Context, QueueSessionIdentity) error +} + +type autoMergeAdmissionRepository interface { + InsertForSessionWithPolicy( + context.Context, + QueueSessionIdentity, + *QueuedMessage, + *QueueAttachmentClaim, + int, + AutoMergePolicy, + ) error + AutoMergeCandidateIntoAboveForSessionWithPolicy( + context.Context, + QueueSessionIdentity, + *QueuedMessage, + *QueueAttachmentClaim, + AutoMergePolicy, + ) (*QueuedMessage, bool, error) + AutoMergeIntoAboveForSessionWithPolicy( + context.Context, + QueueSessionIdentity, + string, + AutoMergePolicy, + ) (*QueuedMessage, bool, error) +} + // NewService creates a Service backed by the supplied repository. maxPerSession // is the per-session cap (entries beyond this return ErrQueueFull on insert); // pass 0 to disable the cap. func NewService(repo Repository, maxPerSession int, log *logger.Logger) *Service { service := &Service{ - repo: repo, - logger: log.WithFields(zap.String("component", "message-queue")), - admissions: make(map[string]*sessionAdmission), + repo: repo, + statusEpoch: uuid.NewString(), + logger: log.WithFields(zap.String("component", "message-queue")), + admissions: make(map[string]*sessionAdmission), } service.SetMaxPerSession(maxPerSession) service.mergeEnabled.Store(true) - service.autoMergeEnabled.Store(true) + service.SetAutoMergePolicy(true, 0) return service } @@ -65,6 +107,19 @@ func NewServiceMemory(log *logger.Logger) *Service { return NewService(NewMemoryRepository(), DefaultMaxPerSession, log) } +// ResolveSessionIdentity returns the repository-authoritative immutable queue +// identity for trusted internal callers migrating legacy session snapshots. +func (s *Service) ResolveSessionIdentity(ctx context.Context, taskID, sessionID string) (QueueSessionIdentity, error) { + return s.repo.ResolveSessionIdentity(ctx, taskID, sessionID) +} + +// SupportsAtomicDeferredMoveTransition reports whether task and queue rows share +// one SQL transaction. The in-memory queue cannot participate in task storage. +func (s *Service) SupportsAtomicDeferredMoveTransition() bool { + _, ok := s.repo.(*sqliteRepository) + return ok +} + // MaxPerSession returns the configured per-session cap. func (s *Service) MaxPerSession() int { return int(s.maxPerSession.Load()) } @@ -101,14 +156,105 @@ func (s *Service) SetMergeEnabled(enabled bool) { s.mergeEnabled.Store(enabled) } -// AutoMergeEnabled reports whether ordinary newly admitted messages may be -// folded into a compatible queue tail. It is independent from manual merge. -func (s *Service) AutoMergeEnabled() bool { return s.autoMergeEnabled.Load() } +// AutoMergeEnabled reports the global admission policy used by sessions that +// do not have an explicit override. +func (s *Service) AutoMergeEnabled() bool { + return s.globalAutoMergePolicy().Enabled +} -// SetAutoMergeEnabled toggles admission-time automatic merging. Existing rows -// are never compacted retroactively. +// SetAutoMergeEnabled updates the global policy while preserving the legacy +// boolean target used by callers that do not own the persisted revision. func (s *Service) SetAutoMergeEnabled(enabled bool) { - s.autoMergeEnabled.Store(enabled) + current := s.globalAutoMergePolicy() + revision := current.Revision + if current.Enabled != enabled { + revision++ + } + s.SetAutoMergePolicy(enabled, revision) +} + +// SetAutoMergePolicy atomically publishes a complete global policy snapshot. +func (s *Service) SetAutoMergePolicy(enabled bool, revision int64) { + policy := &AutoMergePolicy{ + Enabled: enabled, + Source: AutoMergeSourceGlobal, + Revision: revision, + } + s.autoMergePolicy.Store(policy) +} + +// SetAutoMergePolicyLoader installs the shared persisted-policy reader used by +// identity-bound admissions and snapshots. The atomic cache remains available +// to legacy callers that cannot carry a context. +func (s *Service) SetAutoMergePolicyLoader( + load func(context.Context) (bool, int64, error), +) { + if load == nil { + s.autoMergeLoader.Store(nil) + return + } + s.autoMergeLoader.Store(&autoMergePolicyLoader{load: load}) +} + +func (s *Service) globalAutoMergePolicy() AutoMergePolicy { + policy := s.autoMergePolicy.Load() + if policy == nil { + return AutoMergePolicy{Enabled: true, Source: AutoMergeSourceGlobal} + } + return *policy +} + +// ResolveAutoMergePolicy returns the session override when present, otherwise +// the current global policy snapshot. +func (s *Service) ResolveAutoMergePolicy( + ctx context.Context, + identity QueueSessionIdentity, +) (AutoMergePolicy, error) { + override, err := s.repo.GetAutoMergeOverride(ctx, identity) + if err != nil { + return AutoMergePolicy{}, err + } + if override == nil { + return s.loadGlobalAutoMergePolicy(ctx) + } + return AutoMergePolicy{ + Enabled: override.Enabled, + Source: AutoMergeSourceSession, + Revision: override.Revision, + }, nil +} + +func (s *Service) loadGlobalAutoMergePolicy(ctx context.Context) (AutoMergePolicy, error) { + if loader := s.autoMergeLoader.Load(); loader != nil { + enabled, revision, err := loader.load(ctx) + if err != nil { + return AutoMergePolicy{}, err + } + s.SetAutoMergePolicy(enabled, revision) + } + return s.globalAutoMergePolicy(), nil +} + +// SetSessionAutoMerge persists and returns an explicit per-session policy. +func (s *Service) SetSessionAutoMerge( + ctx context.Context, + identity QueueSessionIdentity, + enabled bool, +) (AutoMergePolicy, error) { + var override AutoMergeOverride + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var setErr error + override, setErr = s.repo.SetAutoMergeOverride(admittedCtx, identity, enabled) + return setErr + }) + if err != nil { + return AutoMergePolicy{}, err + } + return AutoMergePolicy{ + Enabled: override.Enabled, + Source: AutoMergeSourceSession, + Revision: override.Revision, + }, nil } // WithSessionAdmission runs fn under the per-session queue admission lock. @@ -161,8 +307,12 @@ func (s *Service) QueueMessage(ctx context.Context, sessionID, taskID, content, // is propagated to the resulting Message row when the queued message is // drained (e.g. sender_task_id for messages sent via message_task_kandev). func (s *Service) QueueMessageWithMetadata(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}) (*QueuedMessage, error) { + identity, err := s.repo.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + return nil, err + } return s.queueMessageWithMetadataAdmission( - ctx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, nil, + ctx, &identity, sessionID, taskID, content, model, userID, planMode, attachments, metadata, nil, nil, ) } @@ -171,91 +321,207 @@ func (s *Service) QueueMessageWithMetadata(ctx context.Context, sessionID, taskI // the snapshotted automatic fold. Callers use the hook to claim staged // attachments before folding; if it returns an error, no fold is attempted. func (s *Service) QueueMessageWithMetadataAfterInsert(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, afterInsert func(context.Context, *QueuedMessage) error) (*QueuedMessage, error) { + identity, err := s.repo.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + return nil, err + } return s.queueMessageWithMetadataAdmission( - ctx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, afterInsert, + ctx, &identity, sessionID, taskID, content, model, userID, planMode, attachments, metadata, nil, afterInsert, ) } -// queueMessageWithMetadataAdmission snapshots policy and completes admission -// under one per-session lock. -func (s *Service) queueMessageWithMetadataAdmission(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, afterInsert func(context.Context, *QueuedMessage) error) (*QueuedMessage, error) { +// QueueMessageWithMetadataForSession admits a browser message using the +// effective policy bound to the supplied immutable session identity. +func (s *Service) QueueMessageWithMetadataForSession( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, +) (*QueuedMessage, error) { + return s.queueMessageWithMetadataAdmission( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, attachments, metadata, nil, nil, + ) +} + +// QueueMessageWithMetadataForSessionWithClaim atomically claims staged +// attachments when admission inserts a queue entry. An empty claim carries +// inline-only attachments and may use the compatible full-queue fold path. +func (s *Service) QueueMessageWithMetadataForSessionWithClaim( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + claim QueueAttachmentClaim, +) (*QueuedMessage, error) { + return s.queueMessageWithMetadataAdmission( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, attachments, metadata, &claim, nil, + ) +} + +// QueueMessageWithMetadataForSessionAfterInsert is the identity-bound variant +// used while attachment claiming still participates in the admission lock. +func (s *Service) QueueMessageWithMetadataForSessionAfterInsert( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + afterInsert func(context.Context, *QueuedMessage) error, +) (*QueuedMessage, error) { + return s.queueMessageWithMetadataAdmission( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, attachments, metadata, nil, afterInsert, + ) +} + +// queueMessageWithMetadataAdmission resolves policy and completes admission +// under one per-session lock. Identity-bound repositories compare that policy +// inside every durable mutation; an override race restarts before admission. +// +//nolint:gocognit // retry boundaries keep policy validation before each mutation. +func (s *Service) queueMessageWithMetadataAdmission(ctx context.Context, identity *QueueSessionIdentity, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, claim *QueueAttachmentClaim, afterInsert func(context.Context, *QueuedMessage) error) (*QueuedMessage, error) { var queued *QueuedMessage err := s.WithSessionAdmission(ctx, sessionID, func(admittedCtx context.Context) error { - maxPerSession := s.MaxPerSession() - autoMergeEnabled := s.AutoMergeEnabled() - source, err := s.insertQueueMessageWithMetadata( - admittedCtx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, maxPerSession, - ) - if err != nil { - if errors.Is(err, ErrQueueFull) && autoMergeEnabled && afterInsert == nil { - // A full queue must still accept a message that would fold into - // the tail: admission-time auto-merge runs after a successful - // insert, so at capacity it could never fire. Fold the - // candidate directly — the fold is the admission. The - // afterInsert hook is excluded because it claims staged - // attachments against a persisted source row, which this path - // never creates. - candidate := &QueuedMessage{ - SessionID: sessionID, - TaskID: taskID, - Content: content, - Model: model, - PlanMode: planMode, - Attachments: attachments, - Metadata: copyMessageMetadata(metadata, 0), - QueuedAt: time.Now().UTC(), - QueuedBy: userID, + for { + policy := s.resolveAdmissionAutoMergePolicy(admittedCtx, identity, sessionID) + source, insertErr := s.insertQueueMessageWithMetadata( + admittedCtx, identity, sessionID, taskID, content, model, userID, planMode, + attachments, metadata, claim, s.MaxPerSession(), policy, + ) + if errors.Is(insertErr, ErrAutoMergePolicyChanged) { + continue + } + autoMergeEnabled := policy != nil && policy.Enabled + if insertErr != nil { + hasStagedAttachments := claim != nil && len(claim.IDs) > 0 + if !errors.Is(insertErr, ErrQueueFull) || !autoMergeEnabled || hasStagedAttachments || afterInsert != nil { + return insertErr } - merged, didMerge, mergeErr := s.repo.AutoMergeCandidateIntoAbove(admittedCtx, candidate) - switch { - case mergeErr != nil: - if errors.Is(mergeErr, ErrTaskInactive) { - // The task was archived or deleted while the admission - // was in flight; surface the inactive-task contract - // instead of a misleading queue-full rejection. - return mergeErr - } - s.logger.Error("automatic merge into full queue failed; preserving queue full rejection", - zap.String("session_id", sessionID), - zap.Error(mergeErr)) - return err - case didMerge && merged != nil: - s.logger.Info("automatically merged queued entry into full tail", - zap.String("session_id", sessionID), - zap.String("surviving_entry_id", merged.ID)) + var merged *QueuedMessage + source, merged, insertErr = s.admitQueueFullMessage( + admittedCtx, identity, insertErr, sessionID, taskID, content, model, userID, planMode, + attachments, metadata, claim, s.MaxPerSession(), policy, + ) + if errors.Is(insertErr, ErrAutoMergePolicyChanged) { + continue + } + if insertErr != nil { + return insertErr + } + if merged != nil { queued = merged return nil - default: - // The fold skipped because the tail is absent or changed — - // a concurrent drain freed capacity (or a concurrent fold or - // drain moved the tail) between the failed insert and the - // fold scan. Retry the ordinary insert once under the held - // admission lock: it succeeds now that capacity is free, and - // still returns ErrQueueFull when another writer keeps the - // cap occupied. Without this retry a stale ErrQueueFull - // would drop a message that is admissible at fold time. - source, err = s.insertQueueMessageWithMetadata( - admittedCtx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, maxPerSession, - ) - if err != nil { - return err - } } - } else { - return err } - } - if afterInsert != nil { - if err := afterInsert(admittedCtx, source); err != nil { - return err + if afterInsert != nil { + if err := afterInsert(admittedCtx, source); err != nil { + return err + } } + queued = s.finalizeAutoMerge(admittedCtx, identity, source, policy) + return nil } - queued = s.finalizeAutoMerge(admittedCtx, source, autoMergeEnabled) - return nil }) return queued, err } +func (s *Service) resolveAdmissionAutoMergePolicy( + ctx context.Context, + identity *QueueSessionIdentity, + sessionID string, +) *AutoMergePolicy { + if identity == nil { + policy := s.globalAutoMergePolicy() + return &policy + } + policy, err := s.ResolveAutoMergePolicy(ctx, *identity) + if err == nil { + return &policy + } + s.logger.Warn("automatic merge policy unavailable; admitting separate message", + zap.String("session_id", sessionID), + zap.Error(err)) + return nil +} + +func (s *Service) autoMergeCandidate( + ctx context.Context, + identity *QueueSessionIdentity, + candidate *QueuedMessage, + claim *QueueAttachmentClaim, + policy *AutoMergePolicy, +) (*QueuedMessage, bool, error) { + if identity == nil { + return s.repo.AutoMergeCandidateIntoAbove(ctx, candidate) + } + if policyRepo, ok := s.repo.(autoMergeAdmissionRepository); ok && policy != nil { + return policyRepo.AutoMergeCandidateIntoAboveForSessionWithPolicy(ctx, *identity, candidate, claim, *policy) + } + if claim == nil { + return s.repo.AutoMergeCandidateIntoAboveForSession(ctx, *identity, candidate) + } + claimRepo, ok := s.repo.(queueAttachmentRepository) + if !ok { + return nil, false, errors.New("transactional attachment queue repository is unavailable") + } + return claimRepo.AutoMergeCandidateIntoAboveForSessionWithClaim(ctx, *identity, candidate, *claim) +} + +// admitQueueFullMessage folds a candidate into the tail or retries insertion +// when the fold observes capacity freed by a concurrent drain. +func (s *Service) admitQueueFullMessage( + ctx context.Context, + identity *QueueSessionIdentity, + queueFullErr error, + sessionID, taskID, content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + claim *QueueAttachmentClaim, + maxPerSession int, + policy *AutoMergePolicy, +) (*QueuedMessage, *QueuedMessage, error) { + candidate := &QueuedMessage{ + SessionID: sessionID, + TaskID: taskID, + Content: content, + Model: model, + PlanMode: planMode, + Attachments: attachments, + Metadata: copyMessageMetadata(metadata, 0), + QueuedAt: time.Now().UTC(), + QueuedBy: userID, + } + merged, didMerge, err := s.autoMergeCandidate(ctx, identity, candidate, claim, policy) + if errors.Is(err, ErrAutoMergePolicyChanged) { + return nil, nil, err + } + if err != nil { + if errors.Is(err, ErrTaskInactive) { + return nil, nil, err + } + s.logger.Error("automatic merge into full queue failed; preserving queue full rejection", + zap.String("session_id", sessionID), + zap.Error(err)) + return nil, nil, queueFullErr + } + if didMerge && merged != nil { + s.logger.Info("automatically merged queued entry into full tail", + zap.String("session_id", sessionID), + zap.String("surviving_entry_id", merged.ID)) + return nil, merged, nil + } + source, err := s.insertQueueMessageWithMetadata( + ctx, identity, sessionID, taskID, content, model, userID, planMode, + attachments, metadata, claim, maxPerSession, policy, + ) + return source, nil, err +} + // queueMessageWithMetadataSeparate is used by retry paths whose existing // contract must not gain admission-time automatic behavior. func (s *Service) queueMessageWithMetadataSeparate(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, error) { @@ -263,37 +529,80 @@ func (s *Service) queueMessageWithMetadataSeparate(ctx context.Context, sessionI err := s.WithSessionAdmission(ctx, sessionID, func(admittedCtx context.Context) error { var err error queued, err = s.insertQueueMessageWithMetadata( - admittedCtx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, maxPerSession, + admittedCtx, nil, sessionID, taskID, content, model, userID, planMode, + attachments, metadata, nil, maxPerSession, nil, ) return err }) return queued, err } -func (s *Service) finalizeAutoMerge(ctx context.Context, source *QueuedMessage, enabled bool) *QueuedMessage { - if !enabled { - return source - } - merged, didMerge, err := s.repo.AutoMergeIntoAbove(ctx, source.SessionID, source.ID) - if err != nil { - s.logger.Error("automatic queue merge failed; preserving separate admission", +func (s *Service) finalizeAutoMerge( + ctx context.Context, + identity *QueueSessionIdentity, + source *QueuedMessage, + policy *AutoMergePolicy, +) *QueuedMessage { + for policy != nil && policy.Enabled { + var merged *QueuedMessage + var didMerge bool + var err error + if identity == nil { + merged, didMerge, err = s.repo.AutoMergeIntoAbove(ctx, source.SessionID, source.ID) + } else if policyRepo, ok := s.repo.(autoMergeAdmissionRepository); ok { + merged, didMerge, err = policyRepo.AutoMergeIntoAboveForSessionWithPolicy(ctx, *identity, source.ID, *policy) + } else { + merged, didMerge, err = s.repo.AutoMergeIntoAboveForSession(ctx, *identity, source.ID) + } + if errors.Is(err, ErrAutoMergePolicyChanged) && identity != nil { + policy = s.resolveAdmissionAutoMergePolicy(ctx, identity, source.SessionID) + continue + } + if err != nil { + s.logger.Error("automatic queue merge failed; preserving separate admission", + zap.String("session_id", source.SessionID), + zap.String("source_entry_id", source.ID), + zap.Error(err)) + return source + } + if !didMerge || merged == nil { + return source + } + s.logger.Info("automatically merged queued entry into tail", zap.String("session_id", source.SessionID), zap.String("source_entry_id", source.ID), - zap.Error(err)) - return source + zap.String("surviving_entry_id", merged.ID)) + return merged } - if !didMerge || merged == nil { - return source + return source +} + +func (s *Service) insertQueueMessage( + ctx context.Context, + identity *QueueSessionIdentity, + msg *QueuedMessage, + claim *QueueAttachmentClaim, + maxPerSession int, + policy *AutoMergePolicy, +) error { + if identity == nil { + return s.repo.Insert(ctx, msg, maxPerSession) } - s.logger.Info("automatically merged queued entry into tail", - zap.String("session_id", source.SessionID), - zap.String("source_entry_id", source.ID), - zap.String("surviving_entry_id", merged.ID)) - return merged + if policyRepo, ok := s.repo.(autoMergeAdmissionRepository); ok && policy != nil { + return policyRepo.InsertForSessionWithPolicy(ctx, *identity, msg, claim, maxPerSession, *policy) + } + if claim == nil { + return s.repo.InsertForSession(ctx, *identity, msg, maxPerSession) + } + claimRepo, ok := s.repo.(queueAttachmentRepository) + if !ok { + return errors.New("transactional attachment queue repository is unavailable") + } + return claimRepo.InsertForSessionWithClaim(ctx, *identity, msg, *claim, maxPerSession) } // insertQueueMessageWithMetadata inserts a message with metadata under the per-session admission lock. -func (s *Service) insertQueueMessageWithMetadata(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, maxPerSession int) (*QueuedMessage, error) { +func (s *Service) insertQueueMessageWithMetadata(ctx context.Context, identity *QueueSessionIdentity, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, claim *QueueAttachmentClaim, maxPerSession int, policy *AutoMergePolicy) (*QueuedMessage, error) { metadataCopy := copyMessageMetadata(metadata, 0) msg := &QueuedMessage{ SessionID: sessionID, @@ -305,7 +614,8 @@ func (s *Service) insertQueueMessageWithMetadata(ctx context.Context, sessionID, Metadata: metadataCopy, QueuedBy: userID, } - if err := s.repo.Insert(ctx, msg, maxPerSession); err != nil { + err := s.insertQueueMessage(ctx, identity, msg, claim, maxPerSession, policy) + if err != nil { if errors.Is(err, ErrQueueFull) { s.logger.Info("queue full", zap.String("session_id", sessionID), @@ -338,6 +648,30 @@ func (s *Service) RestoreMessage(ctx context.Context, msg *QueuedMessage) (*Queu return restored, err } +func (s *Service) RestoreMessageForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, +) (*QueuedMessage, error) { + if msg == nil { + return nil, errors.New("queued message is nil") + } + var restored *QueuedMessage + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + candidate := *msg + candidate.Metadata = copyMessageMetadata(msg.Metadata, 0) + if err := s.validateSessionIdentity(admittedCtx, identity); err != nil { + return err + } + if err := s.repo.RestoreForSession(admittedCtx, identity, &candidate, 0); err != nil { + return err + } + restored = &candidate + return nil + }) + return restored, err +} + // restoreMessage is the admission-checked core of RestoreMessage. func (s *Service) restoreMessage(ctx context.Context, msg *QueuedMessage) (*QueuedMessage, error) { restored := *msg @@ -364,6 +698,35 @@ func (s *Service) QueueMessageWithCoalesceKey(ctx context.Context, sessionID, ta ) } +func (s *Service) QueueMessageWithCoalesceKeyForSession( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + coalesceKey string, + allowInsert bool, +) (*QueuedMessage, bool, error) { + var queued *QueuedMessage + var replaced bool + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + metadataCopy := copyMessageMetadata(metadata, 1) + metadataCopy[MetadataCoalesceKey] = coalesceKey + candidate := &QueuedMessage{ + SessionID: identity.SessionID, TaskID: identity.TaskID, + Content: content, Model: model, PlanMode: planMode, + Attachments: attachments, Metadata: metadataCopy, QueuedBy: userID, + } + var err error + queued, replaced, err = s.repo.InsertOrReplaceByCoalesceKeyForSession( + admittedCtx, identity, candidate, coalesceKey, s.MaxPerSession(), allowInsert, + ) + return err + }) + return queued, replaced, err +} + // queueMessageWithCoalesceKey is the admission-checked core of QueueMessageWithCoalesceKey. func (s *Service) queueMessageWithCoalesceKey(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert bool, maxPerSession int) (*QueuedMessage, bool, error) { var queued *QueuedMessage @@ -451,17 +814,47 @@ func (s *Service) RequeueAtHead(ctx context.Context, msg *QueuedMessage) error { }) } +func (s *Service) RequeueAtHeadForSession( + ctx context.Context, + identity QueueSessionIdentity, + msg *QueuedMessage, +) error { + if msg == nil { + return errors.New("queued message is nil") + } + return s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + if err := s.validateSessionIdentity(admittedCtx, identity); err != nil { + return err + } + return s.repo.RequeuePreservingFIFOForSession(admittedCtx, identity, msg) + }) +} + // QueueLifecycleMessageWithCoalesceKey accepts a lifecycle entry only while // its task remains active. accepted is false for a normal archive/delete win. func (s *Service) QueueLifecycleMessageWithCoalesceKey(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert bool) (*QueuedMessage, bool, bool, error) { - return s.queueLifecycleMessageWithCoalesceKey(ctx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, false, nil) + return s.queueLifecycleMessageWithCoalesceKey(ctx, nil, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, false, nil) +} + +func (s *Service) QueueLifecycleMessageWithCoalesceKeyForSession( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + coalesceKey string, + allowInsert bool, +) (*QueuedMessage, bool, bool, error) { + return s.queueLifecycleMessageWithCoalesceKey( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, + attachments, metadata, coalesceKey, allowInsert, false, nil, + ) } // QueueLifecycleMessageWithCoalesceKeyAfterInsert runs afterInsert while the // queue admission lock is held. If the callback fails, the exact queue // snapshot from before admission is restored before the error is returned. -// Lifecycle dispatch uses this boundary when it must reserve durable attempt -// metadata together with a queue entry. func (s *Service) QueueLifecycleMessageWithCoalesceKeyAfterInsert( ctx context.Context, sessionID, taskID, content, model, userID string, @@ -472,21 +865,59 @@ func (s *Service) QueueLifecycleMessageWithCoalesceKeyAfterInsert( allowInsert bool, afterInsert func(context.Context, *QueuedMessage, bool) error, ) (*QueuedMessage, bool, bool, error) { - return s.queueLifecycleMessageWithCoalesceKey(ctx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, false, afterInsert) + return s.queueLifecycleMessageWithCoalesceKey(ctx, nil, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, false, afterInsert) +} + +func (s *Service) QueueLifecycleMessageWithCoalesceKeyForSessionAfterInsert( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + coalesceKey string, + allowInsert bool, + afterInsert func(context.Context, *QueuedMessage, bool) error, +) (*QueuedMessage, bool, bool, error) { + return s.queueLifecycleMessageWithCoalesceKey( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, + attachments, metadata, coalesceKey, allowInsert, false, afterInsert, + ) } // RequeueLifecycleMessageWithCoalesceKey preserves the generation captured by // an existing durable row. This prevents a stale retry from becoming a new // prompt after the task was archived and then unarchived. func (s *Service) RequeueLifecycleMessageWithCoalesceKey(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert bool) (*QueuedMessage, bool, bool, error) { - return s.queueLifecycleMessageWithCoalesceKey(ctx, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, true, nil) + return s.queueLifecycleMessageWithCoalesceKey(ctx, nil, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, true, nil) +} + +func (s *Service) RequeueLifecycleMessageWithCoalesceKeyForSession( + ctx context.Context, + identity QueueSessionIdentity, + content, model, userID string, + planMode bool, + attachments []MessageAttachment, + metadata map[string]interface{}, + coalesceKey string, + allowInsert bool, +) (*QueuedMessage, bool, bool, error) { + return s.queueLifecycleMessageWithCoalesceKey( + ctx, &identity, identity.SessionID, identity.TaskID, content, model, userID, planMode, + attachments, metadata, coalesceKey, allowInsert, true, nil, + ) } // queueLifecycleMessageWithCoalesceKey is the lifecycle-guarded core of the lifecycle queue methods. -func (s *Service) queueLifecycleMessageWithCoalesceKey(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert, isRetry bool, afterInsert func(context.Context, *QueuedMessage, bool) error) (*QueuedMessage, bool, bool, error) { +func (s *Service) queueLifecycleMessageWithCoalesceKey(ctx context.Context, identity *QueueSessionIdentity, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert, isRetry bool, afterInsert func(context.Context, *QueuedMessage, bool) error) (*QueuedMessage, bool, bool, error) { var queued *QueuedMessage var replaced, accepted bool err := s.WithSessionAdmission(ctx, sessionID, func(admittedCtx context.Context) error { + if identity != nil { + if err := s.validateSessionIdentity(admittedCtx, *identity); err != nil { + return err + } + } var snapshot []QueuedMessage var pendingMove *PendingMove if afterInsert != nil { @@ -502,7 +933,7 @@ func (s *Service) queueLifecycleMessageWithCoalesceKey(ctx context.Context, sess } var err error queued, replaced, accepted, err = s.insertLifecycleMessageWithCoalesceKey( - admittedCtx, sessionID, taskID, content, model, userID, planMode, attachments, + admittedCtx, identity, sessionID, taskID, content, model, userID, planMode, attachments, metadata, coalesceKey, allowInsert, isRetry, ) if err != nil || !accepted || afterInsert == nil { @@ -533,7 +964,7 @@ func (s *Service) restoreCoalescedAdmission(ctx context.Context, sessionID strin } // insertLifecycleMessageWithCoalesceKey inserts or replaces a lifecycle entry under the admission lock. -func (s *Service) insertLifecycleMessageWithCoalesceKey(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert, isRetry bool) (*QueuedMessage, bool, bool, error) { +func (s *Service) insertLifecycleMessageWithCoalesceKey(ctx context.Context, identity *QueueSessionIdentity, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment, metadata map[string]interface{}, coalesceKey string, allowInsert, isRetry bool) (*QueuedMessage, bool, bool, error) { metadataCopy := clearReservedMetadata(metadata) generation, err := s.repo.LifecycleGeneration(ctx, taskID) if err != nil { @@ -559,7 +990,13 @@ func (s *Service) insertLifecycleMessageWithCoalesceKey(ctx context.Context, ses if isRetry { maxPerSession = 0 } - queued, replaced, err := s.repo.InsertOrReplaceLifecycleByCoalesceKey(ctx, msg, coalesceKey, maxPerSession, allowInsert) + var queued *QueuedMessage + var replaced bool + if identity == nil { + queued, replaced, err = s.repo.InsertOrReplaceLifecycleByCoalesceKey(ctx, msg, coalesceKey, maxPerSession, allowInsert) + } else { + queued, replaced, err = s.repo.InsertOrReplaceLifecycleByCoalesceKeyForSession(ctx, *identity, msg, coalesceKey, maxPerSession, allowInsert) + } if errors.Is(err, ErrTaskInactive) || errors.Is(err, ErrLifecycleCancelled) { return nil, false, false, nil } @@ -622,6 +1059,40 @@ func (s *Service) ReserveQueuedWithAutoRun(ctx context.Context, sessionID string return msg, msg != nil, autoRun } +// ReserveQueuedWithAutoRunForSession reserves the FIFO head only for the exact session incarnation. +func (s *Service) ReserveQueuedWithAutoRunForSession(ctx context.Context, identity QueueSessionIdentity) (*QueuedMessage, bool, bool, error) { + var msg *QueuedMessage + var autoRun bool + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + msg, autoRun, err = s.repo.ReserveHeadIfAutoRunForSession(admittedCtx, identity) + return err + }) + if err != nil { + return nil, false, true, err + } + return msg, msg != nil, autoRun, nil +} + +// ReserveQueuedForDeliveryWithAutoRunForSession retains the FIFO head until +// explicit acknowledgement, regardless of producer type. +func (s *Service) ReserveQueuedForDeliveryWithAutoRunForSession( + ctx context.Context, + identity QueueSessionIdentity, +) (*QueuedMessage, bool, bool, error) { + var msg *QueuedMessage + var autoRun bool + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + msg, autoRun, err = s.repo.ReserveHeadForDeliveryIfAutoRunForSession(admittedCtx, identity) + return err + }) + if err != nil { + return nil, false, true, err + } + return msg, msg != nil, autoRun, nil +} + // SetAutoRun persists automatic-drain policy through the queue admission gate. func (s *Service) SetAutoRun(ctx context.Context, sessionID string, enabled bool) error { return s.WithSessionAdmission(ctx, sessionID, func(admittedCtx context.Context) error { @@ -629,6 +1100,13 @@ func (s *Service) SetAutoRun(ctx context.Context, sessionID string, enabled bool }) } +// SetAutoRunForSession persists automatic-drain policy only for the exact session incarnation. +func (s *Service) SetAutoRunForSession(ctx context.Context, identity QueueSessionIdentity, enabled bool) error { + return s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return s.repo.SetAutoRunForSession(admittedCtx, identity, enabled) + }) +} + // PauseAutoRunIfPending atomically parks a visible backlog, if one exists. func (s *Service) PauseAutoRunIfPending(ctx context.Context, sessionID string) (bool, error) { var paused bool @@ -639,6 +1117,18 @@ func (s *Service) PauseAutoRunIfPending(ctx context.Context, sessionID string) ( }) return paused, err } +func (s *Service) PauseAutoRunIfPendingForSession( + ctx context.Context, + identity QueueSessionIdentity, +) (bool, error) { + var paused bool + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + paused, err = s.repo.PauseAutoRunIfPendingForSession(admittedCtx, identity) + return err + }) + return paused, err +} // AcknowledgeQueued removes a server-reserved entry after prompt acceptance. func (s *Service) AcknowledgeQueued(ctx context.Context, sessionID, entryID string) error { @@ -651,6 +1141,56 @@ func (s *Service) AcknowledgeQueued(ctx context.Context, sessionID, entryID stri return err } +func (s *Service) AcknowledgeQueuedForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + if err := s.validateSessionIdentity(admittedCtx, identity); err != nil { + return err + } + return s.repo.AcknowledgeByIDForSession(admittedCtx, identity, entryID) + }) + if errors.Is(err, ErrEntryNotFound) { + return nil + } + return err +} + +// ReleaseQueuedDeliveryForSession makes an unaccepted retained entry visible +// again for the same session incarnation. +func (s *Service) ReleaseQueuedDeliveryForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return s.repo.ReleaseDeliveryReservationForSession(admittedCtx, identity, entryID) + }) + if errors.Is(err, ErrEntryNotFound) { + return nil + } + return err +} + +// DiscardLifecycleReservation removes a reservation still owned by a replaced +// session incarnation. The repository compares persisted ownership, so this +// cannot delete a reservation subsequently claimed by another incarnation. +func (s *Service) DiscardLifecycleReservation( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) error { + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return s.repo.DiscardLifecycleReservation(admittedCtx, identity, entryID) + }) + if errors.Is(err, ErrEntryNotFound) { + return nil + } + return err +} + // IsCurrentLifecycleReservation verifies that msg is still the durable row // accepted for the task's current lifecycle generation. A task archive/delete // removes that row and advances the generation, so callers can discard a @@ -667,15 +1207,30 @@ func (s *Service) IsCurrentLifecycleReservation(ctx context.Context, msg *Queued if err != nil || generation != expectedGeneration { return false } + if msg.reservationIdentity.SessionIncarnationID != "" { + current, err := s.repo.ResolveSessionIdentity( + ctx, + msg.reservationIdentity.TaskID, + msg.reservationIdentity.SessionID, + ) + if err != nil || current != msg.reservationIdentity { + return false + } + } entries, err := s.repo.ListBySession(ctx, msg.SessionID) if err != nil { return false } for i := range entries { entry := &entries[i] - if entry.ID == msg.ID && entry.TaskID == msg.TaskID && entry.IsDurableLifecycle() { - return true + if entry.ID != msg.ID || entry.TaskID != msg.TaskID || !entry.IsDurableLifecycle() { + continue } + if msg.reservationIdentity.SessionIncarnationID != "" && + lifecycleReservationIncarnation(entry.Metadata) != msg.reservationIdentity.SessionIncarnationID { + return false + } + return true } return false } @@ -696,18 +1251,27 @@ func copyMessageMetadata(metadata map[string]interface{}, extraCapacity int) map // queued_by matches userID. Otherwise inserts a new entry. Returns // ErrQueueFull when an insert would exceed the cap. func (s *Service) AppendContent(ctx context.Context, sessionID, taskID, content, model, userID string, planMode bool, attachments []MessageAttachment) (*QueuedMessage, bool, error) { + identity, err := s.repo.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + return nil, false, err + } + return s.AppendContentForSession(ctx, identity, content, model, userID, planMode, attachments) +} + +// AppendContentForSession admits content only for the exact session incarnation. +func (s *Service) AppendContentForSession(ctx context.Context, identity QueueSessionIdentity, content, model, userID string, planMode bool, attachments []MessageAttachment) (*QueuedMessage, bool, error) { var msg *QueuedMessage var appended bool - err := s.WithSessionAdmission(ctx, sessionID, func(admittedCtx context.Context) error { + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { var err error - msg, appended, err = s.repo.AppendOrInsertTail(admittedCtx, sessionID, taskID, content, model, userID, planMode, attachments, nil, s.MaxPerSession()) + msg, appended, err = s.repo.AppendOrInsertTailForSession(admittedCtx, identity, content, model, userID, planMode, attachments, nil, s.MaxPerSession()) return err }) if err != nil { return nil, false, err } s.logger.Info("queue append-or-insert", - zap.String("session_id", sessionID), + zap.String("session_id", identity.SessionID), zap.String("entry_id", msg.ID), zap.Bool("appended", appended)) return msg, appended, nil @@ -800,6 +1364,33 @@ func (s *Service) TakeQueuedEntry(ctx context.Context, sessionID, entryID string return msg, true, nil } +func (s *Service) TakeQueuedEntryForSession( + ctx context.Context, + identity QueueSessionIdentity, + entryID string, +) (*QueuedMessage, bool, error) { + var msg *QueuedMessage + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + msg, err = s.repo.TakeByIDForSession(admittedCtx, identity, entryID) + return err + }) + if err != nil { + s.logger.Error("take by id for session failed", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", entryID), + zap.Error(err)) + return nil, false, err + } + if msg == nil { + return nil, false, nil + } + s.logger.Info("message dequeued by id", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", msg.ID)) + return msg, true, nil +} + // UpdateMessage replaces the content (and optionally attachments) of a queued // entry. The sessionID scope is mandatory — callers can't update an entry by // guessing its UUID across sessions. Returns ErrEntryNotFound when the entry @@ -847,6 +1438,23 @@ func (s *Service) ClaimSendNow(ctx context.Context, sessionID string, expected [ return claim, nil } +// ClaimSendNowForSession claims the exact pending snapshot only for the exact session incarnation. +func (s *Service) ClaimSendNowForSession(ctx context.Context, identity QueueSessionIdentity, expected []QueuedMessage) (*SendNowClaim, error) { + var claim *SendNowClaim + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + claim, err = s.repo.ClaimSendNowForSession(admittedCtx, identity, expected) + return err + }) + if err != nil { + return nil, err + } + s.logger.Info("claimed queued messages for send now", + zap.String("session_id", identity.SessionID), + zap.Int("source_count", len(claim.Sources))) + return claim, nil +} + // RestoreSendNowClaim restores every source from an interrupted replacement // dispatch. Durable lifecycle reservations are cleared as part of the same // repository operation. @@ -877,6 +1485,22 @@ func (s *Service) AcknowledgeSendNowClaim(ctx context.Context, claim *SendNowCla return nil } +// GetEntryForSession returns an entry from an identity-bound repository +// snapshot, preventing edits from observing a reincarnated session. +func (s *Service) GetEntryForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) (*QueuedMessage, error) { + snapshot, err := s.repo.Snapshot(ctx, identity) + if err != nil { + return nil, err + } + for index := range snapshot.Entries { + if snapshot.Entries[index].ID == entryID { + entry := snapshot.Entries[index] + return &entry, nil + } + } + return nil, ErrEntryNotFound +} + // UpdateMessageWithMetadata atomically edits queue content and applies // metadata replacements while retaining unrelated metadata keys. func (s *Service) UpdateMessageWithMetadata(ctx context.Context, sessionID, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { @@ -891,6 +1515,37 @@ func (s *Service) UpdateMessageWithMetadata(ctx context.Context, sessionID, entr return nil } +// UpdateMessageWithMetadataForSession edits content only for the exact session incarnation. +func (s *Service) UpdateMessageWithMetadataForSession(ctx context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string) error { + if err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return s.repo.UpdateContentAndMetadataForSession(admittedCtx, identity, entryID, content, attachments, metadataUpdates, queuedBy) + }); err != nil { + return err + } + s.logger.Info("queued entry updated", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", entryID)) + return nil +} + +// UpdateMessageWithMetadataForSessionWithClaim claims newly referenced staged +// attachments and updates the exact queue row atomically. +func (s *Service) UpdateMessageWithMetadataForSessionWithClaim(ctx context.Context, identity QueueSessionIdentity, entryID, content string, attachments []MessageAttachment, metadataUpdates map[string]interface{}, queuedBy string, claim QueueAttachmentClaim) error { + if err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + claimRepo, ok := s.repo.(queueAttachmentRepository) + if !ok { + return errors.New("transactional attachment queue repository is unavailable") + } + return claimRepo.UpdateContentAndMetadataForSessionWithClaim(admittedCtx, identity, entryID, content, attachments, metadataUpdates, queuedBy, claim) + }); err != nil { + return err + } + s.logger.Info("queued entry updated", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", entryID)) + return nil +} + // RemoveEntry deletes a single entry. The sessionID scope is mandatory — see // the rationale on the Repository.DeleteByID contract for why. Returns // ErrEntryNotFound when no entry matches. @@ -906,6 +1561,24 @@ func (s *Service) RemoveEntry(ctx context.Context, sessionID, entryID string) er return nil } +// RemoveEntryForSession deletes an entry only for the exact session incarnation +// and returns the removed entry plus every retained row from the same transaction. +func (s *Service) RemoveEntryForSession(ctx context.Context, identity QueueSessionIdentity, entryID string) (*QueueRemovalResult, error) { + var result *QueueRemovalResult + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + result, err = s.repo.DeleteByIDForSession(admittedCtx, identity, entryID) + return err + }) + if err != nil { + return nil, err + } + s.logger.Info("queued entry removed", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", entryID)) + return result, nil +} + // MergeIntoAbove folds the entry identified by entryID into the entry directly // above it within the same session. See Repository.MergeIntoAbove for the merge // rules and error mapping (ErrEntryNotFound / ErrNoMergeTarget). @@ -928,6 +1601,26 @@ func (s *Service) MergeIntoAbove(ctx context.Context, sessionID, entryID, queued return merged, nil } +// MergeIntoAboveForSession merges an entry only for the exact session incarnation. +func (s *Service) MergeIntoAboveForSession(ctx context.Context, identity QueueSessionIdentity, entryID, queuedBy string) (*QueuedMessage, error) { + if !s.MergeEnabled() { + return nil, ErrMergeDisabled + } + var merged *QueuedMessage + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + merged, err = s.repo.MergeIntoAboveForSession(admittedCtx, identity, entryID, queuedBy) + return err + }) + if err != nil { + return nil, err + } + s.logger.Info("queued entry merged into entry above", + zap.String("session_id", identity.SessionID), + zap.String("entry_id", merged.ID)) + return merged, nil +} + // ReorderEntries rewrites the pending FIFO order of a session's queue to // match orderedIDs. Returns ErrQueueChanged when the submitted order is empty, // contains duplicates, or does not match the current visible pending set (a @@ -948,6 +1641,22 @@ func (s *Service) ReorderEntries(ctx context.Context, sessionID string, orderedI return nil } +// ReorderEntriesForSession reorders pending entries only for the exact session incarnation. +func (s *Service) ReorderEntriesForSession(ctx context.Context, identity QueueSessionIdentity, orderedIDs []string) error { + if err := validateReorderInput(orderedIDs); err != nil { + return err + } + if err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return s.repo.ReorderEntriesForSession(admittedCtx, identity, orderedIDs) + }); err != nil { + return err + } + s.logger.Info("queued entries reordered", + zap.String("session_id", identity.SessionID), + zap.Int("entry_count", len(orderedIDs))) + return nil +} + // CancelAll clears every queued entry for a session. Returns the number of // rows removed. func (s *Service) CancelAll(ctx context.Context, sessionID string) (int, error) { @@ -966,6 +1675,36 @@ func (s *Service) CancelAll(ctx context.Context, sessionID string) (int, error) return n, nil } +// CancelAllForSession clears pending entries only for the exact session +// incarnation and returns both removed and retained rows atomically. +func (s *Service) CancelAllForSession(ctx context.Context, identity QueueSessionIdentity) (*QueueRemovalResult, error) { + var result *QueueRemovalResult + err := s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + var err error + result, err = s.repo.DeleteAllBySessionForIdentity(admittedCtx, identity) + return err + }) + if err != nil { + return nil, err + } + s.logger.Info("queue cancel-all", + zap.String("session_id", identity.SessionID), + zap.Int("removed", len(result.Removed))) + return result, nil +} + +// PurgeDeletedSession removes all ephemeral queue state after the authoritative +// task-session row is gone. It never matches a replacement incarnation. +func (s *Service) PurgeDeletedSession(ctx context.Context, identity QueueSessionIdentity) error { + purger, ok := s.repo.(deletedSessionRepository) + if !ok { + return nil + } + return s.WithSessionAdmission(ctx, identity.SessionID, func(admittedCtx context.Context) error { + return purger.PurgeDeletedSession(admittedCtx, identity) + }) +} + // GetStatus returns the full pending list and capacity info for a session. func (s *Service) GetStatus(ctx context.Context, sessionID string) *QueueStatus { maxPerSession := s.MaxPerSession() @@ -1003,6 +1742,58 @@ func (s *Service) GetStatus(ctx context.Context, sessionID string) *QueueStatus } } +// Snapshot returns an ordered status bound to one immutable session identity. +func (s *Service) Snapshot(ctx context.Context, identity QueueSessionIdentity) (*QueueStatus, error) { + snapshot, err := s.repo.Snapshot(ctx, identity) + if err != nil { + return nil, err + } + pending := make([]QueuedMessage, 0, len(snapshot.Entries)) + for _, entry := range snapshot.Entries { + if !entry.IsReservedInFlight() { + pending = append(pending, entry) + } + } + status := &QueueStatus{ + Entries: pending, + Count: len(pending), + Max: s.MaxPerSession(), + TaskID: identity.TaskID, + SessionID: identity.SessionID, + SessionIncarnationID: identity.SessionIncarnationID, + StatusEpoch: s.statusEpoch, + StatusGeneration: snapshot.StatusGeneration, + AutoRun: snapshot.AutoRun, + MergeEnabled: s.MergeEnabled(), + } + if snapshot.AutoMergeError != nil { + s.logger.Warn("automatic merge policy unavailable while reading queue status", + zap.String("session_id", identity.SessionID), + zap.Error(snapshot.AutoMergeError)) + return status, nil + } + var policy AutoMergePolicy + if snapshot.AutoMergeOverride != nil { + policy = AutoMergePolicy{ + Enabled: snapshot.AutoMergeOverride.Enabled, Source: AutoMergeSourceSession, + Revision: snapshot.AutoMergeOverride.Revision, + } + } else { + policy, err = s.loadGlobalAutoMergePolicy(ctx) + if err != nil { + s.logger.Warn("global automatic merge policy unavailable while reading queue status", + zap.String("session_id", identity.SessionID), + zap.Error(err)) + return status, nil + } + } + status.AutoMergeAvailable = true + status.AutoMergeEnabled = &policy.Enabled + status.AutoMergeSource = policy.Source + status.AutoMergeRevision = &policy.Revision + return status, nil +} + // CountPendingByTaskIDs returns the pending prompt count per task, keyed by // task_id, for every requested task. Reserved in-flight lifecycle rows are // excluded, matching GetStatus. Used by task-list assembly and the status @@ -1042,6 +1833,15 @@ func (s *Service) SnapshotSession(ctx context.Context, sessionID string) ([]Queu return entries, move, nil } +// SnapshotSessionForIdentity captures queue and deferred-move state for one exact incarnation. +func (s *Service) SnapshotSessionForIdentity(ctx context.Context, identity QueueSessionIdentity) ([]QueuedMessage, *PendingMove, error) { + snapshot, err := s.repo.Snapshot(ctx, identity) + if err != nil { + return nil, nil, fmt.Errorf("snapshot session queue: %w", err) + } + return snapshot.Entries, snapshot.PendingMove, nil +} + // TransferSession moves any queued messages and pending move from one session // to another. Used by workflow session switches. Returns an error so the // caller can fail closed instead of silently leaving entries orphaned on the @@ -1061,6 +1861,41 @@ func (s *Service) TransferSession(ctx context.Context, oldSessionID, newSessionI return nil } +func (s *Service) validateSessionIdentity(ctx context.Context, identity QueueSessionIdentity) error { + current, err := s.repo.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + if err != nil { + return err + } + if current != identity { + return ErrSessionIdentityMismatch + } + return nil +} + +// TransferSessionIdentities moves queue state between exact live incarnations of the same task. +func (s *Service) TransferSessionIdentities(ctx context.Context, source, destination QueueSessionIdentity) error { + if source.TaskID != destination.TaskID { + return ErrSessionIdentityMismatch + } + if err := s.validateSessionIdentity(ctx, source); err != nil { + return err + } + if err := s.validateSessionIdentity(ctx, destination); err != nil { + return err + } + if err := s.repo.TransferSessionIdentities(ctx, source, destination); err != nil { + s.logger.Error("transfer session failed", + zap.String("from_session_id", source.SessionID), + zap.String("to_session_id", destination.SessionID), + zap.Error(err)) + return err + } + s.logger.Info("transferred queue between sessions", + zap.String("from_session_id", source.SessionID), + zap.String("to_session_id", destination.SessionID)) + return nil +} + // RestoreSession replaces a session's queue and pending move from a snapshot, // preserving queued-message identity fields. func (s *Service) RestoreSession(ctx context.Context, sessionID string, entries []QueuedMessage, pendingMove *PendingMove) error { @@ -1077,19 +1912,38 @@ func (s *Service) RestoreSession(ctx context.Context, sessionID string, entries return nil } +// RestoreSessionForIdentity replaces queue state only for the captured session incarnation. +func (s *Service) RestoreSessionForIdentity(ctx context.Context, identity QueueSessionIdentity, entries []QueuedMessage, pendingMove *PendingMove) error { + if err := s.validateSessionIdentity(ctx, identity); err != nil { + return err + } + if err := s.repo.ReplaceSessionForIdentity(ctx, identity, entries, pendingMove); err != nil { + s.logger.Error("restore session queue failed", + zap.String("session_id", identity.SessionID), + zap.Error(err)) + return err + } + s.logger.Info("restored session queue", + zap.String("session_id", identity.SessionID), + zap.Int("entries", len(entries)), + zap.Bool("pending_move", pendingMove != nil)) + return nil +} + // SetPendingMove records a pending move for a session (replaces any existing one). // The move is applied by handleAgentReady when the agent's current turn completes. -func (s *Service) SetPendingMove(ctx context.Context, sessionID string, move *PendingMove) { +func (s *Service) SetPendingMove(ctx context.Context, sessionID string, move *PendingMove) error { if err := s.repo.SetPendingMove(ctx, sessionID, move); err != nil { s.logger.Error("set pending move failed", zap.String("session_id", sessionID), zap.Error(err)) - return + return err } s.logger.Info("pending move recorded", zap.String("session_id", sessionID), zap.String("task_id", move.TaskID), zap.String("workflow_step_id", move.WorkflowStepID)) + return nil } // GetPendingMoveWithError retrieves the pending move for a session without diff --git a/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_policy_test.go b/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_policy_test.go new file mode 100644 index 0000000000..063bcb9fd5 --- /dev/null +++ b/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_policy_test.go @@ -0,0 +1,338 @@ +package messagequeue + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "sync" + "testing" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/common/logger" +) + +type failingAutoMergeReadRepository struct { + Repository +} + +func (r *failingAutoMergeReadRepository) GetAutoMergeOverride( + context.Context, + QueueSessionIdentity, +) (*AutoMergeOverride, error) { + return nil, errors.New("policy unavailable") +} + +type pausingAutoMergePolicyRepository struct { + Repository + admission autoMergeAdmissionRepository + captured chan struct{} + release chan struct{} + once sync.Once +} + +func (r *pausingAutoMergePolicyRepository) GetAutoMergeOverride( + ctx context.Context, + identity QueueSessionIdentity, +) (*AutoMergeOverride, error) { + override, err := r.Repository.GetAutoMergeOverride(ctx, identity) + r.once.Do(func() { + close(r.captured) + <-r.release + }) + return override, err +} + +func (r *pausingAutoMergePolicyRepository) InsertForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + message *QueuedMessage, + claim *QueueAttachmentClaim, + maxPerSession int, + policy AutoMergePolicy, +) error { + return r.admission.InsertForSessionWithPolicy(ctx, identity, message, claim, maxPerSession, policy) +} + +func (r *pausingAutoMergePolicyRepository) AutoMergeCandidateIntoAboveForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + candidate *QueuedMessage, + claim *QueueAttachmentClaim, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + return r.admission.AutoMergeCandidateIntoAboveForSessionWithPolicy(ctx, identity, candidate, claim, policy) +} + +func (r *pausingAutoMergePolicyRepository) AutoMergeIntoAboveForSessionWithPolicy( + ctx context.Context, + identity QueueSessionIdentity, + sourceID string, + policy AutoMergePolicy, +) (*QueuedMessage, bool, error) { + return r.admission.AutoMergeIntoAboveForSessionWithPolicy(ctx, identity, sourceID, policy) +} + +func (r *failingAutoMergeReadRepository) Snapshot( + ctx context.Context, + identity QueueSessionIdentity, +) (RepositorySnapshot, error) { + snapshot, err := r.Repository.Snapshot(ctx, identity) + if err != nil { + return RepositorySnapshot{}, err + } + snapshot.AutoMergeOverride = nil + snapshot.AutoMergeError = errors.New("policy unavailable") + return snapshot, nil +} + +type autoMergePolicyService interface { + SetAutoMergePolicy(bool, int64) + ResolveAutoMergePolicy(context.Context, QueueSessionIdentity) (AutoMergePolicy, error) + SetSessionAutoMerge(context.Context, QueueSessionIdentity, bool) (AutoMergePolicy, error) +} +type identityQueueAdmission interface { + QueueMessageWithMetadataForSession( + context.Context, + QueueSessionIdentity, + string, + string, + string, + bool, + []MessageAttachment, + map[string]interface{}, + ) (*QueuedMessage, error) +} + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 +func TestServiceResolvesGlobalPolicyUntilSessionOverride(t *testing.T) { + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "console", OutputPath: "stderr"}) + if err != nil { + t.Fatalf("logger: %v", err) + } + repository := NewMemoryRepository() + service := NewService(repository, DefaultMaxPerSession, log) + policies, ok := interface{}(service).(autoMergePolicyService) + if !ok { + t.Fatal("message queue service does not expose effective Auto-merge policy") + } + ctx := context.Background() + identity := QueueSessionIdentity{TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation"} + untouched := QueueSessionIdentity{TaskID: "task-2", SessionID: "session-2", SessionIncarnationID: "incarnation-2"} + seedQueueSessionIdentity(t, repository, identity) + seedQueueSessionIdentity(t, repository, untouched) + + policies.SetAutoMergePolicy(true, 4) + resolved, err := policies.ResolveAutoMergePolicy(ctx, identity) + assertAutoMergePolicy(t, resolved, err, AutoMergePolicy{ + Enabled: true, Source: AutoMergeSourceGlobal, Revision: 4, + }) + written, err := policies.SetSessionAutoMerge(ctx, identity, false) + if err != nil { + t.Fatalf("set session Auto-merge: %v", err) + } + if written != (AutoMergePolicy{Enabled: false, Source: AutoMergeSourceSession, Revision: 1}) { + t.Fatalf("written session policy = %+v", written) + } + + policies.SetAutoMergePolicy(false, 5) + resolved, err = policies.ResolveAutoMergePolicy(ctx, identity) + assertAutoMergePolicy(t, resolved, err, written) + resolved, err = policies.ResolveAutoMergePolicy(ctx, untouched) + assertAutoMergePolicy(t, resolved, err, AutoMergePolicy{ + Enabled: false, Source: AutoMergeSourceGlobal, Revision: 5, + }) +} + +func TestServiceReloadsSharedGlobalPolicyForIdentity(t *testing.T) { + repository := NewMemoryRepository() + service := NewService(repository, DefaultMaxPerSession, logger.Default()) + identity := QueueSessionIdentity{ + TaskID: "task-shared", SessionID: "session-shared", SessionIncarnationID: "inc-shared", + } + seedQueueSessionIdentity(t, repository, identity) + service.SetAutoMergePolicy(true, 2) + service.SetAutoMergePolicyLoader(func(context.Context) (bool, int64, error) { + return false, 3, nil + }) + + resolved, err := service.ResolveAutoMergePolicy(context.Background(), identity) + assertAutoMergePolicy(t, resolved, err, AutoMergePolicy{ + Enabled: false, Source: AutoMergeSourceGlobal, Revision: 3, + }) + if service.AutoMergeEnabled() { + t.Fatal("persisted policy reload did not refresh the local cache") + } + status, err := service.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("snapshot shared policy: %v", err) + } + if !status.AutoMergeAvailable || status.AutoMergeEnabled == nil || + *status.AutoMergeEnabled || status.AutoMergeRevision == nil || + *status.AutoMergeRevision != 3 { + t.Fatalf("snapshot shared policy = %+v, want false revision 3", status) + } +} + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 +func TestIdentityAdmissionUsesOneEffectiveSessionPolicy(t *testing.T) { + repository := NewMemoryRepository() + service := NewService(repository, DefaultMaxPerSession, logger.Default()) + admissions, ok := interface{}(service).(identityQueueAdmission) + if !ok { + t.Fatal("message queue service does not expose identity-bound admission") + } + ctx := context.Background() + explicitOff := QueueSessionIdentity{TaskID: "task-off", SessionID: "session-off", SessionIncarnationID: "inc-off"} + inheritedOn := QueueSessionIdentity{TaskID: "task-on", SessionID: "session-on", SessionIncarnationID: "inc-on"} + seedQueueSessionIdentity(t, repository, explicitOff) + seedQueueSessionIdentity(t, repository, inheritedOn) + service.SetAutoMergePolicy(true, 2) + if _, err := service.SetSessionAutoMerge(ctx, explicitOff, false); err != nil { + t.Fatalf("set explicit policy: %v", err) + } + for _, identity := range []QueueSessionIdentity{explicitOff, inheritedOn} { + for _, content := range []string{"first", "second"} { + if _, err := admissions.QueueMessageWithMetadataForSession( + ctx, identity, content, "", QueuedByUser, false, nil, nil, + ); err != nil { + t.Fatalf("queue %s: %v", identity.SessionID, err) + } + } + } + offEntries, err := service.repo.ListBySession(ctx, explicitOff.SessionID) + if err != nil { + t.Fatalf("list explicit OFF queue: %v", err) + } + onEntries, err := service.repo.ListBySession(ctx, inheritedOn.SessionID) + if err != nil { + t.Fatalf("list inherited ON queue: %v", err) + } + if len(offEntries) != 2 || len(onEntries) != 1 { + t.Fatalf("entry counts: explicit OFF=%d inherited ON=%d", len(offEntries), len(onEntries)) + } +} + +func TestIdentityAdmissionSkipsAutomaticMergeWhenPolicyReadFails(t *testing.T) { + baseRepository := NewMemoryRepository() + seedQueueSessionIdentity(t, baseRepository, QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + }) + repository := &failingAutoMergeReadRepository{Repository: baseRepository} + service := NewService(repository, DefaultMaxPerSession, logger.Default()) + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + } + for _, content := range []string{"first", "second"} { + if _, err := service.QueueMessageWithMetadataForSession( + context.Background(), identity, content, "", QueuedByUser, false, nil, nil, + ); err != nil { + t.Fatalf("queue with unavailable policy: %v", err) + } + } + entries, err := repository.ListBySession(context.Background(), identity.SessionID) + if err != nil { + t.Fatalf("list queue: %v", err) + } + if len(entries) != 2 { + t.Fatalf("queue length = %d, want 2 separate rows", len(entries)) + } +} + +func TestIdentityAdmissionRechecksPolicyChangedByAnotherConnection(t *testing.T) { + path := filepath.Join(t.TempDir(), "queue-policy-race.db") + open := func() *sql.DB { + raw, err := sql.Open("sqlite3", path+"?_foreign_keys=on&_busy_timeout=5000") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + raw.SetMaxOpenConns(2) + t.Cleanup(func() { _ = raw.Close() }) + return raw + } + firstRaw := open() + secondRaw := open() + if _, err := firstRaw.Exec(` + CREATE TABLE tasks (id TEXT PRIMARY KEY, archived_at TIMESTAMP, updated_at TIMESTAMP); + CREATE TABLE task_sessions ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + queue_incarnation_id TEXT NOT NULL + ); + `); err != nil { + t.Fatalf("create shared authority tables: %v", err) + } + firstDB := sqlx.NewDb(firstRaw, "sqlite3") + secondDB := sqlx.NewDb(secondRaw, "sqlite3") + firstRepository, err := NewSQLiteRepository(firstDB, firstDB) + if err != nil { + t.Fatalf("create first repository: %v", err) + } + secondRepository, err := NewSQLiteRepository(secondDB, secondDB) + if err != nil { + t.Fatalf("create second repository: %v", err) + } + identity := QueueSessionIdentity{ + TaskID: "task-policy-race", SessionID: "session-policy-race", SessionIncarnationID: "inc-policy-race", + } + if _, err := firstRaw.Exec(`INSERT INTO tasks (id, updated_at) VALUES (?, CURRENT_TIMESTAMP)`, identity.TaskID); err != nil { + t.Fatal(err) + } + if _, err := firstRaw.Exec(` + INSERT INTO task_sessions (id, task_id, queue_incarnation_id) VALUES (?, ?, ?) + `, identity.SessionID, identity.TaskID, identity.SessionIncarnationID); err != nil { + t.Fatal(err) + } + firstSQLite := firstRepository.(*sqliteRepository) + secondSQLite := secondRepository.(*sqliteRepository) + firstSQLite.tasksTablePresent = true + secondSQLite.tasksTablePresent = true + if err := firstRepository.InsertForSession(context.Background(), identity, &QueuedMessage{ + TaskID: identity.TaskID, SessionID: identity.SessionID, Content: "first", QueuedBy: QueuedByUser, + }, DefaultMaxPerSession); err != nil { + t.Fatalf("seed queue: %v", err) + } + pausing := &pausingAutoMergePolicyRepository{ + Repository: firstRepository, + admission: firstSQLite, + captured: make(chan struct{}), + release: make(chan struct{}), + } + firstService := NewService(pausing, DefaultMaxPerSession, logger.Default()) + secondService := NewService(secondRepository, DefaultMaxPerSession, logger.Default()) + + result := make(chan error, 1) + go func() { + _, queueErr := firstService.QueueMessageWithMetadataForSession( + context.Background(), identity, "second", "", QueuedByUser, false, nil, nil, + ) + result <- queueErr + }() + <-pausing.captured + if _, err := secondService.SetSessionAutoMerge(context.Background(), identity, false); err != nil { + t.Fatalf("change policy from second connection: %v", err) + } + close(pausing.release) + if err := <-result; err != nil { + t.Fatalf("admit after policy race: %v", err) + } + entries, err := firstRepository.ListBySession(context.Background(), identity.SessionID) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0].Content != "first" || entries[1].Content != "second" { + t.Fatalf("queue after OFF override race = %+v", entries) + } +} + +func assertAutoMergePolicy(t *testing.T, got AutoMergePolicy, err error, want AutoMergePolicy) { + t.Helper() + if err != nil { + t.Fatalf("resolve Auto-merge policy: %v", err) + } + if got != want { + t.Fatalf("Auto-merge policy = %+v, want %+v", got, want) + } +} diff --git a/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_test.go b/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_test.go index a952ad9a0d..c525105db8 100644 --- a/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/service_auto_merge_test.go @@ -119,6 +119,32 @@ func TestService_AutoMergeFoldsCompatibleAdmissionAtCapacity(t *testing.T) { } } +func TestService_AutoMergeFoldsInlineAttachmentAdmissionAtCapacity(t *testing.T) { + repository := NewMemoryRepository() + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + } + seedQueueSessionIdentity(t, repository, identity) + svc := newAutoMergeTestServiceWithRepository(t, repository, 1) + first, err := svc.QueueMessageWithMetadataForSession( + context.Background(), identity, "first", "", QueuedByUser, false, nil, nil, + ) + if err != nil { + t.Fatalf("queue first: %v", err) + } + inline := MessageAttachment{Type: "image", Data: "aW1hZ2U=", MimeType: "image/png"} + second, err := svc.QueueMessageWithMetadataForSessionWithClaim( + context.Background(), identity, "second", "", QueuedByUser, false, + []MessageAttachment{inline}, nil, QueueAttachmentClaim{}, + ) + if err != nil { + t.Fatalf("inline attachment admission at capacity = %v, want fold", err) + } + if second.ID != first.ID || len(second.Attachments) != 1 || second.Attachments[0] != inline { + t.Fatalf("second admission = %+v, want folded inline attachment on %s", second, first.ID) + } +} + func TestService_AutoMergeAtFullQueueRejectsIncompatibleAdmission(t *testing.T) { svc := newAutoMergeTestService(t, 1) if _, err := svc.QueueMessage(context.Background(), "session", "task", "first", "model-a", QueuedByUser, false, nil); err != nil { @@ -171,6 +197,10 @@ func (r *autoMergeCandidateErrorRepository) AutoMergeCandidateIntoAbove(context. return nil, false, r.err } +func (r *autoMergeCandidateErrorRepository) AutoMergeCandidateIntoAboveForSession(context.Context, QueueSessionIdentity, *QueuedMessage) (*QueuedMessage, bool, error) { + return nil, false, r.err +} + // failNextInsertRepository fails the next N Insert calls with ErrQueueFull, // simulating a cross-process admission that observed a stale full count while // the underlying queue already drained. @@ -191,6 +221,17 @@ func (r *failNextInsertRepository) Insert(ctx context.Context, msg *QueuedMessag return r.Repository.Insert(ctx, msg, maxPerSession) } +func (r *failNextInsertRepository) InsertForSession(ctx context.Context, identity QueueSessionIdentity, msg *QueuedMessage, maxPerSession int) error { + r.mu.Lock() + if r.failNext > 0 { + r.failNext-- + r.mu.Unlock() + return ErrQueueFull + } + r.mu.Unlock() + return r.Repository.InsertForSession(ctx, identity, msg, maxPerSession) +} + func TestService_AutoMergeFullQueueRetriesInsertAfterFoldSkip(t *testing.T) { repo := &failNextInsertRepository{Repository: NewMemoryRepository()} svc := newAutoMergeTestServiceWithRepository(t, repo, 1) @@ -460,6 +501,52 @@ func (r *autoMergeErrorRepository) AutoMergeIntoAbove(context.Context, string, s return nil, false, r.err } +func (r *autoMergeErrorRepository) AutoMergeIntoAboveForSession(context.Context, QueueSessionIdentity, string) (*QueuedMessage, bool, error) { + return nil, false, r.err +} + +func TestService_SetSessionAutoMergeWaitsForAdmissionGate(t *testing.T) { + ctx := context.Background() + svc := newAutoMergeTestService(t, 10) + if _, err := svc.QueueMessage(ctx, "session", "task", "first", "", QueuedByUser, false, nil); err != nil { + t.Fatal(err) + } + identity, err := svc.ResolveSessionIdentity(ctx, "task", "session") + if err != nil { + t.Fatal(err) + } + + held := make(chan struct{}) + release := make(chan struct{}) + admissionDone := make(chan error, 1) + go func() { + admissionDone <- svc.WithSessionAdmission(ctx, identity.SessionID, func(context.Context) error { + close(held) + <-release + return nil + }) + }() + <-held + + overrideDone := make(chan error, 1) + go func() { + _, setErr := svc.SetSessionAutoMerge(ctx, identity, false) + overrideDone <- setErr + }() + select { + case err := <-overrideDone: + t.Fatalf("session override bypassed admission gate: %v", err) + case <-time.After(50 * time.Millisecond): + } + close(release) + if err := <-admissionDone; err != nil { + t.Fatal(err) + } + if err := <-overrideDone; err != nil { + t.Fatal(err) + } +} + func newAutoMergeTestService(t *testing.T, max int) *Service { t.Helper() return newAutoMergeTestServiceWithRepository(t, NewMemoryRepository(), max) diff --git a/apps/backend/internal/orchestrator/messagequeue/service_snapshot_test.go b/apps/backend/internal/orchestrator/messagequeue/service_snapshot_test.go new file mode 100644 index 0000000000..5d125bf42f --- /dev/null +++ b/apps/backend/internal/orchestrator/messagequeue/service_snapshot_test.go @@ -0,0 +1,171 @@ +package messagequeue + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/kandev/kandev/internal/common/logger" +) + +type queueSnapshotter interface { + Snapshot(context.Context, QueueSessionIdentity) (*QueueStatus, error) +} + +type pausingSnapshotRepository struct { + Repository + firstCaptured chan struct{} + releaseFirst chan struct{} + calls atomic.Int32 +} + +func (r *pausingSnapshotRepository) Snapshot( + ctx context.Context, + identity QueueSessionIdentity, +) (RepositorySnapshot, error) { + snapshot, err := r.Repository.Snapshot(ctx, identity) + if r.calls.Add(1) == 1 { + close(r.firstCaptured) + <-r.releaseFirst + } + return snapshot, err +} + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.5 +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 +func TestSnapshotProjectsIdentityAndEffectiveAutoMergePolicy(t *testing.T) { + repository := NewMemoryRepository() + service := NewService(repository, 7, logger.Default()) + identity := QueueSessionIdentity{TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation"} + seedQueueSessionIdentity(t, repository, identity) + service.SetAutoMergePolicy(true, 8) + if _, err := service.SetSessionAutoMerge(context.Background(), identity, false); err != nil { + t.Fatalf("set session Auto-merge: %v", err) + } + snapshots, ok := interface{}(service).(queueSnapshotter) + if !ok { + t.Fatal("message queue service does not expose identity-bound snapshots") + } + + status, err := snapshots.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if status.TaskID != identity.TaskID || status.SessionID != identity.SessionID || status.SessionIncarnationID != identity.SessionIncarnationID { + t.Fatalf("snapshot identity = %+v", status) + } + if !status.AutoMergeAvailable || status.AutoMergeEnabled == nil || *status.AutoMergeEnabled || + status.AutoMergeSource != AutoMergeSourceSession || + status.AutoMergeRevision == nil || *status.AutoMergeRevision != 1 { + t.Fatalf("snapshot Auto-merge policy = %+v", status) + } + if status.StatusEpoch == "" || status.StatusEpoch == identity.SessionIncarnationID || status.Max != 7 { + t.Fatalf("snapshot ordering/capacity = %+v", status) + } + second, err := snapshots.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("second snapshot: %v", err) + } + if second.StatusEpoch != status.StatusEpoch || second.StatusGeneration <= status.StatusGeneration { + t.Fatalf("snapshot ordering did not advance in epoch: first=%+v second=%+v", status, second) + } +} + +func TestSnapshotGenerationIsSharedAcrossBackendServices(t *testing.T) { + repository := newTestSQLiteRepo(t) + firstService := NewService(repository, 7, logger.Default()) + secondService := NewService(repository, 7, logger.Default()) + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + } + seedQueueSessionIdentity(t, repository, identity) + + first, err := firstService.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("first backend snapshot: %v", err) + } + second, err := secondService.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("second backend snapshot: %v", err) + } + if first.StatusEpoch == second.StatusEpoch { + t.Fatalf("backend epochs unexpectedly match: %q", first.StatusEpoch) + } + if second.StatusGeneration <= first.StatusGeneration { + t.Fatalf("shared generation did not advance: first=%+v second=%+v", first, second) + } +} + +func TestSnapshotGenerationOrdersConcurrentSnapshotStarts(t *testing.T) { + baseRepository := NewMemoryRepository() + repository := &pausingSnapshotRepository{ + Repository: baseRepository, + firstCaptured: make(chan struct{}), + releaseFirst: make(chan struct{}), + } + service := NewService(repository, 7, logger.Default()) + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + } + seedQueueSessionIdentity(t, baseRepository, identity) + + firstResult := make(chan *QueueStatus, 1) + firstErr := make(chan error, 1) + go func() { + status, err := service.Snapshot(context.Background(), identity) + firstResult <- status + firstErr <- err + }() + <-repository.firstCaptured + if err := baseRepository.Insert(context.Background(), &QueuedMessage{ + ID: "new-entry", TaskID: identity.TaskID, SessionID: identity.SessionID, Content: "new", + }, 7); err != nil { + t.Fatalf("insert between snapshots: %v", err) + } + second, err := service.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("second snapshot: %v", err) + } + close(repository.releaseFirst) + first := <-firstResult + if err := <-firstErr; err != nil { + t.Fatalf("first snapshot: %v", err) + } + + if first.Count != 0 || second.Count != 1 { + t.Fatalf("snapshot setup failed: first=%+v second=%+v", first, second) + } + if first.StatusGeneration >= second.StatusGeneration { + t.Fatalf("snapshot generations do not preserve start order: first=%d second=%d", + first.StatusGeneration, second.StatusGeneration) + } +} + +func TestSnapshotPreservesEntriesAndReportsUnavailablePolicy(t *testing.T) { + baseRepository := NewMemoryRepository() + repository := &failingAutoMergeReadRepository{Repository: baseRepository} + service := NewService(repository, 7, logger.Default()) + identity := QueueSessionIdentity{ + TaskID: "task", SessionID: "session", SessionIncarnationID: "incarnation", + } + seedQueueSessionIdentity(t, baseRepository, identity) + if err := repository.Insert(context.Background(), &QueuedMessage{ + ID: "entry", TaskID: identity.TaskID, SessionID: identity.SessionID, Content: "queued", + }, 7); err != nil { + t.Fatalf("insert queue entry: %v", err) + } + + status, err := service.Snapshot(context.Background(), identity) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + if status.AutoMergeAvailable { + t.Fatalf("AutoMergeAvailable = true, want false: %+v", status) + } + if status.AutoMergeEnabled != nil || status.AutoMergeSource != "" || status.AutoMergeRevision != nil { + t.Fatalf("unavailable policy leaked values: %+v", status) + } + if status.Count != 1 || len(status.Entries) != 1 || status.Entries[0].ID != "entry" { + t.Fatalf("snapshot entries = %+v, want preserved entry", status) + } +} diff --git a/apps/backend/internal/orchestrator/messagequeue/service_test.go b/apps/backend/internal/orchestrator/messagequeue/service_test.go index cf0e857cba..e01b4885b8 100644 --- a/apps/backend/internal/orchestrator/messagequeue/service_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/service_test.go @@ -811,13 +811,8 @@ func TestGetStatus(t *testing.T) { assert.True(t, reserved.IsReservedLifecycleDelivery()) assert.Equal(t, 0, svc.GetStatus(ctx, "s").Count) - // A failed delivery requeues the same entry and it becomes visible again. - _, _, accepted, err = svc.RequeueLifecycleMessageWithCoalesceKey( - ctx, "s", "t", reserved.Content, "", QueuedByWorkflow, false, nil, - reserved.Metadata, "github-pr:repo:1:merged", true, - ) - require.NoError(t, err) - require.True(t, accepted) + // A failed delivery releases the same entry and makes it visible again. + require.NoError(t, svc.RequeueAtHead(ctx, reserved)) assert.Equal(t, 1, svc.GetStatus(ctx, "s").Count) require.NoError(t, svc.AcknowledgeQueued(ctx, "s", reserved.ID)) @@ -849,7 +844,7 @@ func TestTransferSession(t *testing.T) { _, err := svc.QueueMessage(ctx, "old", "task-1", "hand-off", "", "u", false, nil) require.NoError(t, err) - svc.SetPendingMove(ctx, "old", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-b"}) + require.NoError(t, svc.SetPendingMove(ctx, "old", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-b"})) require.NoError(t, svc.TransferSession(ctx, "old", "new")) @@ -885,11 +880,11 @@ func TestRestoreSession(t *testing.T) { {Type: "image", Data: "abc", MimeType: "image/png"}, }, map[string]interface{}{"sender": "task-a"}) require.NoError(t, err) - svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-a"}) + require.NoError(t, svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-a"})) _, err = svc.QueueMessage(ctx, "s", "task-1", "mutated", "", "user", false, nil) require.NoError(t, err) - svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-b"}) + require.NoError(t, svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "task-1", WorkflowStepID: "step-b"})) require.NoError(t, svc.RestoreSession(ctx, "s", []QueuedMessage{*original}, &PendingMove{ TaskID: "task-1", @@ -917,7 +912,7 @@ func TestPendingMove(t *testing.T) { svc := setupService(t) ctx := context.Background() - svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowID: "w1", WorkflowStepID: "step-2", Position: 3, SenderSessionID: "sender-s"}) + require.NoError(t, svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowID: "w1", WorkflowStepID: "step-2", Position: 3, SenderSessionID: "sender-s"})) got, ok := svc.TakePendingMove(ctx, "s") require.True(t, ok) @@ -935,8 +930,8 @@ func TestPendingMove(t *testing.T) { svc := setupService(t) ctx := context.Background() - svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowStepID: "a"}) - svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowStepID: "b"}) + require.NoError(t, svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowStepID: "a"})) + require.NoError(t, svc.SetPendingMove(ctx, "s", &PendingMove{TaskID: "t1", WorkflowStepID: "b"})) got, ok := svc.TakePendingMove(ctx, "s") require.True(t, ok) diff --git a/apps/backend/internal/orchestrator/messagequeue/types.go b/apps/backend/internal/orchestrator/messagequeue/types.go index aff4206abb..b798eb7fd5 100644 --- a/apps/backend/internal/orchestrator/messagequeue/types.go +++ b/apps/backend/internal/orchestrator/messagequeue/types.go @@ -66,15 +66,18 @@ const MetadataLifecycleDurable = "lifecycle_durable_until_accepted" // so stale retries cannot revive work after an archive then unarchive. const MetadataLifecycleGeneration = "lifecycle_queue_generation" -// MetadataLifecycleReserved marks a durable lifecycle entry that ReserveHead -// already handed to a dispatch attempt. The row stays in storage for crash -// recovery, but it is no longer a pending message, so queue status hides it — -// otherwise the delivered prompt and its own reservation row are both visible -// and the reservation looks like a stuck duplicate. Cleared implicitly: a -// requeue rewrites the row's metadata from the in-memory copy, which never -// carries the flag. +// MetadataLifecycleReserved marks a retained queue entry already handed to a +// dispatch attempt. The historical key name is preserved for durable rows +// written by older builds. Reserved rows stay in storage for crash recovery +// but are hidden from pending queue status until acknowledged or released. const MetadataLifecycleReserved = "lifecycle_reserved_in_flight" +// MetadataLifecycleReservationIncarnation binds a retained row to the +// immutable session incarnation that reserved it. A replacement incarnation +// may discard that stale reservation but must never deliver it. The historical +// key name is retained for storage compatibility. +const MetadataLifecycleReservationIncarnation = "lifecycle_reservation_incarnation_id" + // MetadataSenderTaskID identifies the task that produced an agent message. Two // agent entries may only merge when their sender task ids match, so the merge // never mixes prompts issued by different agents. @@ -114,11 +117,52 @@ var ( // ErrTaskInactive means a lifecycle prompt could not be accepted because // its task was deleted or archived before the queue transaction claimed it. ErrTaskInactive = errors.New("queue task is inactive") + // ErrSessionIdentityMismatch means the supplied immutable session identity + // no longer names the authoritative task-session row. + ErrSessionIdentityMismatch = errors.New("queue session identity mismatch") // ErrLifecycleCancelled means an archive/delete purge invalidated a // previously accepted lifecycle entry before it could be retried. ErrLifecycleCancelled = errors.New("lifecycle queue entry cancelled") + // ErrAutoMergePolicyChanged means admission's immutable policy snapshot no + // longer matches the durable session override. The caller must re-resolve + // policy before deciding whether to fold. + ErrAutoMergePolicyChanged = errors.New("queue Auto-merge policy changed") ) +// QueueSessionIdentity is the immutable authority for session-scoped queue work. +type QueueSessionIdentity struct { + TaskID string `json:"task_id"` + SessionID string `json:"session_id"` + SessionIncarnationID string `json:"session_incarnation_id"` +} + +// QueueAttachmentClaim carries authenticated staged-attachment ownership into +// the queue repository transaction. +type QueueAttachmentClaim struct { + OwnerID string + WorkspaceID string + IDs []string +} +type AutoMergeSource string + +const ( + AutoMergeSourceGlobal AutoMergeSource = "global" + AutoMergeSourceSession AutoMergeSource = "session" +) + +// AutoMergePolicy is one immutable effective policy snapshot. +type AutoMergePolicy struct { + Enabled bool + Source AutoMergeSource + Revision int64 +} + +// AutoMergeOverride is the explicit per-session value and revision. +type AutoMergeOverride struct { + Enabled bool + Revision int64 +} + // QueuedMessage represents a single FIFO entry queued for a session. type QueuedMessage struct { ID string `json:"id"` @@ -134,9 +178,17 @@ type QueuedMessage struct { QueuedBy string `json:"queued_by"` // reservedLifecycleDelivery is process-local evidence that ReserveHead - // retained this durable row for acknowledgement. It deliberately is not - // persisted in metadata, where a restart could leak it into a retry. + // retained this durable row for acknowledgement. reservedLifecycleDelivery bool + reservationIdentity QueueSessionIdentity +} + +// QueueRemovalResult is the atomic outcome of a user-driven queue deletion. +// Retained includes reserved in-flight rows so attachment cleanup cannot remove +// a descriptor still owned by surviving work. +type QueueRemovalResult struct { + Removed []QueuedMessage + Retained []QueuedMessage } // IsDurableLifecycle reports whether this entry uses reserve/ack delivery. @@ -153,10 +205,10 @@ func (m *QueuedMessage) IsDurableLifecycle() bool { return origin == "github_pr_automation" || origin == "ci_automation" } -// IsReservedInFlight reports whether this durable row was already reserved for -// a dispatch attempt and should not be shown as a pending queue entry. +// IsReservedInFlight reports whether this row was retained for an in-flight +// dispatch and should not be shown as a pending queue entry. func (m *QueuedMessage) IsReservedInFlight() bool { - if m == nil || !m.IsDurableLifecycle() { + if m == nil { return false } reserved, _ := m.Metadata[MetadataLifecycleReserved].(bool) @@ -172,20 +224,32 @@ func (m *QueuedMessage) IsReservedLifecycleDelivery() bool { // markReservedMetadata returns a copy of metadata carrying the in-flight // reservation marker. func markReservedMetadata(metadata map[string]interface{}) map[string]interface{} { - marked := make(map[string]interface{}, len(metadata)+1) + return markReservedMetadataForIncarnation(metadata, "") +} + +func markReservedMetadataForIncarnation(metadata map[string]interface{}, incarnationID string) map[string]interface{} { + marked := make(map[string]interface{}, len(metadata)+2) for k, v := range metadata { marked[k] = v } marked[MetadataLifecycleReserved] = true + if incarnationID != "" { + marked[MetadataLifecycleReservationIncarnation] = incarnationID + } return marked } -// clearReservedMetadata removes the transient in-process delivery marker from -// copies returned to dispatch or written back for retry. +func lifecycleReservationIncarnation(metadata map[string]interface{}) string { + incarnationID, _ := metadata[MetadataLifecycleReservationIncarnation].(string) + return incarnationID +} + +// clearReservedMetadata removes transient delivery ownership from copies +// returned to dispatch or written back for retry. func clearReservedMetadata(metadata map[string]interface{}) map[string]interface{} { cleared := make(map[string]interface{}, len(metadata)) for k, v := range metadata { - if k != MetadataLifecycleReserved { + if k != MetadataLifecycleReserved && k != MetadataLifecycleReservationIncarnation { cleared[k] = v } } @@ -206,28 +270,36 @@ type MessageAttachment struct { // QueueStatus is the per-session view returned to clients: full ordered list of // pending entries plus capacity info. type QueueStatus struct { - Entries []QueuedMessage `json:"entries"` - Count int `json:"count"` - Max int `json:"max"` - AutoRun bool `json:"auto_run"` - // MergeEnabled mirrors Service.MergeEnabled so clients can hide the - // "Merge with above" affordance without a separate settings fetch. - MergeEnabled bool `json:"merge_enabled"` + Entries []QueuedMessage `json:"entries"` + Count int `json:"count"` + Max int `json:"max"` + TaskID string `json:"task_id,omitempty"` + SessionID string `json:"session_id,omitempty"` + SessionIncarnationID string `json:"session_incarnation_id,omitempty"` + StatusEpoch string `json:"status_epoch,omitempty"` + StatusGeneration int64 `json:"status_generation,omitempty"` + AutoRun bool `json:"auto_run"` + MergeEnabled bool `json:"merge_enabled"` + AutoMergeAvailable bool `json:"auto_merge_available"` + AutoMergeEnabled *bool `json:"auto_merge_enabled,omitempty"` + AutoMergeSource AutoMergeSource `json:"auto_merge_source,omitempty"` + AutoMergeRevision *int64 `json:"auto_merge_revision,omitempty"` } // PendingMove represents a workflow step move requested by an agent (via // move_task_kandev) while its turn is still active. Applied by handleAgentReady // once the turn ends. type PendingMove struct { - // MoveID identifies one deferred move request across queue snapshots. A - // rollback can restore a previously consumed snapshot, so the orchestrator - // needs a durable identity to reject that stale replay. - MoveID string `json:"move_id"` - TaskID string `json:"task_id"` - WorkflowID string `json:"workflow_id"` - WorkflowStepID string `json:"workflow_step_id"` - Position int `json:"position"` - QueuedAt time.Time `json:"queued_at"` + // MoveID is the durable effect token for one deferred move request across + // queue snapshots. Rollback can restore a consumed snapshot, so replay uses + // this token to suppress a second workflow effect. + MoveID string `json:"move_id"` + SessionIncarnationID string `json:"session_incarnation_id,omitempty"` + TaskID string `json:"task_id"` + WorkflowID string `json:"workflow_id"` + WorkflowStepID string `json:"workflow_step_id"` + Position int `json:"position"` + QueuedAt time.Time `json:"queued_at"` // Actor records provenance across the deferred move boundary. Agent is the // value used by move_task_kandev; it prevents owner identity leakage. Actor string `json:"actor,omitempty"` diff --git a/apps/backend/internal/orchestrator/peer_message_resume_test.go b/apps/backend/internal/orchestrator/peer_message_resume_test.go index 3012ea7ec5..239d9058a8 100644 --- a/apps/backend/internal/orchestrator/peer_message_resume_test.go +++ b/apps/backend/internal/orchestrator/peer_message_resume_test.go @@ -67,10 +67,7 @@ func TestQueueAndInterruptForPeerMessage_StoppedSessionResumesAgent(t *testing.T t.Fatalf("UpsertExecutorRunning: %v", err) } - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage( - ctx, "task1", "session1", "subtask status update", - map[string]interface{}{"sender_task_id": "subtask-1"}, - ) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "subtask status update", map[string]interface{}{"sender_task_id": "subtask-1"}) if err != nil { t.Fatalf("queue and interrupt for peer message: %v", err) } diff --git a/apps/backend/internal/orchestrator/pending_move_reaper.go b/apps/backend/internal/orchestrator/pending_move_reaper.go index 7965245838..167b90dfea 100644 --- a/apps/backend/internal/orchestrator/pending_move_reaper.go +++ b/apps/backend/internal/orchestrator/pending_move_reaper.go @@ -117,7 +117,8 @@ func (s *Service) pendingMoveReapReason( if record.Move.IsStaleAt(now, messagequeue.PendingMoveTTL) { return "ttl_expired", true } - if _, err := s.repo.GetTaskSession(ctx, record.SessionID); err != nil { + session, err := s.repo.GetTaskSession(ctx, record.SessionID) + if err != nil { if isTaskSessionNotFound(err) { return "session_missing", true } @@ -126,6 +127,13 @@ func (s *Service) pendingMoveReapReason( zap.Error(err)) return "", false } + if record.Move.TaskID != "" && record.Move.TaskID != session.TaskID { + return "task_identity_changed", true + } + if record.Move.SessionIncarnationID != "" && + record.Move.SessionIncarnationID != session.QueueIncarnationID { + return "session_incarnation_changed", true + } return "", false } @@ -205,6 +213,39 @@ func (s *Service) handlePendingMoveAtAgentReady( return false } + identityChanged := move.TaskID != taskID || + move.SessionIncarnationID == "" || + move.SessionIncarnationID != session.QueueIncarnationID + if identityChanged { + moveID := normalizedPendingMoveID(sessionID, move) + handoffEntryID, listed := s.pendingMoveHandoffPromptID( + ctx, sessionID, move.TaskID, moveID, + ) + if !listed { + settleSession() + return true + } + record := messagequeue.PendingMoveRecord{SessionID: sessionID, Move: *move} + removed, err := s.messageQueue.DeletePendingMoveIfMatch(ctx, record, handoffEntryID) + if err != nil { + s.logger.Warn("failed to discard identity-mismatched pending move", + zap.String("task_id", taskID), + zap.String("session_id", sessionID), + zap.Error(err)) + settleSession() + return true + } + if !removed { + continue + } + s.logger.Warn("discarding pending move for replaced session identity", + pendingMoveLogFields(sessionID, move, moveID)...) + s.pendingMoveHandoffPromptRemoved( + ctx, sessionID, move.TaskID, moveID, handoffEntryID, + ) + return false + } + stale, retryPending := s.discardStalePendingMove(ctx, taskID, sessionID, move) if retryPending { settleSession() @@ -214,23 +255,10 @@ func (s *Service) handlePendingMoveAtAgentReady( return false } - record := messagequeue.PendingMoveRecord{SessionID: sessionID, Move: *move} - removed, err := s.messageQueue.DeletePendingMoveIfMatch(ctx, record, "") - if err != nil { - s.logger.Warn("failed to claim pending move; row preserved for retry", - zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) - settleSession() - return true - } - if !removed { - // A replacement raced the read. Re-read once so the successor can be - // applied by this completed turn instead of waiting for an unrelated - // ready event. The bounded loop preserves the row if races continue. - continue - } // The turn is already complete. Settle the session before applying the - // move so early validation or storage failures inside applyPendingMove - // cannot leave it RUNNING with no active turn. + // move so validation or storage failures cannot leave it RUNNING with no + // active turn. applyPendingMove atomically consumes the exact pending row + // with the task transition; failures leave the row armed for retry. settleSession() s.applyPendingMove(ctx, taskID, sessionID, session, move) return true diff --git a/apps/backend/internal/orchestrator/pending_move_reaper_test.go b/apps/backend/internal/orchestrator/pending_move_reaper_test.go index 25dc4d094f..d5de3387bc 100644 --- a/apps/backend/internal/orchestrator/pending_move_reaper_test.go +++ b/apps/backend/internal/orchestrator/pending_move_reaper_test.go @@ -95,7 +95,7 @@ func TestPendingMove_ExpiredMoveIsNotReplayed(t *testing.T) { // Re-arm the move the scenario queued, aged past the TTL. The hand-off // prompt the scenario queued alongside it stays as it was. - sc.svc.messageQueue.SetPendingMove(sc.ctx, sc.reviewSessionID, &messagequeue.PendingMove{ + requireSetPendingMove(t, sc.svc, sc.ctx, sc.reviewSessionID, &messagequeue.PendingMove{ TaskID: "task-1", WorkflowID: "wf1", WorkflowStepID: stepInProgressID, @@ -203,13 +203,18 @@ func TestPendingMove_ReadFailurePreservesTurnForRetry(t *testing.T) { func TestPendingMove_ReplaysReplacementAfterClaimRace(t *testing.T) { sc := buildPendingMoveScenario(t) + session, err := sc.repo.GetTaskSession(sc.ctx, sc.reviewSessionID) + if err != nil { + t.Fatalf("load review session: %v", err) + } queueRepo := messagequeue.NewMemoryRepository() initialMove := &messagequeue.PendingMove{ - MoveID: "move-initial", - TaskID: "task-1", - WorkflowID: "wf1", - WorkflowStepID: stepInProgressID, - QueuedAt: time.Now().UTC().Add(-time.Minute), + MoveID: "move-initial", + SessionIncarnationID: session.QueueIncarnationID, + TaskID: "task-1", + WorkflowID: "wf1", + WorkflowStepID: stepInProgressID, + QueuedAt: time.Now().UTC().Add(-time.Minute), } replacement := *initialMove replacement.MoveID = "move-replacement" @@ -240,7 +245,7 @@ func TestPendingMove_ReplaysReplacementAfterClaimRace(t *testing.T) { // TTL check: a move armed moments ago must still apply exactly as before. func TestPendingMove_FreshMoveStillReplays(t *testing.T) { sc := buildPendingMoveScenario(t) - sc.svc.messageQueue.SetPendingMove(sc.ctx, sc.reviewSessionID, &messagequeue.PendingMove{ + requireSetPendingMove(t, sc.svc, sc.ctx, sc.reviewSessionID, &messagequeue.PendingMove{ TaskID: "task-1", WorkflowID: "wf1", WorkflowStepID: stepInProgressID, @@ -280,7 +285,7 @@ func TestDiscardStalePendingMove(t *testing.T) { t.Run("stale move is discarded without mutating the caller", func(t *testing.T) { move := &messagequeue.PendingMove{TaskID: "task-1", QueuedAt: staleQueuedAt()} - svc.messageQueue.SetPendingMove(ctx, "sess-1", move) + requireSetPendingMove(t, svc, ctx, "sess-1", move) stale, retryPending := svc.discardStalePendingMove(ctx, "task-1", "sess-1", move) if !stale || retryPending { t.Fatal("stale move was not discarded") @@ -315,8 +320,30 @@ func seedPendingMoveSession(t *testing.T, repo *sqliterepo.Repository, taskID, s } } -func armPendingMove(svc *Service, sessionID, taskID, stepID string, queuedAt time.Time) { - svc.messageQueue.SetPendingMove(context.Background(), sessionID, &messagequeue.PendingMove{ +func requireSetPendingMove( + t *testing.T, + svc *Service, + ctx context.Context, + sessionID string, + move *messagequeue.PendingMove, +) { + t.Helper() + if move.SessionIncarnationID == "" { + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, move.TaskID, sessionID) + if err == nil { + move.SessionIncarnationID = identity.SessionIncarnationID + } else { + move.SessionIncarnationID = "test:" + sessionID + } + } + if err := svc.messageQueue.SetPendingMove(ctx, sessionID, move); err != nil { + t.Fatalf("set pending move: %v", err) + } +} + +func armPendingMove(t *testing.T, svc *Service, sessionID, taskID, stepID string, queuedAt time.Time) { + t.Helper() + requireSetPendingMove(t, svc, context.Background(), sessionID, &messagequeue.PendingMove{ MoveID: "move-" + sessionID, TaskID: taskID, WorkflowID: "wf1", @@ -349,25 +376,38 @@ func TestReapStalePendingMoves(t *testing.T) { fresh := time.Now().UTC().Add(-time.Minute) seedPendingMoveSession(t, repo, "task-expired", "sess-expired") - armPendingMove(svc, "sess-expired", "task-expired", "step-blocked", staleQueuedAt()) + armPendingMove(t, svc, "sess-expired", "task-expired", "step-blocked", staleQueuedAt()) seedPendingMoveSession(t, repo, "task-fresh", "sess-fresh") - armPendingMove(svc, "sess-fresh", "task-fresh", "step-work", fresh) + armPendingMove(t, svc, "sess-fresh", "task-fresh", "step-work", fresh) + + seedPendingMoveSession(t, repo, "task-replaced", "sess-replaced") + if err := svc.messageQueue.SetPendingMove(ctx, "sess-replaced", &messagequeue.PendingMove{ + MoveID: "move-replaced", + SessionIncarnationID: "superseded-incarnation", + TaskID: "task-replaced", + WorkflowID: "wf1", + WorkflowStepID: "step-work", + QueuedAt: fresh, + }); err != nil { + t.Fatalf("SetPendingMove: %v", err) + } // Orphan: armed well within the TTL, but the keyed session is gone. - armPendingMove(svc, "sess-orphan", "task-orphan", "step-human-qa", fresh) + armPendingMove(t, svc, "sess-orphan", "task-orphan", "step-human-qa", fresh) // One task, two sessions: one stale, one fresh. seedPendingMoveSession(t, repo, "task-multi", "sess-multi-stale") seedPendingMoveSession(t, repo, "task-multi", "sess-multi-fresh") - armPendingMove(svc, "sess-multi-stale", "task-multi", "step-ci-fixup", staleQueuedAt()) - armPendingMove(svc, "sess-multi-fresh", "task-multi", "step-work", fresh) + armPendingMove(t, svc, "sess-multi-stale", "task-multi", "step-ci-fixup", staleQueuedAt()) + armPendingMove(t, svc, "sess-multi-fresh", "task-multi", "step-work", fresh) svc.reapStalePendingMovesOnce(ctx) for sessionID, wantArmed := range map[string]bool{ "sess-expired": false, // aged past the TTL "sess-orphan": false, // keyed session no longer exists + "sess-replaced": false, // immutable session incarnation no longer matches "sess-multi-stale": false, // aged past the TTL "sess-fresh": true, // healthy "sess-multi-fresh": true, // healthy sibling of a reaped row @@ -391,7 +431,7 @@ func TestReapStalePendingMoves_PreservesRowOnSessionLookupError(t *testing.T) { svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) ctx := context.Background() - armPendingMove(svc, "sess-unknown", "task-1", "step-blocked", time.Now().UTC().Add(-time.Minute)) + armPendingMove(t, svc, "sess-unknown", "task-1", "step-blocked", time.Now().UTC().Add(-time.Minute)) // Close the database so GetTaskSession fails with a real infrastructure // error rather than sql.ErrNoRows. Only ErrNoRows becomes @@ -432,7 +472,7 @@ func TestReapStalePendingMoves_RetriesAfterPromptRemovalFailure(t *testing.T) { ); err != nil { t.Fatalf("queue hand-off prompt: %v", err) } - svc.messageQueue.SetPendingMove(ctx, sessionID, &messagequeue.PendingMove{ + requireSetPendingMove(t, svc, ctx, sessionID, &messagequeue.PendingMove{ MoveID: moveID, TaskID: taskID, WorkflowID: "wf", WorkflowStepID: "target", QueuedAt: staleQueuedAt(), }) @@ -462,14 +502,14 @@ func TestReapStalePendingMoves_PreservesReplacementBetweenListAndDelete(t *testi queueRepo := &replaceBeforePendingMoveDeleteRepository{ Repository: messagequeue.NewMemoryRepository(), replacement: &messagequeue.PendingMove{ - MoveID: "move-b", TaskID: "task-b", WorkflowID: "wf-b", - WorkflowStepID: "step-b", QueuedAt: queuedAtB, + MoveID: "move-b", SessionIncarnationID: "memory:sess-replaced", + TaskID: "task-a", WorkflowID: "wf-b", WorkflowStepID: "step-b", QueuedAt: queuedAtB, }, } svc.messageQueue = messagequeue.NewService(queueRepo, messagequeue.DefaultMaxPerSession, testLogger()) ctx := context.Background() const sessionID = "sess-replaced" - svc.messageQueue.SetPendingMove(ctx, sessionID, &messagequeue.PendingMove{ + requireSetPendingMove(t, svc, ctx, sessionID, &messagequeue.PendingMove{ MoveID: "move-a", TaskID: "task-a", WorkflowID: "wf-a", WorkflowStepID: "step-a", QueuedAt: staleQueuedAt(), }) @@ -480,13 +520,19 @@ func TestReapStalePendingMoves_PreservesReplacementBetweenListAndDelete(t *testi if !exists { t.Fatal("fresh replacement move B was removed by stale move A's sweep") } - if got.MoveID != "move-b" || got.TaskID != "task-b" || !got.QueuedAt.Equal(queuedAtB) { + if got.MoveID != "move-b" || got.TaskID != "task-a" || !got.QueuedAt.Equal(queuedAtB) { t.Fatalf("replacement move B changed: %+v", got) } } -func armPendingMoveInWorkflow(svc *Service, sessionID, taskID, workflowID string, queuedAt time.Time) { - svc.messageQueue.SetPendingMove(context.Background(), sessionID, &messagequeue.PendingMove{ +func armPendingMoveInWorkflow( + t *testing.T, + svc *Service, + sessionID, taskID, workflowID string, + queuedAt time.Time, +) { + t.Helper() + requireSetPendingMove(t, svc, context.Background(), sessionID, &messagequeue.PendingMove{ MoveID: "move-" + sessionID, TaskID: taskID, WorkflowID: workflowID, @@ -518,17 +564,17 @@ func TestReapStalePendingMoves_IsRowLocalAcrossWorkflows(t *testing.T) { // Workflow A: one stale row and one healthy row. seedPendingMoveSession(t, repo, "task-a-stale", "sess-a-stale") seedPendingMoveSession(t, repo, "task-a-fresh", "sess-a-fresh") - armPendingMoveInWorkflow(svc, "sess-a-stale", "task-a-stale", workflowA, staleQueuedAt()) - armPendingMoveInWorkflow(svc, "sess-a-fresh", "task-a-fresh", workflowA, fresh) + armPendingMoveInWorkflow(t, svc, "sess-a-stale", "task-a-stale", workflowA, staleQueuedAt()) + armPendingMoveInWorkflow(t, svc, "sess-a-fresh", "task-a-fresh", workflowA, fresh) // Workflow B: the same shapes, plus an orphan within the TTL. Production // had exactly this — an orphan in a workflow other than the one under // investigation. seedPendingMoveSession(t, repo, "task-b-stale", "sess-b-stale") seedPendingMoveSession(t, repo, "task-b-fresh", "sess-b-fresh") - armPendingMoveInWorkflow(svc, "sess-b-stale", "task-b-stale", workflowB, staleQueuedAt()) - armPendingMoveInWorkflow(svc, "sess-b-fresh", "task-b-fresh", workflowB, fresh) - armPendingMoveInWorkflow(svc, "sess-b-orphan", "task-b-orphan", workflowB, fresh) + armPendingMoveInWorkflow(t, svc, "sess-b-stale", "task-b-stale", workflowB, staleQueuedAt()) + armPendingMoveInWorkflow(t, svc, "sess-b-fresh", "task-b-fresh", workflowB, fresh) + armPendingMoveInWorkflow(t, svc, "sess-b-orphan", "task-b-orphan", workflowB, fresh) svc.reapStalePendingMovesOnce(ctx) diff --git a/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go b/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go new file mode 100644 index 0000000000..6af49c078e --- /dev/null +++ b/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go @@ -0,0 +1,50 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + "github.com/kandev/kandev/internal/task/models" +) + +func TestPromptTaskExpectedIdentityRejectsReplacementAfterClaim(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + const taskID, sessionID = "task-prompt-identity", "session-prompt-identity" + seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput) + seedExecutorRunning(t, repo, sessionID, taskID, "execution-prompt-identity") + + agentMgr := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + t.Fatalf("resolve session identity: %v", err) + } + + _, err = svc.promptTask( + ctx, taskID, sessionID, "queued prompt", "", false, nil, true, + promptTaskOptions{ + expectedSessionIdentity: &identity, + afterClaim: func() error { + _, updateErr := repo.DB().ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + sessionID, + ) + return updateErr + }, + }, + ) + if !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + t.Fatalf("promptTask error = %v, want session identity mismatch", err) + } + if got := len(agentMgr.capturedPrompts); got != 0 { + t.Fatalf("replacement session prompts = %d, want 0", got) + } +} diff --git a/apps/backend/internal/orchestrator/queue_auto_run_review_test.go b/apps/backend/internal/orchestrator/queue_auto_run_review_test.go index 234ee81df0..510c63df61 100644 --- a/apps/backend/internal/orchestrator/queue_auto_run_review_test.go +++ b/apps/backend/internal/orchestrator/queue_auto_run_review_test.go @@ -107,3 +107,16 @@ func TestPublishTaskQueueStatusEventIncludesQueuePolicy(t *testing.T) { require.Equal(t, false, data["auto_run"]) require.Equal(t, false, data["merge_enabled"]) } + +func TestPublishTaskQueueStatusEventMarksTaskOnlyFallbackScope(t *testing.T) { + svc := createTestService(setupTestRepo(t), newMockStepGetter(), newMockTaskRepo()) + eventBus := &recordingEventBus{} + svc.eventBus = eventBus + + svc.publishTaskQueueStatusEvent(context.Background(), "missing-task", "") + + require.Len(t, eventBus.events, 1) + data, ok := eventBus.events[0].event.Data.(map[string]interface{}) + require.True(t, ok) + require.Equal(t, "task", data["queue_status_scope"]) +} diff --git a/apps/backend/internal/orchestrator/queue_auto_run_test.go b/apps/backend/internal/orchestrator/queue_auto_run_test.go index 640d268854..7d74234395 100644 --- a/apps/backend/internal/orchestrator/queue_auto_run_test.go +++ b/apps/backend/internal/orchestrator/queue_auto_run_test.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/orchestrator/watcher" "github.com/kandev/kandev/internal/task/models" wfmodels "github.com/kandev/kandev/internal/workflow/models" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -72,6 +73,69 @@ func TestSetQueueAutoRunOnDispatchesPromptableHead(t *testing.T) { assert.Equal(t, 0, svc.messageQueue.GetStatus(ctx, "session-1").Count) } +func TestExecuteQueuedMessageDrainsNextMessageAfterCompletion(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "task-1", "session-1", "step-1") + seedExecutorRunning(t, repo, "session-1", "task-1", "exec-1") + session, err := repo.GetTaskSession(ctx, "session-1") + require.NoError(t, err) + session.State = models.TaskSessionStateWaitingForInput + require.NoError(t, repo.UpdateTaskSession(ctx, session)) + + prompts := make(chan string, 2) + agentManager := &mockAgentManager{ + isAgentRunning: true, + repoForExecutionLookup: repo, + promptAgentFunc: func(ctx context.Context, _ string, prompt string, _ []v1.MessageAttachment, _ bool) (*executor.PromptResult, error) { + // Model the ready event that can arrive before the queue worker + // releases its in-flight reservation. + current, getErr := repo.GetTaskSession(ctx, "session-1") + if getErr != nil { + return nil, getErr + } + current.State = models.TaskSessionStateWaitingForInput + if updateErr := repo.UpdateTaskSession(ctx, current); updateErr != nil { + return nil, updateErr + } + prompts <- prompt + return &executor.PromptResult{}, nil + }, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentManager) + svc.executor = executor.NewExecutor(agentManager, repo, testLogger(), executor.ExecutorConfig{}) + svc.messageCreator = &mockMessageCreator{} + svc.messageQueue.SetAutoMergeEnabled(false) + require.NoError(t, svc.messageQueue.SetAutoRun(ctx, "session-1", true)) + for _, prompt := range []string{"first queued", "second queued"} { + _, err := svc.messageQueue.QueueMessage( + ctx, "session-1", "task-1", prompt, "", messagequeue.QueuedByUser, false, nil, + ) + require.NoError(t, err) + } + + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, "task-1", "session-1") + require.NoError(t, err) + queued, ok, _, err := svc.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + require.NoError(t, err) + require.True(t, ok) + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, queued) + + svc.executeQueuedMessageWithReservation("session-1", queued, reservation) + + for i := 0; i < 2; i++ { + select { + case <-prompts: + case <-time.After(5 * time.Second): + agentManager.mu.Lock() + captured := append([]string(nil), agentManager.capturedPrompts...) + agentManager.mu.Unlock() + t.Fatalf("timed out waiting for queued prompt %d: queue_count=%d captured=%q", i+1, svc.messageQueue.GetStatus(ctx, "session-1").Count, captured) + } + } + assert.Equal(t, 0, svc.messageQueue.GetStatus(ctx, "session-1").Count) +} + func TestSetQueueAutoRunOnDoesNotBypassClarification(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/orchestrator/queue_send_now.go b/apps/backend/internal/orchestrator/queue_send_now.go index 3fbc5c1fa0..865dc27a54 100644 --- a/apps/backend/internal/orchestrator/queue_send_now.go +++ b/apps/backend/internal/orchestrator/queue_send_now.go @@ -37,6 +37,15 @@ var ( // shared cancellation coordinator, then the exact claim is handed to one // replacement prompt. The explicit Cancel path is deliberately not involved. func (s *Service) SendQueuedNow(ctx context.Context, sessionID, scope, entryID string) (int, error) { + return s.sendQueuedNow(ctx, nil, sessionID, scope, entryID) +} + +// SendQueuedNowForSession dispatches the selection only for the exact session incarnation. +func (s *Service) SendQueuedNowForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity, scope, entryID string) (int, error) { + return s.sendQueuedNow(ctx, &identity, identity.SessionID, scope, entryID) +} + +func (s *Service) sendQueuedNow(ctx context.Context, identity *messagequeue.QueueSessionIdentity, sessionID, scope, entryID string) (int, error) { // Dispatching a queued prompt puts an agent to work: session.prompt. if err := s.authorizeSessionPrompt(ctx, sessionID); err != nil { return 0, err @@ -52,7 +61,7 @@ func (s *Service) SendQueuedNow(ctx context.Context, sessionID, scope, entryID s if err != nil { return 0, err } - return s.sendQueuedNowAfterCapture(ctx, sessionID, scope, entryID, turnBefore) + return s.sendQueuedNowAfterCapture(ctx, identity, sessionID, scope, entryID, turnBefore) } type sendNowGuard struct { @@ -90,6 +99,7 @@ func (guard *sendNowGuard) close() { func (s *Service) sendQueuedNowAfterCapture( ctx context.Context, + identity *messagequeue.QueueSessionIdentity, sessionID, scope, entryID, turnBefore string, ) (int, error) { guard := newSendNowGuard(s, sessionID) @@ -105,12 +115,13 @@ func (s *Service) sendQueuedNowAfterCapture( return 0, err } - taskID, sessionState, entries, err := s.loadSendNowSelection(ctx, sessionID, scope, entryID) + taskID, sessionState, entries, err := s.loadSendNowSelection(ctx, identity, sessionID, scope, entryID) if err != nil { return 0, err } return s.dispatchSendNowSelection( ctx, + identity, sessionID, taskID, sessionState, @@ -147,7 +158,7 @@ func (s *Service) restorePendingQueuedDispatchForSendNow(ctx context.Context, se zap.String("queue_id", reservation.source.ID), zap.Error(err), ) - s.publishQueueStatusEvent(ctx, sessionID) + s.publishQueueStatusEventForIdentity(ctx, reservation.identity) return ErrSendNowQueueChanged } return s.supersedeQueuedDispatchForSendNow(sessionID, reservation) @@ -158,6 +169,7 @@ func (s *Service) sendNowRestoreClaimForReservation( reservation *queuedDispatchReservation, ) (*messagequeue.SendNowClaim, error) { restore := &messagequeue.SendNowClaim{ + Identity: reservation.identity, Sources: []messagequeue.QueuedMessage{*reservation.source}, SourceGenerations: make(map[string]int64), } @@ -208,6 +220,7 @@ func (s *Service) verifySendNowTurn(ctx context.Context, sessionID, expectedTurn func (s *Service) loadSendNowSelection( ctx context.Context, + identity *messagequeue.QueueSessionIdentity, sessionID, scope, entryID string, ) (string, models.TaskSessionState, []messagequeue.QueuedMessage, error) { session, err := s.repo.GetTaskSession(ctx, sessionID) @@ -217,7 +230,16 @@ func (s *Service) loadSendNowSelection( if session == nil { return "", "", nil, ErrSessionNotPromptable } - entries, _, err := selectSendNowEntries(s.messageQueue.GetStatus(ctx, sessionID), scope, entryID) + var status *messagequeue.QueueStatus + if identity != nil { + status, err = s.messageQueue.Snapshot(ctx, *identity) + if err != nil { + return "", "", nil, err + } + } else { + status = s.messageQueue.GetStatus(ctx, sessionID) + } + entries, _, err := selectSendNowEntries(status, scope, entryID) if err != nil { return "", "", nil, err } @@ -229,6 +251,7 @@ func (s *Service) loadSendNowSelection( func (s *Service) dispatchSendNowSelection( ctx context.Context, + identity *messagequeue.QueueSessionIdentity, sessionID, taskID string, sessionState models.TaskSessionState, scope string, @@ -238,7 +261,7 @@ func (s *Service) dispatchSendNowSelection( ) (int, error) { promptabilityErr := s.checkSessionPromptable(taskID, sessionID, sessionState) if promptabilityErr == nil { - dispatched, err := s.claimAndDispatchSendNow(ctx, sessionID, scope, entries) + dispatched, err := s.claimAndDispatchSendNow(ctx, identity, sessionID, scope, entries) if err != nil { return 0, err } @@ -258,7 +281,7 @@ func (s *Service) dispatchSendNowSelection( unlockGuard, relockGuard, func(actionCtx context.Context) (bool, error) { - return s.claimAndDispatchSendNow(actionCtx, sessionID, scope, entries) + return s.claimAndDispatchSendNow(actionCtx, identity, sessionID, scope, entries) }, cancellationKindQueueSendNow, turnBefore, @@ -292,13 +315,24 @@ func selectSendNowEntries(status *messagequeue.QueueStatus, scope, entryID strin return entries, entryIDs, nil } -func (s *Service) claimAndDispatchSendNow(ctx context.Context, sessionID, scope string, entries []messagequeue.QueuedMessage) (bool, error) { - claim, err := s.messageQueue.ClaimSendNow(ctx, sessionID, entries) +func (s *Service) claimAndDispatchSendNow(ctx context.Context, identity *messagequeue.QueueSessionIdentity, sessionID, scope string, entries []messagequeue.QueuedMessage) (bool, error) { + var claim *messagequeue.SendNowClaim + var err error + if identity != nil { + claim, err = s.messageQueue.ClaimSendNowForSession(ctx, *identity, entries) + } else { + claim, err = s.messageQueue.ClaimSendNow(ctx, sessionID, entries) + } if err != nil { return false, mapSendNowClaimError(scope, err) } - s.publishQueueStatusEvent(ctx, sessionID) - reservation := s.markQueuedDispatchInFlightWithSourceLocked(sessionID, claim.Dispatch.ID, nil) + s.publishQueueStatusEventForIdentity(ctx, claim.Identity) + var reservation *queuedDispatchReservation + if claim.Identity.SessionIncarnationID != "" { + reservation = s.markQueuedDispatchInFlightWithIdentityLocked(claim.Identity, claim.Dispatch.ID, nil) + } else { + reservation = s.markQueuedDispatchInFlightWithSourceLocked(sessionID, claim.Dispatch.ID, nil) + } if reservation != nil { reservation.liveEligible.Store(true) } @@ -308,7 +342,7 @@ func (s *Service) claimAndDispatchSendNow(ctx context.Context, sessionID, scope zap.String("session_id", sessionID), zap.Error(restoreErr)) } s.clearQueuedDispatchInFlightIfCurrent(sessionID, reservation) - s.publishQueueStatusEvent(context.Background(), sessionID) + s.publishQueueStatusEventForIdentity(context.Background(), claim.Identity) return false, nil } return true, nil @@ -408,7 +442,7 @@ func (s *Service) executeSendNowClaimWithContext( s.logger.Error("failed to restore send-now queue claim", zap.String("session_id", sessionID), zap.Error(err)) } - s.publishQueueStatusEvent(ctx, sessionID) + s.publishQueueStatusEventForIdentity(ctx, claim.Identity) } if s.isSessionResetInProgress(sessionID) { restore() @@ -420,6 +454,20 @@ func (s *Service) executeSendNowClaimWithContext( restore() return } + if claim.Identity.SessionIncarnationID != "" { + current, err := s.messageQueue.ResolveSessionIdentity( + ctx, + claim.Identity.TaskID, + claim.Identity.SessionID, + ) + if err != nil || current != claim.Identity { + s.logger.Info("discarding send-now dispatch for replaced session", + zap.String("session_id", sessionID), + zap.String("task_id", claim.Identity.TaskID)) + restore() + return + } + } if err := s.promptSendNowClaim(ctx, claim); err != nil { s.logger.Warn("send-now replacement prompt failed; restoring queue claim", zap.String("session_id", sessionID), zap.Error(err)) @@ -429,10 +477,10 @@ func (s *Service) executeSendNowClaimWithContext( if err := s.acknowledgeSendNowClaimWithRetry(ctx, claim); err != nil { s.logger.Error("failed to acknowledge accepted send-now queue claim", zap.String("session_id", sessionID), zap.Error(err)) - s.publishQueueStatusEvent(context.Background(), sessionID) + s.publishQueueStatusEventForIdentity(context.Background(), claim.Identity) return } - s.publishQueueStatusEvent(ctx, sessionID) + s.publishQueueStatusEventForIdentity(ctx, claim.Identity) } func (s *Service) claimSendNowExecution(sessionID, dispatchID string) error { @@ -463,26 +511,30 @@ func (s *Service) promptSendNowClaim(ctx context.Context, claim *messagequeue.Se references := entityrefs.NormalizePersisted(claim.Dispatch.Metadata[messagequeue.MetadataEntityReferences]) promptContent := AppendEntityReferenceContext(claim.Dispatch.Content, references) promptContent = appendStepHandoffToPrompt(promptContent, stepHandoffFromQueuedMetadata(claim.Dispatch.Metadata)) - if err := s.recordQueuedUserMessage(ctx, &claim.Dispatch, attachments); err != nil { - s.logger.Warn("failed to record send-now user message before prompt", - zap.String("session_id", sessionID), zap.Error(err)) - } else if s.messageCreator != nil { - for i := range claim.Sources { - markQueuedUserMessageRecorded(&claim.Sources[i]) - } - } - if session, err := s.repo.GetTaskSession(ctx, sessionID); err == nil && session != nil { - // This transition is intentionally not rolled back if promptTask later - // rejects the replacement. The ordinary FIFO handoff uses the same - // ordering: workflow admission precedes executor prompt acceptance, and - // the restored claim is retried through the normal ready path. - if !turnStartAlreadyProcessed(claim.Dispatch.Metadata) { - s.processOnTurnStartViaEngine(ctx, claim.Dispatch.TaskID, session) - } - } _, err := s.promptTask(ctx, claim.Dispatch.TaskID, sessionID, promptContent, claim.Dispatch.Model, - claim.Dispatch.PlanMode, attachments, false, promptTaskOptions{claimEntryID: claim.Dispatch.ID}) + claim.Dispatch.PlanMode, attachments, false, promptTaskOptions{ + claimEntryID: claim.Dispatch.ID, + afterClaim: func() error { + if !s.queuedDispatchIdentityIsCurrent(ctx, claim.Identity) { + return errLifecyclePromptReservationSuperseded + } + if err := s.recordQueuedUserMessage(ctx, &claim.Dispatch, attachments); err != nil { + s.logger.Warn("failed to record send-now user message after prompt claim", + zap.String("session_id", sessionID), zap.Error(err)) + } else if s.messageCreator != nil { + for i := range claim.Sources { + markQueuedUserMessageRecorded(&claim.Sources[i]) + } + } + if session, loadErr := s.repo.GetTaskSession(ctx, sessionID); loadErr == nil && + s.queuedSessionMatchesIdentity(session, claim.Identity) && + !turnStartAlreadyProcessed(claim.Dispatch.Metadata) { + s.processOnTurnStartViaEngine(ctx, claim.Dispatch.TaskID, session) + } + return nil + }, + }) return err } diff --git a/apps/backend/internal/orchestrator/queue_send_now_test.go b/apps/backend/internal/orchestrator/queue_send_now_test.go index ce707e949b..6e39d33c9c 100644 --- a/apps/backend/internal/orchestrator/queue_send_now_test.go +++ b/apps/backend/internal/orchestrator/queue_send_now_test.go @@ -186,6 +186,63 @@ func TestExecuteSendNowClaimRestorePreservesRecordedSources(t *testing.T) { } } +func TestExecuteSendNowClaimRejectsRecreatedSessionBeforePrompt(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-1", "session-1", models.TaskSessionStateWaitingForInput) + seedExecutorRunning(t, repo, "session-1", "task-1", "exec-1") + session, err := repo.GetTaskSession(ctx, "session-1") + if err != nil { + t.Fatalf("get session: %v", err) + } + agentMgr := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.messageQueue = newAuthoritativeMemoryQueue(repo, testLogger()) + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID) + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } + messageCreator := &mockMessageCreator{} + svc.messageCreator = messageCreator + source, err := svc.messageQueue.QueueMessageWithMetadataForSession( + ctx, identity, "old prompt", "", messagequeue.QueuedByUser, false, nil, nil, + ) + if err != nil { + t.Fatalf("queue old prompt: %v", err) + } + claim, err := svc.messageQueue.ClaimSendNowForSession( + ctx, + identity, + []messagequeue.QueuedMessage{*source}, + ) + if err != nil { + t.Fatalf("claim old prompt: %v", err) + } + reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, claim.Dispatch.ID, nil) + reservation.liveEligible.Store(true) + + if _, err := repo.DB().ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "replacement-incarnation", + session.ID, + ); err != nil { + t.Fatalf("replace session incarnation: %v", err) + } + svc.executeSendNowClaimWithContext(ctx, claim, reservation) + + if got := len(agentMgr.capturedPrompts); got != 0 { + t.Fatalf("replacement session prompts = %d, want 0", got) + } + if got := len(messageCreator.userMessages); got != 0 { + t.Fatalf("replacement user messages = %d, want 0", got) + } + if got := svc.messageQueue.GetStatus(ctx, identity.SessionID).Count; got != 0 { + t.Fatalf("old claim restored into replacement queue: count=%d", got) + } +} + func TestPromptSendNowClaimSkipsOnTurnStartWhenAlreadyProcessed(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/orchestrator/queued_dispatch.go b/apps/backend/internal/orchestrator/queued_dispatch.go index 5037668f3c..2f11919050 100644 --- a/apps/backend/internal/orchestrator/queued_dispatch.go +++ b/apps/backend/internal/orchestrator/queued_dispatch.go @@ -23,6 +23,7 @@ const ( type queuedDispatchReservation struct { sessionID string entryID string + identity messagequeue.QueueSessionIdentity source *messagequeue.QueuedMessage phase atomic.Uint32 // liveEligible is set only for Send Now reservations. It allows the @@ -37,10 +38,15 @@ type queuedDispatchReservation struct { successorTurn atomic.Value } -func newQueuedDispatchReservation(sessionID, entryID string, source *messagequeue.QueuedMessage) *queuedDispatchReservation { +func newQueuedDispatchReservation( + sessionID, entryID string, + identity messagequeue.QueueSessionIdentity, + source *messagequeue.QueuedMessage, +) *queuedDispatchReservation { reservation := &queuedDispatchReservation{ sessionID: sessionID, entryID: entryID, + identity: identity, source: source, } reservation.phase.Store(uint32(queuedDispatchPending)) @@ -85,7 +91,7 @@ func (s *Service) markQueuedDispatchInFlightWithSourceLocked( if sessionID == "" || entryID == "" { return nil } - reservation := newQueuedDispatchReservation(sessionID, entryID, source) + reservation := newQueuedDispatchReservation(sessionID, entryID, messagequeue.QueueSessionIdentity{}, source) if previous, ok := s.dispatchingQueued.Load(sessionID); ok { if previousReservation, ok := previous.(*queuedDispatchReservation); ok { previousReservation.phase.Store(uint32(queuedDispatchSupersededByNewDispatch)) @@ -100,6 +106,37 @@ func (s *Service) markQueuedDispatchInFlightWithSourceLocked( return reservation } +func (s *Service) markQueuedDispatchInFlightWithIdentityLocked( + identity messagequeue.QueueSessionIdentity, + entryID string, + source *messagequeue.QueuedMessage, +) *queuedDispatchReservation { + if identity.TaskID == "" || identity.SessionID == "" || + identity.SessionIncarnationID == "" || entryID == "" { + return nil + } + reservation := newQueuedDispatchReservation(identity.SessionID, entryID, identity, source) + if previous, ok := s.dispatchingQueued.Load(identity.SessionID); ok { + if previousReservation, ok := previous.(*queuedDispatchReservation); ok { + previousReservation.phase.Store(uint32(queuedDispatchSupersededByNewDispatch)) + } + } + s.acceptedQueuedDispatch.Delete(identity.SessionID) + s.dispatchingQueued.Store(identity.SessionID, reservation) + return reservation +} + +func (reservation *queuedDispatchReservation) matchesSessionIdentity( + taskID, sessionID, incarnationID string, +) bool { + if reservation == nil || reservation.identity.SessionIncarnationID == "" { + return true + } + return reservation.identity.TaskID == taskID && + reservation.identity.SessionID == sessionID && + reservation.identity.SessionIncarnationID == incarnationID +} + func (s *Service) pendingQueuedDispatch(sessionID string) *queuedDispatchReservation { value, ok := s.dispatchingQueued.Load(sessionID) if !ok { diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index a47e6787cd..a7f6378754 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -80,6 +80,14 @@ type AttachmentReader interface { OpenClaimed(ctx context.Context, id, taskID, sessionID string) (io.ReadCloser, string, string, int64, error) } +type sessionAttachmentDeleter interface { + DeleteTaskSessionWithAttachments(context.Context, *models.TaskSession) ([]*models.TaskMessageAttachment, error) +} + +type attachmentBytesRemover interface { + RemoveBytes([]*models.TaskMessageAttachment) +} + // LaunchAttachmentClaimer is the narrow task-service seam used to bind staged // attachment descriptors to an authorized task/session before launch dispatch. type LaunchAttachmentClaimer interface { @@ -329,6 +337,16 @@ type sessionExecutorStore interface { UpdateTaskSessionStateIfCurrent(ctx context.Context, id string, expected, status models.TaskSessionState, errorMessage string) (bool, time.Time, error) UpdateTaskSessionState(ctx context.Context, id string, state models.TaskSessionState, errorMessage string) error ClaimPromptableTaskSessionIfActive(ctx context.Context, id string) (models.PromptableTaskSessionClaim, error) + ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, id, incarnationID string, + ) (models.PromptableTaskSessionClaim, error) + UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, id, incarnationID string, + expected, status models.TaskSessionState, + errorMessage string, + ) (bool, time.Time, error) UpdateTaskSessionBaseCommit(ctx context.Context, id string, baseCommitSHA string) error GetTaskSessionByTaskAndAgent(ctx context.Context, taskID, agentInstanceID string) (*models.TaskSession, error) UpdateTaskSessionWorktreeBranch(ctx context.Context, sessionID, branch string) error @@ -382,7 +400,7 @@ type sessionExecutorStore interface { // Session listing + delete ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error) ListNonTerminalSessionsByAgentInstance(ctx context.Context, agentInstanceID string) ([]*models.TaskSession, error) - DeleteTaskSession(ctx context.Context, id string) error + DeleteTaskSession(ctx context.Context, session *models.TaskSession) error // Messages — used by resume to backfill the initial user prompt when a // prior launch failed before recordInitialMessage ran. ListMessages(ctx context.Context, sessionID string) ([]*models.Message, error) @@ -545,6 +563,10 @@ func (o *reservedPromptCallbackOwner) stop() { o.workers.Wait() } +type passthroughDispatchToken struct { + cancel context.CancelFunc +} + // Service is the main orchestrator service type Service struct { config ServiceConfig @@ -562,7 +584,9 @@ type Service struct { watcher *watcher.Watcher // Message queue service for queueing messages while agent is running - messageQueue *messagequeue.Service + messageQueue *messagequeue.Service + passthroughDispatchMu sync.Mutex + passthroughDispatches map[string]map[*passthroughDispatchToken]struct{} // Message creator for saving agent responses messageCreator MessageCreator @@ -591,6 +615,7 @@ type Service struct { // launchAttachmentClaimer binds staged descriptors before any launch intent // can dispatch them to the runtime. Inline attachments need no claim. launchAttachmentClaimer LaunchAttachmentClaimer + attachmentReader AttachmentReader // sessionAccessCheck enforces per-user workspace scoping on the // session-keyed WS actions. Nil = unscoped. See SetSessionAccessChecker. @@ -1491,6 +1516,13 @@ func NewService( parkedLoopCtx: parkedLoopCtx, parkedLoopCancel: parkedLoopCancel, } + if registrar, ok := repo.(interface { + SetTaskQueuePurgePreparer(func(context.Context, string)) + }); ok { + registrar.SetTaskQueuePurgePreparer(func(_ context.Context, taskID string) { + s.cancelPassthroughDispatches(taskID) + }) + } // Always publish queue-status after a task-scoped queue purge so the // status-summary projector zeros queued_prompt_count. Unlike the // ephemeral purger above, this must never call PurgeTask on the SQLite @@ -1584,6 +1616,46 @@ func NewService( return s } +func (s *Service) beginPassthroughDispatch( + parent context.Context, + taskID string, +) (context.Context, func()) { + dispatchCtx, cancel := context.WithCancel(parent) + token := &passthroughDispatchToken{cancel: cancel} + s.passthroughDispatchMu.Lock() + if s.passthroughDispatches == nil { + s.passthroughDispatches = make(map[string]map[*passthroughDispatchToken]struct{}) + } + tokens := s.passthroughDispatches[taskID] + if tokens == nil { + tokens = make(map[*passthroughDispatchToken]struct{}) + s.passthroughDispatches[taskID] = tokens + } + tokens[token] = struct{}{} + s.passthroughDispatchMu.Unlock() + return dispatchCtx, func() { + s.passthroughDispatchMu.Lock() + if tokens := s.passthroughDispatches[taskID]; tokens != nil { + delete(tokens, token) + if len(tokens) == 0 { + delete(s.passthroughDispatches, taskID) + } + } + s.passthroughDispatchMu.Unlock() + cancel() + } +} + +func (s *Service) cancelPassthroughDispatches(taskID string) { + s.passthroughDispatchMu.Lock() + tokens := s.passthroughDispatches[taskID] + delete(s.passthroughDispatches, taskID) + s.passthroughDispatchMu.Unlock() + for token := range tokens { + token.cancel() + } +} + // SetMessageCreator sets the message creator for saving agent responses to the database. // // This must be called before starting the orchestrator if you want agent messages, tool calls, @@ -1619,6 +1691,7 @@ func (s *Service) SetSubagentContextRecorder(r SubagentContextRecorder) { // SetAttachmentReader wires the backend attachment store into passthrough // prompt delivery so claimed descriptors can be streamed into the workspace. func (s *Service) SetAttachmentReader(reader AttachmentReader) { + s.attachmentReader = reader s.executor.SetAttachmentReader(reader) } diff --git a/apps/backend/internal/orchestrator/session_ensure_office_test.go b/apps/backend/internal/orchestrator/session_ensure_office_test.go index 022ff7a490..d2a2fffc56 100644 --- a/apps/backend/internal/orchestrator/session_ensure_office_test.go +++ b/apps/backend/internal/orchestrator/session_ensure_office_test.go @@ -45,6 +45,20 @@ func seedOfficeTaskAndSessions(t *testing.T, repo officeSeedRepo) { } } +func deleteOfficeTestSession(t *testing.T, repo interface { + GetTaskSession(context.Context, string) (*models.TaskSession, error) + DeleteTaskSession(context.Context, *models.TaskSession) error +}, ctx context.Context, id string) { + t.Helper() + session, err := repo.GetTaskSession(ctx, id) + if err != nil { + t.Fatalf("GetTaskSession %s: %v", id, err) + } + if err := repo.DeleteTaskSession(ctx, session); err != nil { + t.Fatalf("DeleteTaskSession %s: %v", id, err) + } +} + func TestFindExistingSession_OfficeTaskResolvesToAssignee(t *testing.T) { repo := setupTestRepo(t) seedOfficeTaskAndSessions(t, repo) @@ -82,9 +96,7 @@ func TestFindExistingSession_OfficeTaskWithViewerAgent(t *testing.T) { func TestFindExistingSession_OfficeTaskWithOnlyOtherAgentSessionReturnsNil(t *testing.T) { repo := setupTestRepo(t) seedOfficeTaskAndSessions(t, repo) - if err := repo.DeleteTaskSession(context.Background(), "s-assignee"); err != nil { - t.Fatalf("delete assignee session: %v", err) - } + deleteOfficeTestSession(t, repo, context.Background(), "s-assignee") svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) if got := svc.findExistingSession(context.Background(), "t-office"); got != nil { @@ -96,9 +108,7 @@ func TestFindExistingSession_OfficeTaskUsesResolvedMetadataProfile(t *testing.T) repo := setupTestRepo(t) ctx := context.Background() seedOfficeTaskAndSessions(t, repo) - if err := repo.DeleteTaskSession(ctx, "s-assignee"); err != nil { - t.Fatalf("delete assignee session: %v", err) - } + deleteOfficeTestSession(t, repo, ctx, "s-assignee") task, err := repo.GetTask(ctx, "t-office") if err != nil { t.Fatalf("get task: %v", err) @@ -123,12 +133,8 @@ func TestPrepareTaskSession_OfficeFlagUsesAssigneeForSessionIdentity(t *testing. ctx := context.Background() repo := setupTestRepo(t) seedOfficeTaskAndSessions(t, repo) - if err := repo.DeleteTaskSession(ctx, "s-assignee"); err != nil { - t.Fatalf("delete assignee session: %v", err) - } - if err := repo.DeleteTaskSession(ctx, "s-reviewer"); err != nil { - t.Fatalf("delete reviewer session: %v", err) - } + deleteOfficeTestSession(t, repo, ctx, "s-assignee") + deleteOfficeTestSession(t, repo, ctx, "s-reviewer") taskRepo := newMockTaskRepo() taskRepo.tasks["t-office"] = &v1.Task{ diff --git a/apps/backend/internal/orchestrator/steer_test.go b/apps/backend/internal/orchestrator/steer_test.go index 49dc932a12..0cab201151 100644 --- a/apps/backend/internal/orchestrator/steer_test.go +++ b/apps/backend/internal/orchestrator/steer_test.go @@ -30,6 +30,19 @@ func (r *blockingQueueRepository) Insert(ctx context.Context, msg *messagequeue. return r.Repository.Insert(ctx, msg, maxPerSession) } +func (r *blockingQueueRepository) InsertForSession( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + msg *messagequeue.QueuedMessage, + maxPerSession int, +) error { + r.insertOnce.Do(func() { + close(r.insertStarted) + <-r.releaseInsert + }) + return r.Repository.InsertForSession(ctx, identity, msg, maxPerSession) +} + // TestSteerEligible_GatedByFlagCapabilityAndState covers every condition the // steer gate depends on. The matrix is the observable contract behind the // composer's steer-vs-queue affordance. @@ -511,6 +524,10 @@ func TestDrainPaths_DeferToInFlightSteer(t *testing.T) { t.Fatalf("seed queued message: %v", err) } entryID := svc.messageQueue.GetStatus(ctx, sessionID).Entries[0].ID + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + t.Fatalf("resolve queue identity: %v", err) + } // Simulate the race window directly, the same way TestSteerTask_SingleInFlight // does: a steer has claimed the in-flight slot but not yet dispatched. @@ -526,7 +543,7 @@ func TestDrainPaths_DeferToInFlightSteer(t *testing.T) { }) t.Run("takeIfPromptableLocked backs off", func(t *testing.T) { - dispatched, err := svc.takeIfPromptableLocked(ctx, taskID, sessionID, entryID) + dispatched, err := svc.takeIfPromptableLocked(ctx, identity, entryID) if err != nil { t.Fatalf("takeIfPromptableLocked: %v", err) } diff --git a/apps/backend/internal/orchestrator/step_handoff_carry.go b/apps/backend/internal/orchestrator/step_handoff_carry.go index 6627776392..7172a73593 100644 --- a/apps/backend/internal/orchestrator/step_handoff_carry.go +++ b/apps/backend/internal/orchestrator/step_handoff_carry.go @@ -49,8 +49,16 @@ func (s *Service) drainQueuedMessageForPromptableSessionWithHandoff( if s.messageQueue == nil || s.isQueuedDispatchInFlight(sessionID) || s.isSteerInFlight(sessionID) { return false } - queuedMsg, ok, autoRun := s.messageQueue.ReserveQueuedWithAutoRun(ctx, sessionID) - if !autoRun || !ok || queuedMsg == nil { + identity := messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, + SessionID: session.ID, + SessionIncarnationID: session.QueueIncarnationID, + } + if identity.TaskID != taskID || identity.SessionIncarnationID == "" { + return false + } + queuedMsg, ok, autoRun, err := s.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + if err != nil || !autoRun || !ok || queuedMsg == nil { return false } if queuedMsg.Content != "" || len(queuedMsg.Attachments) > 0 { @@ -58,7 +66,7 @@ func (s *Service) drainQueuedMessageForPromptableSessionWithHandoff( queuedMsg = withStepHandoffMetadata(queuedMsg, handoffText) } } - return s.dispatchTakenQueuedMessage(ctx, sessionID, queuedMsg, ok) + return s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok) } // withStepHandoffMetadata returns a shallow copy of msg carrying handoffText diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index 53759ac878..bcc5f284d6 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -1529,7 +1529,11 @@ func (s *Service) prepareSessionForStart( // session row. Compensate before returning so callers never observe a // partial sibling session when the required parent/group workspace is // unavailable. - if deleteErr := s.repo.DeleteTaskSession(ctx, sessionID); deleteErr != nil { + createdSession, lookupErr := s.repo.GetTaskSession(ctx, sessionID) + if lookupErr != nil { + s.logger.Warn("failed to load inherited workspace session for compensation", + zap.String("session_id", sessionID), zap.Error(lookupErr)) + } else if deleteErr := s.deleteSessionAndCleanAttachments(ctx, createdSession); deleteErr != nil { s.logger.Warn("failed to compensate inherited workspace session", zap.String("session_id", sessionID), zap.Error(deleteErr)) } @@ -3495,10 +3499,22 @@ func (s *Service) DeleteSession(ctx context.Context, sessionID string) error { zap.String("task_id", taskID), zap.String("state", string(session.State)), zap.Bool("was_primary", wasPrimary)) - - if err := s.deleteSessionAndPublishError(ctx, taskID, sessionID); err != nil { + if err := s.deleteSessionAndPublishError(ctx, session); err != nil { return fmt.Errorf("failed to delete session: %w", err) } + // Remove ephemeral state only when it is still bound to the deleted + // incarnation. This purge includes reserved rows and cannot erase a + // replacement session that reuses the textual session ID. + if s.messageQueue != nil { + identity := messagequeue.QueueSessionIdentity{ + TaskID: taskID, + SessionID: sessionID, + SessionIncarnationID: session.QueueIncarnationID, + } + if err := s.messageQueue.PurgeDeletedSession(ctx, identity); err != nil { + return fmt.Errorf("failed to purge deleted session queue: %w", err) + } + } // The row is gone, so retire the detached-launch attestation and parked // projection before any later session event can reuse this ID. s.clearParkedProjectionOnSessionDeleted(ctx, taskID, sessionID) @@ -3518,11 +3534,9 @@ func (s *Service) DeleteSession(ctx context.Context, sessionID string) error { // through a stale activity pointer. s.clearTurnActivity(sessionID) - // Pending queue rows are keyed by session_id but counted for the sidebar - // badge by task_id. Cancel them after the session row is gone and publish - // queue-status so the status-summary projector drops the orphaned count. - // Best-effort: the session delete already committed. - s.cancelDeletedSessionQueue(ctx, taskID, sessionID) + // Queue rows and per-session policy are removed atomically with the task + // session row. Publish only the task-scoped recount after commit. + s.publishTaskQueueStatusEvent(ctx, taskID, "") // Auto-promote another session if we deleted the primary if wasPrimary { @@ -3532,16 +3546,33 @@ func (s *Service) DeleteSession(ctx context.Context, sessionID string) error { return nil } -func (s *Service) deleteSessionAndPublishError(ctx context.Context, taskID, sessionID string) error { - lock, release := s.acquireTaskSessionErrorGuard(taskID) +func (s *Service) deleteSessionAndCleanAttachments(ctx context.Context, session *models.TaskSession) error { + var deletedAttachments []*models.TaskMessageAttachment + var err error + if deleter, ok := s.repo.(sessionAttachmentDeleter); ok { + deletedAttachments, err = deleter.DeleteTaskSessionWithAttachments(ctx, session) + } else { + err = s.repo.DeleteTaskSession(ctx, session) + } + if err != nil { + return err + } + if remover, ok := s.attachmentReader.(attachmentBytesRemover); ok { + remover.RemoveBytes(deletedAttachments) + } + return nil +} + +func (s *Service) deleteSessionAndPublishError(ctx context.Context, session *models.TaskSession) error { + lock, release := s.acquireTaskSessionErrorGuard(session.TaskID) defer release() lock.Lock() defer lock.Unlock() - if err := s.repo.DeleteTaskSession(ctx, sessionID); err != nil { + if err := s.deleteSessionAndCleanAttachments(ctx, session); err != nil { return err } - s.publishDeletedSessionError(ctx, taskID, sessionID) + s.publishDeletedSessionError(ctx, session.TaskID, session.ID) return nil } @@ -3644,21 +3675,6 @@ func (s *Service) publishTaskSessionErrorEvent( )) } -// cancelDeletedSessionQueue removes pending prompts left on a deleted session -// and publishes a task-scoped queue status event for the live badge path. -func (s *Service) cancelDeletedSessionQueue(ctx context.Context, taskID, sessionID string) { - if s.messageQueue == nil { - return - } - if _, err := s.messageQueue.CancelAll(ctx, sessionID); err != nil { - s.logger.Warn("failed to cancel queued prompts after session delete", - zap.String("session_id", sessionID), - zap.String("task_id", taskID), - zap.Error(err)) - } - s.publishTaskQueueStatusEvent(ctx, taskID, sessionID) -} - // quiesceSessionExecutionBeforeDeletion stops the in-memory lifecycle // execution, if one exists, before a session row is removed. A runtime that // explicitly reports the exact execution as absent is already quiesced; other @@ -4236,7 +4252,8 @@ type promptTaskOptions struct { // onAccepted runs at the agentctl acceptance boundary, before PromptTask // waits for the turn to finish. Automation callers use it to bind durable // attempt identity to the exact turn. - onAccepted func(turnID string) + onAccepted func(turnID string) + expectedSessionIdentity *messagequeue.QueueSessionIdentity // promptAlreadyComposed and fallbackRetryPrompt mirror the composed-prompt // seam autoStartStepPrompt's own ErrExecutionNotFound branch uses (see // fallbackFreshLaunchOnMissingExecution). When promptAlreadyComposed is @@ -4425,7 +4442,7 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom ctx, taskID, sessionID, options.claimEntryID, options.lifecyclePrompt, options.reserveTurnUntilDispatch, options.promptDispatchRecovery, options.afterClaim, foregroundClaim, options.expectedCurrentTurnID, - options.requireNonterminalSession, + options.requireNonterminalSession, options.expectedSessionIdentity, ) if err != nil { s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch) @@ -4482,6 +4499,15 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom // Only bounded-ack callers preserve request context; ordinary prompts can take minutes. promptCtx := options.executorContext(ctx) + session, identityValidationErr := s.validateQueuedPromptDispatch( + promptCtx, rollback.sessionIdentity, taskID, sessionID, session, + ) + if identityValidationErr != nil { + return s.finishPromptDispatchFailure( + ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, + rollback, options, identityValidationErr, foregroundDispatch, false, nil, + ) + } s.bindPromptTurnID(promptCtx, session, rollback.turnID) s.beginInteractivePromptAttempt( promptCtx, @@ -4492,8 +4518,9 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom dispatchOutcome := &promptDispatchOutcome{} dispatchOutcome.turnID = rollback.turnID dispatchOutcome.onAccepted = options.onAccepted - onDispatched := s.promptDispatchCallback( - promptCtx, taskID, sessionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome, + onDispatched := s.promptDispatchCallbackForIdentity( + promptCtx, taskID, sessionID, rollback.sessionIdentity, + session.AgentExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome, ) if releaseDispatchGuard != nil { originalOnDispatched := onDispatched @@ -4683,9 +4710,40 @@ func (s *Service) promptDispatchCallback( reservedTurn *models.Turn, dispatch *foregroundDispatch, outcome *promptDispatchOutcome, +) func() { + return s.promptDispatchCallbackForIdentity( + ctx, taskID, sessionID, messagequeue.QueueSessionIdentity{}, "", + reservedTurn, dispatch, outcome, + ) +} + +func (s *Service) promptDispatchIdentityIsCurrent( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + expectedExecutionID, executionID string, +) bool { + if identity.SessionIncarnationID == "" { + return true + } + current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID) + return err == nil && current == identity && executionID == expectedExecutionID +} + +func (s *Service) promptDispatchCallbackForIdentity( + ctx context.Context, + taskID, sessionID string, + identity messagequeue.QueueSessionIdentity, + expectedExecutionID string, + reservedTurn *models.Turn, + dispatch *foregroundDispatch, + outcome *promptDispatchOutcome, ) func() { return func() { executionID, _ := s.agentManager.GetExecutionIDForSession(ctx, sessionID) + if !s.promptDispatchIdentityIsCurrent(ctx, identity, expectedExecutionID, executionID) { + outcome.recordAccepted(nil) + return + } s.bindPromptAttemptToExecution(ctx, sessionID, executionID) var publicationErr error if reservedTurn != nil && s.turnService != nil { @@ -4752,6 +4810,7 @@ type promptClaimRollback struct { previousSessionState models.TaskSessionState previousTaskState v1.TaskState taskStateClaimed bool + sessionIdentity messagequeue.QueueSessionIdentity turnID string createdTurn bool reservedTurn *models.Turn @@ -4769,7 +4828,12 @@ func (s *Service) claimPromptDispatch( foregroundClaim *foregroundClaim, expectedCurrentTurnID string, requireNonterminalSession bool, + expectedIdentities ...*messagequeue.QueueSessionIdentity, ) (*models.TaskSession, promptClaimRollback, error) { + var expectedIdentity *messagequeue.QueueSessionIdentity + if len(expectedIdentities) > 0 { + expectedIdentity = expectedIdentities[0] + } if lifecyclePrompt { return s.claimLifecyclePromptDispatch( ctx, taskID, sessionID, claimEntryID, afterClaim, @@ -4778,6 +4842,7 @@ func (s *Service) claimPromptDispatch( claimed, previousState, turnID, createdTurn, reservedTurn, err := s.claimSessionRunningForPrompt( ctx, taskID, sessionID, claimEntryID, reserveTurnUntilDispatch, promptDispatchRecovery, foregroundClaim, expectedCurrentTurnID, requireNonterminalSession, + expectedIdentity, afterClaim, ) if err != nil { return nil, promptClaimRollback{}, err @@ -4788,6 +4853,12 @@ func (s *Service) claimPromptDispatch( createdTurn: createdTurn, reservedTurn: reservedTurn, } + if reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID); reservation != nil { + rollback.sessionIdentity = reservation.identity + } + if expectedIdentity != nil { + rollback.sessionIdentity = *expectedIdentity + } return claimed, rollback, nil } @@ -4807,6 +4878,9 @@ func (s *Service) claimLifecyclePromptDispatch( turnID: turnID, createdTurn: createdTurn, } + if reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID); reservation != nil { + rollback.sessionIdentity = reservation.identity + } rollback.previousTaskState, rollback.taskStateClaimed, err = s.reconcileLifecycleClaim( ctx, taskID, sessionID, previousSessionState, session, ) @@ -4874,7 +4948,11 @@ func (s *Service) rollbackPromptClaim( if rollback.dispatchGuardRelease != nil { rollback.dispatchGuardRelease() } - s.restoreLifecycleClaim(ctx, taskID, sessionID, rollback.previousSessionState) + if rollback.sessionIdentity.SessionIncarnationID != "" { + s.restoreLifecycleClaimForIdentity(ctx, rollback.sessionIdentity, rollback.previousSessionState) + } else { + s.restoreLifecycleClaim(ctx, taskID, sessionID, rollback.previousSessionState) + } if rollback.taskStateClaimed { s.restoreLifecycleTaskState(ctx, taskID, rollback.previousTaskState) } @@ -5124,7 +5202,20 @@ func (s *Service) claimSessionRunningForPrompt( foregroundClaim *foregroundClaim, expectedCurrentTurnID string, requireNonterminalSession bool, + optionalClaimArgs ...interface{}, ) (*models.TaskSession, models.TaskSessionState, string, bool, *models.Turn, error) { + var ( + afterClaim func() error + expectedIdentity *messagequeue.QueueSessionIdentity + ) + for _, arg := range optionalClaimArgs { + switch value := arg.(type) { + case func() error: + afterClaim = value + case *messagequeue.QueueSessionIdentity: + expectedIdentity = value + } + } lock, release := s.acquireCancelInFlightGuard(sessionID) defer release() lock.Lock() @@ -5170,6 +5261,20 @@ func (s *Service) claimSessionRunningForPrompt( if freshSession == nil { return nil, "", "", false, nil, errQueuedDispatchSuperseded } + reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID) + if !reservation.matchesSessionIdentity( + freshSession.TaskID, + freshSession.ID, + freshSession.QueueIncarnationID, + ) { + return nil, "", "", false, nil, messagequeue.ErrSessionIdentityMismatch + } + if expectedIdentity != nil && + (freshSession.TaskID != expectedIdentity.TaskID || + freshSession.ID != expectedIdentity.SessionID || + freshSession.QueueIncarnationID != expectedIdentity.SessionIncarnationID) { + return nil, "", "", false, nil, messagequeue.ErrSessionIdentityMismatch + } if requireNonterminalSession && isTerminalSessionState(freshSession.State) { return nil, "", "", false, nil, newWorkflowAutoStartSessionTerminalizedError(freshSession) } @@ -5179,10 +5284,20 @@ func (s *Service) claimSessionRunningForPrompt( return nil, "", "", false, nil, promptErr } previousState := freshSession.State - s.setSessionRunning(ctx, taskID, sessionID, freshSession) - // setSessionRunning refreshes freshSession in place when its guarded write - // loses, so a cancellation landing after the promptability check is visible - // here as a terminal state. + switch { + case reservation != nil && reservation.identity.SessionIncarnationID != "": + if err := s.setQueuedSessionRunningForIdentity(ctx, reservation.identity, freshSession); err != nil { + return nil, "", "", false, nil, err + } + case expectedIdentity != nil: + if err := s.setQueuedSessionRunningForIdentity(ctx, *expectedIdentity, freshSession); err != nil { + return nil, "", "", false, nil, err + } + default: + s.setSessionRunning(ctx, taskID, sessionID, freshSession) + } + // The guarded state transition refreshes freshSession in place, so a + // competing terminal transition remains visible before turn creation. if requireNonterminalSession && isTerminalSessionState(freshSession.State) { return nil, "", "", false, nil, newWorkflowAutoStartSessionTerminalizedError(freshSession) } @@ -5196,9 +5311,14 @@ func (s *Service) claimSessionRunningForPrompt( ctx, sessionID, reserveTurnUntilDispatch, promptDispatchRecovery, ) if err != nil { - s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, promptClaimRollback{ - previousSessionState: previousState, - }) + rollback := promptClaimRollback{previousSessionState: previousState} + if reservation != nil { + rollback.sessionIdentity = reservation.identity + } + if expectedIdentity != nil { + rollback.sessionIdentity = *expectedIdentity + } + s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback) return nil, "", "", false, nil, fmt.Errorf("persist prompt turn: %w", err) } if reservedTurn != nil && s.turnService != nil { @@ -5209,30 +5329,80 @@ func (s *Service) claimSessionRunningForPrompt( createdTurn: createdTurn, reservedTurn: reservedTurn, } + if reservation != nil { + rollback.sessionIdentity = reservation.identity + } + if expectedIdentity != nil { + rollback.sessionIdentity = *expectedIdentity + } s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback) return nil, "", "", false, nil, fmt.Errorf("persist prompt dispatch attempt: %w", err) } } - reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID) + reservation = s.queuedDispatchReservationForEntry(sessionID, claimEntryID) + rollback := promptClaimRollback{ + previousSessionState: previousState, + turnID: turnID, + createdTurn: createdTurn, + reservedTurn: reservedTurn, + } + if reservation != nil { + rollback.sessionIdentity = reservation.identity + } + if expectedIdentity != nil { + rollback.sessionIdentity = *expectedIdentity + } + if err := s.acknowledgePromptClaim(ctx, taskID, sessionID, afterClaim, rollback); err != nil { + return nil, "", "", false, nil, err + } + // A Send Now reservation becomes replaceable only after the guarded session + // claim and its identity-bound visible side effects have succeeded. if reservation != nil && reservation.liveEligible.Load() { - // A Send Now reservation becomes replaceable only after this guarded - // session claim. The turn was started/adopted under the same guard above, - // so the phase and turn ownership cannot be observed half-way through the - // handoff. s.markAcceptedDispatchLiveLocked(sessionID, reservation) } // Remove only the pre-acceptance marker here. The accepted ownership record // remains until the turn settles, so Send Now cannot cancel or duplicate // this successor while the executor call is still in progress. if claimEntryID != "" { - s.releaseQueuedDispatchPendingIfCurrent( - sessionID, - s.queuedDispatchReservationForEntry(sessionID, claimEntryID), - ) + s.releaseQueuedDispatchPendingIfCurrent(sessionID, reservation) } return freshSession, previousState, turnID, createdTurn, reservedTurn, nil } +func (s *Service) validateQueuedPromptDispatch( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + taskID, sessionID string, + session *models.TaskSession, +) (*models.TaskSession, error) { + if identity.SessionIncarnationID == "" { + return session, nil + } + if s.messageQueue == nil { + return nil, messagequeue.ErrSessionIdentityMismatch + } + expectedExecutionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID) + if err != nil || expectedExecutionID == "" { + return nil, executor.ErrExecutionNotFound + } + fresh, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return nil, err + } + if fresh == nil || + fresh.TaskID != taskID || + fresh.ID != identity.SessionID || + fresh.QueueIncarnationID != identity.SessionIncarnationID { + return nil, messagequeue.ErrSessionIdentityMismatch + } + currentExecutionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID) + if err != nil || currentExecutionID != expectedExecutionID { + return nil, executor.ErrExecutionNotFound + } + fresh.AgentExecutionID = expectedExecutionID + return fresh, nil +} + func (s *Service) claimLifecycleSessionRunning( ctx context.Context, taskID, sessionID, claimEntryID string, @@ -5262,7 +5432,8 @@ func (s *Service) claimLifecycleSessionRunning( if err := s.validateLifecyclePromptSelection(ctx, taskID, sessionID); err != nil { return nil, "", "", false, err } - claim, err := s.repo.ClaimPromptableTaskSessionIfActive(ctx, sessionID) + reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID) + claim, err := s.claimPromptableSessionForIdentity(ctx, taskID, sessionID, reservation) if err != nil { return nil, "", "", false, fmt.Errorf("%w: %v", errLifecyclePromptClaim, err) } @@ -5273,18 +5444,26 @@ func (s *Service) claimLifecycleSessionRunning( return nil, "", "", false, fmt.Errorf("%w: lifecycle prompt session is busy", ErrAgentPromptInProgress) } if claimEntryID != "" && !s.isCurrentQueuedDispatch(sessionID, claimEntryID) { - s.restoreLifecycleClaim(ctx, taskID, sessionID, claim.PreviousState) + s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState) return nil, "", "", false, errQueuedDispatchSuperseded } if s.isSessionResetInProgress(sessionID) { - s.restoreLifecycleClaim(ctx, taskID, sessionID, claim.PreviousState) + s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState) return nil, "", "", false, ErrSessionResetInProgress } freshSession, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil || freshSession == nil { - s.restoreLifecycleClaim(ctx, taskID, sessionID, claim.PreviousState) + s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState) return nil, "", "", false, errLifecyclePromptInactive } + if !reservation.matchesSessionIdentity( + freshSession.TaskID, + freshSession.ID, + freshSession.QueueIncarnationID, + ) { + s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState) + return nil, "", "", false, messagequeue.ErrSessionIdentityMismatch + } // Keep the lifecycle admission guard through durable turn creation. A reset // that follows this claim must observe this turn and quiesce it before it // replaces the provider context. @@ -5292,16 +5471,89 @@ func (s *Service) claimLifecycleSessionRunning( if err != nil { failureCtx, cancel := (promptTaskOptions{}).failureContext(ctx) defer cancel() - s.restoreLifecycleClaim(failureCtx, taskID, sessionID, claim.PreviousState) + s.restoreLifecycleClaimForReservation( + failureCtx, taskID, sessionID, reservation, claim.PreviousState, + ) return nil, "", "", false, fmt.Errorf("persist lifecycle prompt turn: %w", err) } - s.releaseQueuedDispatchPendingIfCurrent( - sessionID, - s.queuedDispatchReservationForEntry(sessionID, claimEntryID), - ) + s.releaseQueuedDispatchPendingIfCurrent(sessionID, reservation) return freshSession, claim.PreviousState, turnID, createdTurn, nil } +func (s *Service) claimPromptableSessionForIdentity( + ctx context.Context, + taskID, sessionID string, + reservation *queuedDispatchReservation, +) (models.PromptableTaskSessionClaim, error) { + if reservation == nil || reservation.identity.SessionIncarnationID == "" { + return s.repo.ClaimPromptableTaskSessionIfActive(ctx, sessionID) + } + return s.repo.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, + reservation.identity.TaskID, + reservation.identity.SessionID, + reservation.identity.SessionIncarnationID, + ) +} + +func (s *Service) restoreLifecycleClaimForReservation( + ctx context.Context, + taskID, sessionID string, + reservation *queuedDispatchReservation, + previousState models.TaskSessionState, +) { + if reservation != nil && reservation.identity.SessionIncarnationID != "" { + s.restoreLifecycleClaimForIdentity(ctx, reservation.identity, previousState) + return + } + s.restoreLifecycleClaim(ctx, taskID, sessionID, previousState) +} + +func (s *Service) restoreLifecycleClaimForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + previousState models.TaskSessionState, +) { + restored, updatedAt, err := s.repo.UpdateTaskSessionStateIfCurrentIdentity( + ctx, + identity.TaskID, + identity.SessionID, + identity.SessionIncarnationID, + models.TaskSessionStateRunning, + previousState, + "", + ) + if err != nil { + s.logger.Error("failed to restore lifecycle prompt session state", + zap.String("task_id", identity.TaskID), + zap.String("session_id", identity.SessionID), + zap.Error(err)) + return + } + if !restored { + return + } + session, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil || session == nil || + session.TaskID != identity.TaskID || + session.QueueIncarnationID != identity.SessionIncarnationID { + return + } + s.publishTaskSessionStateChanged( + ctx, + identity.TaskID, + identity.SessionID, + models.TaskSessionStateRunning, + previousState, + "", + &updatedAt, + session, + ) + s.republishTaskActivityOnSettle( + ctx, identity.TaskID, models.TaskSessionStateRunning, previousState, + ) +} + func (s *Service) validateLifecyclePromptSelection( ctx context.Context, taskID, sessionID string, @@ -5579,6 +5831,51 @@ func (s *Service) DrainQueuedMessage(ctx context.Context, sessionID string) (boo return s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID), nil } +// DrainQueuedMessageForSession dispatches one message only for the exact session incarnation. +func (s *Service) DrainQueuedMessageForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity) (bool, error) { + if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" { + return false, messagequeue.ErrSessionIdentityMismatch + } + lock, release := s.acquireCancelInFlightGuard(identity.SessionID) + defer release() + lock.Lock() + guardLocked := true + defer func() { + if guardLocked { + lock.Unlock() + } + }() + for s.isCancelInFlight(identity.SessionID) { + lock.Unlock() + guardLocked = false + if err := s.waitForCancelInFlight(ctx, identity.SessionID); err != nil { + return false, err + } + lock.Lock() + guardLocked = true + } + session, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil { + return false, fmt.Errorf("failed to get session: %w", err) + } + if session == nil || session.TaskID != identity.TaskID { + return false, messagequeue.ErrSessionIdentityMismatch + } + if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil { + return false, err + } + if s.sessionHasPendingClarification(ctx, identity.SessionID) { + return false, ErrSessionNotPromptable + } + if s.messageQueue == nil { + return false, errors.New("message queue is not configured") + } + if err := s.messageQueue.SetAutoRunForSession(ctx, identity, true); err != nil { + return false, fmt.Errorf("resume queue Auto-run: %w", err) + } + return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity) +} + // SetQueueAutoRun persists the queue policy and, when enabling an eligible // session, immediately attempts the FIFO head. It never cancels an active turn // and treats busy, clarification, and lifecycle guards as a successful armed @@ -5618,6 +5915,115 @@ func (s *Service) SetQueueAutoRun(ctx context.Context, sessionID string, enabled return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, dispatched, nil } +func (s *Service) queueAutoRunDrainBlocked(sessionID string) bool { + return s.isCancelInFlight(sessionID) || + s.isQueuedDispatchInFlight(sessionID) || + s.isSteerInFlight(sessionID) +} + +// SetQueueAutoRunForSession persists policy and dispatches only for the exact session incarnation. +func (s *Service) SetQueueAutoRunForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity, enabled bool) (bool, bool, error) { + if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" { + return false, false, messagequeue.ErrSessionIdentityMismatch + } + if s.messageQueue == nil { + return false, false, errors.New("message queue is not configured") + } + lock, release := s.acquireCancelInFlightGuard(identity.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() + + if err := s.messageQueue.SetAutoRunForSession(ctx, identity, enabled); err != nil { + return false, false, fmt.Errorf("set queue Auto-run: %w", err) + } + if !enabled || s.queueAutoRunDrainBlocked(identity.SessionID) { + return enabled, false, nil + } + if s.sessionHasPendingClarification(ctx, identity.SessionID) { + return enabled, false, nil + } + session, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil { + return enabled, false, fmt.Errorf("load session for queue Auto-run: %w", err) + } + if session == nil || session.TaskID != identity.TaskID { + return enabled, false, messagequeue.ErrSessionIdentityMismatch + } + if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil { + if isSessionBusyError(err) { + return enabled, false, nil + } + return enabled, false, err + } + dispatched, err := s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity) + return enabled, dispatched, err +} + +func (s *Service) resolveQueueIdentityForSession( + ctx context.Context, + session *models.TaskSession, +) (messagequeue.QueueSessionIdentity, error) { + if s.messageQueue == nil || session == nil || session.ID == "" || session.TaskID == "" { + return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch + } + identity, err := s.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID) + if err != nil { + return messagequeue.QueueSessionIdentity{}, err + } + if session.QueueIncarnationID != "" && + session.QueueIncarnationID != identity.SessionIncarnationID { + return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch + } + return identity, nil +} + +func (s *Service) drainQueuedMessageForPromptableSessionForIdentity( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, +) (bool, error) { + lock, release := s.acquireCancelInFlightGuard(identity.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() + if s.isCancelInFlight(identity.SessionID) { + return false, nil + } + session, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil { + return false, err + } + if session == nil || + session.TaskID != identity.TaskID || + session.QueueIncarnationID != identity.SessionIncarnationID { + return false, messagequeue.ErrSessionIdentityMismatch + } + if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil { + if isSessionBusyError(err) { + return false, nil + } + return false, err + } + if s.sessionHasPendingClarification(ctx, identity.SessionID) { + return false, nil + } + return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity) +} + +func (s *Service) drainQueuedMessageForPromptableSessionLockedForIdentity(ctx context.Context, identity messagequeue.QueueSessionIdentity) (bool, error) { + if s.isCancelInFlight(identity.SessionID) || s.isQueuedDispatchInFlight(identity.SessionID) || s.isSteerInFlight(identity.SessionID) { + return false, nil + } + queuedMsg, ok, autoRun, err := s.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity) + if err != nil { + return false, err + } + if !autoRun { + return false, nil + } + return s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok), nil +} + // cancelInFlightGuard is a per-session mutex serializing cancel/interrupt/ // queue-take decisions (see the Service.cancelInFlight field doc comment), // paired with a reference count so the registry can safely reclaim the @@ -6730,7 +7136,19 @@ func (s *Service) finishCancelledAgentTurn(ctx context.Context, sessionID string return fmt.Errorf("reconcile cancelled turn: %w", err) } if s.messageQueue != nil { - paused, err := s.messageQueue.PauseAutoRunIfPending(ctx, sessionID) + var ( + paused bool + err error + ) + if session != nil { + var identity messagequeue.QueueSessionIdentity + identity, err = s.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, sessionID) + if err == nil { + paused, err = s.messageQueue.PauseAutoRunIfPendingForSession(ctx, identity) + } + } else { + paused, err = s.messageQueue.PauseAutoRunIfPending(ctx, sessionID) + } if err != nil { s.logger.Warn("failed to pause queued messages after cancel", zap.String("session_id", sessionID), @@ -6812,17 +7230,21 @@ func (s *Service) reconcileCancelledTaskReview(ctx context.Context, taskID, sess // which checks the marker itself before delegating here). See // drainQueuedMessageForPromptableSessionLocked for the *non-cancelling* // FIFO-head drain path, which does check the marker directly. -func (s *Service) takeAndDispatchEntryLocked(ctx context.Context, sessionID, entryID string) (bool, error) { +func (s *Service) takeAndDispatchEntryLocked( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + entryID string, +) (bool, error) { if s.messageQueue != nil { - queuedMsg, ok, err := s.messageQueue.TakeQueuedEntry(ctx, sessionID, entryID) + queuedMsg, ok, err := s.messageQueue.TakeQueuedEntryForSession(ctx, identity, entryID) if err != nil { return false, fmt.Errorf("take targeted queued message: %w", err) } - if s.dispatchTakenQueuedMessage(ctx, sessionID, queuedMsg, ok) { + if s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok) { return true, nil } } - return s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID), nil + return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity) } // takeIfPromptableLocked takes and dispatches entryID only if sessionID is @@ -6843,15 +7265,21 @@ func (s *Service) takeAndDispatchEntryLocked(ctx context.Context, sessionID, ent // Also defers to an admitted-but-not-yet-dispatched steer (isSteerInFlight; // see steerInFlight's field doc comment) for the same reason: this path // never cancels, so it has no way to supersede the steer's dispatch either. -func (s *Service) takeIfPromptableLocked(ctx context.Context, taskID, sessionID, entryID string) (bool, error) { - if s.isQueuedDispatchInFlight(sessionID) || s.isSteerInFlight(sessionID) { +func (s *Service) takeIfPromptableLocked( + ctx context.Context, + identity messagequeue.QueueSessionIdentity, + entryID string, +) (bool, error) { + if s.isQueuedDispatchInFlight(identity.SessionID) || s.isSteerInFlight(identity.SessionID) { return false, nil } - session, err := s.repo.GetTaskSession(ctx, sessionID) - if err != nil || session == nil || s.checkSessionPromptable(taskID, sessionID, session.State) != nil { + session, err := s.repo.GetTaskSession(ctx, identity.SessionID) + if err != nil || session == nil || + session.QueueIncarnationID != identity.SessionIncarnationID || + s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State) != nil { return false, nil } - return s.takeAndDispatchEntryLocked(ctx, sessionID, entryID) + return s.takeAndDispatchEntryLocked(ctx, identity, entryID) } // cancelAndTakeForPeerMessage cancels sessionID's in-flight turn and takes @@ -6880,28 +7308,31 @@ func (s *Service) takeIfPromptableLocked(ctx context.Context, taskID, sessionID, // task to REVIEW (writeTaskReviewStateOnCancel). func (s *Service) cancelAndTakeForPeerMessage( ctx context.Context, - taskID string, - sessionID string, + identity messagequeue.QueueSessionIdentity, entryID string, unlockGuard func(), relockGuard func(), ) (bool, error) { return s.cancelAgentSilentWithGuardActionKind( ctx, - taskID, - sessionID, + identity.TaskID, + identity.SessionID, unlockGuard, relockGuard, func(actionCtx context.Context) (bool, error) { - return s.takeAndDispatchEntryLocked(actionCtx, sessionID, entryID) + return s.takeAndDispatchEntryLocked(actionCtx, identity, entryID) }, cancellationKindPeer, ) } -// QueueAndInterruptForPeerMessage atomically queues prompt for sessionID -// and then interrupts the child's in-flight turn to deliver it immediately, -// instead of waiting for the turn to end naturally. It is used only by +// QueueAndInterruptForPeerMessage accepts the caller-captured session identity, +// validates it after acquiring the session guard, atomically queues prompt, and +// then interrupts the child's in-flight turn to deliver it immediately, +// instead of waiting for the turn to end naturally. Capturing the identity +// before contending for the guard prevents a delete-and-recreate race from +// retargeting an old peer operation into a replacement session incarnation. +// It is used only by // handleMessageTask (mcp/handlers) when the sender is the target task's // parent: a long-running child otherwise leaves its parent's control/steer // messages parked on the FIFO queue until the child's current turn finishes @@ -6965,7 +7396,9 @@ func (s *Service) cancelAndTakeForPeerMessage( // hasn't yet reached PromptTask — so it must still be allowed to cancel // and take its own entry rather than being blocked by a marker the cancel // itself is about to invalidate. -func (s *Service) QueueAndInterruptForPeerMessage(ctx context.Context, taskID, sessionID, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { +func (s *Service) QueueAndInterruptForPeerMessage(ctx context.Context, identity messagequeue.QueueSessionIdentity, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) { + taskID := identity.TaskID + sessionID := identity.SessionID if s.messageQueue == nil { return nil, false, errors.New("message queue not available") } @@ -6998,11 +7431,13 @@ func (s *Service) QueueAndInterruptForPeerMessage(ctx context.Context, taskID, s release() }() - queued, err := s.messageQueue.QueueMessageWithMetadata(ctx, sessionID, taskID, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata) + queued, err := s.messageQueue.QueueMessageWithMetadataForSession( + ctx, identity, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata, + ) if err != nil { return nil, false, err } - s.publishQueueStatusEvent(ctx, sessionID) + s.publishQueueStatusEventForIdentity(ctx, identity) if operation := s.currentCancellation(sessionID); operation != nil && operation.kind != cancellationKindPeer { s.logger.Debug("leaving peer message queued while a different cancellation source is in progress", zap.String("task_id", taskID), @@ -7024,12 +7459,12 @@ func (s *Service) QueueAndInterruptForPeerMessage(ctx context.Context, taskID, s zap.String("session_id", sessionID), zap.String("turn_before_wait", turnBeforeWait), zap.Bool("turn_peek_error", turnPeekErr != nil || turnNowErr != nil)) - dispatched, fallbackErr := s.takeIfPromptableLocked(ctx, taskID, sessionID, queued.ID) + dispatched, fallbackErr := s.takeIfPromptableLocked(ctx, identity, queued.ID) return queued, dispatched, fallbackErr } } - dispatched, err := s.cancelAndTakeForPeerMessage(ctx, taskID, sessionID, queued.ID, unlockGuard, relockGuard) + dispatched, err := s.cancelAndTakeForPeerMessage(ctx, identity, queued.ID, unlockGuard, relockGuard) return queued, dispatched, err } diff --git a/apps/backend/internal/orchestrator/task_operations_auto_merge_test.go b/apps/backend/internal/orchestrator/task_operations_auto_merge_test.go index 53a2593066..d6f1d414e9 100644 --- a/apps/backend/internal/orchestrator/task_operations_auto_merge_test.go +++ b/apps/backend/internal/orchestrator/task_operations_auto_merge_test.go @@ -31,9 +31,7 @@ func TestQueueAndInterruptForPeerMessage_DispatchesSurvivingAutoMergedID(t *test t.Fatalf("queue target: %v", err) } - queued, dispatched, err := service.QueueAndInterruptForPeerMessage( - ctx, "task1", "session1", "second parent message", metadata, - ) + queued, dispatched, err := queueAndInterruptForPeerMessage(service, ctx, "task1", "session1", "second parent message", metadata) if err != nil || !dispatched { t.Fatalf("queue and interrupt: dispatched=%v err=%v", dispatched, err) } diff --git a/apps/backend/internal/orchestrator/task_operations_test.go b/apps/backend/internal/orchestrator/task_operations_test.go index 1b15c67c78..98652df683 100644 --- a/apps/backend/internal/orchestrator/task_operations_test.go +++ b/apps/backend/internal/orchestrator/task_operations_test.go @@ -2187,6 +2187,19 @@ func TestCancelAgent_QueuedMessageRunsAfterExplicitDrain(t *testing.T) { } } +func queueAndInterruptForPeerMessage( + svc *Service, + ctx context.Context, + taskID, sessionID, prompt string, + metadata map[string]interface{}, +) (*messagequeue.QueuedMessage, bool, error) { + identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID) + if err != nil { + return nil, false, err + } + return svc.QueueAndInterruptForPeerMessage(ctx, identity, prompt, metadata) +} + // --- QueueAndInterruptForPeerMessage --- // TestQueueAndInterruptForPeerMessage_DeliversQueuedMessageWithoutUserCancelSideEffects @@ -2208,7 +2221,7 @@ func TestQueueAndInterruptForPeerMessage_DeliversQueuedMessageWithoutUserCancelS seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) seedExecutorRunning(t, repo, "session1", "task1", "exec-1") - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) if err != nil { t.Fatalf("queue and interrupt for peer message: %v", err) } @@ -2293,9 +2306,7 @@ func TestQueueAndInterruptForPeerMessage_LeavesMessageQueuedDuringUserCancel(t * var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage( - ctx, "task1", "session1", "peer message during cancel", nil, - ) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "peer message during cancel", nil) close(interruptDone) }() @@ -2349,7 +2360,7 @@ func TestQueueAndInterruptForPeerMessage_CancelFailurePropagatesAndKeepsMessageQ seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) seedExecutorRunning(t, repo, "session1", "task1", "exec-1") - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) if err == nil { t.Fatal("expected QueueAndInterruptForPeerMessage to propagate the cancel failure") } @@ -2392,7 +2403,7 @@ func TestQueueAndInterruptForPeerMessage_DeliversTargetedEntryAheadOfOlderQueued t.Fatalf("queue older message: %v", err) } - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) if err != nil { t.Fatalf("queue and interrupt for peer message: %v", err) } @@ -2450,7 +2461,7 @@ func TestQueueAndInterruptForPeerMessage_WaitsForConcurrentHolderThenDelivers(t // the guard around the lifecycle call, and blocks inside CancelAgent. firstDone := make(chan struct{}) go func() { - _, _, _ = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "first parent message", nil) + _, _, _ = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "first parent message", nil) close(firstDone) }() @@ -2466,7 +2477,7 @@ func TestQueueAndInterruptForPeerMessage_WaitsForConcurrentHolderThenDelivers(t var dispatched bool var secondErr error go func() { - queued, dispatched, secondErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "second parent message", nil) + queued, dispatched, secondErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "second parent message", nil) close(secondDone) }() @@ -2559,6 +2570,14 @@ func (r *erroringTakeByIDRepository) TakeByID(context.Context, string, string) ( return nil, r.takeByIDErr } +func (r *erroringTakeByIDRepository) TakeByIDForSession( + context.Context, + messagequeue.QueueSessionIdentity, + string, +) (*messagequeue.QueuedMessage, error) { + return nil, r.takeByIDErr +} + // TestQueueAndInterruptForPeerMessage_TargetedTakeErrorPropagatesWithoutFIFOFallback // pins the error-vs-not-found distinction on the targeted take: a genuine // repository error (e.g. a transient DB failure) must propagate rather than @@ -2592,7 +2611,7 @@ func TestQueueAndInterruptForPeerMessage_TargetedTakeErrorPropagatesWithoutFIFOF t.Fatalf("queue older message: %v", err) } - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) if err == nil { t.Fatal("expected QueueAndInterruptForPeerMessage to propagate the targeted-take error") } @@ -2621,6 +2640,58 @@ func TestQueueAndInterruptForPeerMessage_TargetedTakeErrorPropagatesWithoutFIFOF } } +func TestQueueAndInterruptForPeerMessageRejectsReplacementIncarnationWhileWaiting(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + agentMgr := &mockAgentManager{isAgentRunning: false, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + original, err := repo.GetTaskSession(ctx, "session1") + require.NoError(t, err) + originalIdentity := messagequeue.QueueSessionIdentity{ + TaskID: original.TaskID, SessionID: original.ID, SessionIncarnationID: original.QueueIncarnationID, + } + + guard, releaseGuard := svc.acquireCancelInFlightGuard(original.ID) + guard.Lock() + result := make(chan struct { + queued *messagequeue.QueuedMessage + dispatched bool + err error + }, 1) + go func() { + queued, dispatched, callErr := svc.QueueAndInterruptForPeerMessage( + ctx, originalIdentity, "stale parent message", nil, + ) + result <- struct { + queued *messagequeue.QueuedMessage + dispatched bool + err error + }{queued: queued, dispatched: dispatched, err: callErr} + }() + coordinatorStopWaitForGuardRefs(t, svc, original.ID, 2) + + require.NoError(t, repo.DeleteTaskSession(ctx, original)) + replacement := &models.TaskSession{ + ID: original.ID, TaskID: original.TaskID, State: models.TaskSessionStateRunning, + StartedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC(), + } + require.NoError(t, repo.CreateTaskSession(ctx, replacement)) + replacementIdentity, err := svc.messageQueue.ResolveSessionIdentity(ctx, replacement.TaskID, replacement.ID) + require.NoError(t, err) + require.NotEqual(t, originalIdentity.SessionIncarnationID, replacementIdentity.SessionIncarnationID) + guard.Unlock() + releaseGuard() + + outcome := <-result + require.ErrorIs(t, outcome.err, messagequeue.ErrSessionIdentityMismatch) + require.Nil(t, outcome.queued) + require.False(t, outcome.dispatched) + status, err := svc.messageQueue.Snapshot(ctx, replacementIdentity) + require.NoError(t, err) + require.Empty(t, status.Entries) +} + // blockingGetTaskSessionRepo wraps a sessionExecutorStore and blocks the // first GetTaskSession call for sessionID until release is closed. This // lets tests pause handleAgentReady *before* it even reaches its pre-guard @@ -2728,7 +2799,7 @@ func TestQueueAndInterruptForPeerMessage_ClosesStaleEarlyCheckRace(t *testing.T) var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) close(interruptDone) }() @@ -2894,7 +2965,7 @@ func TestQueueAndInterruptForPeerMessage_CancelFailureDoesNotStrandMessageWhenRe var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) close(interruptDone) }() @@ -2998,7 +3069,7 @@ func TestQueueAndInterruptForPeerMessage_CancelFailureLeavesMessageQueuedWhenSti seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) seedExecutorRunning(t, repo, "session1", "task1", "exec-1") - queued, dispatched, err := svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, err := queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) if err == nil { t.Fatal("expected the genuine cancel failure to propagate while the session is still RUNNING") } @@ -3061,7 +3132,7 @@ func TestQueueAndInterruptForPeerMessage_DoesNotCancelUnrelatedSuccessorTurn(t * var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) close(interruptDone) }() @@ -3140,7 +3211,7 @@ func TestQueueAndInterruptForPeerMessage_RacesManualDrainForSameSession(t *testi var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) close(interruptDone) }() select { @@ -3258,7 +3329,7 @@ func TestQueueAndInterruptForPeerMessage_RacesClarificationTimeoutRecovery(t *te var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage(ctx, "task1", "session1", "parent steer message", nil) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer message", nil) close(interruptDone) }() select { @@ -3368,9 +3439,7 @@ func TestClarificationRecovery_ReleasesGuardAfterRetryDispatch(t *testing.T) { var dispatched bool var interruptErr error go func() { - queued, dispatched, interruptErr = svc.QueueAndInterruptForPeerMessage( - ctx, "task1", "session1", "parent steer", nil, - ) + queued, dispatched, interruptErr = queueAndInterruptForPeerMessage(svc, ctx, "task1", "session1", "parent steer", nil) close(interruptDone) }() <-turnSync.snapshotTaken @@ -3596,14 +3665,16 @@ func TestCancelIntentDoesNotFollowSharedGuard(t *testing.T) { func requirePersistedSessionLaunchError(t *testing.T, repo *sqliterepo.Repository, sessionID string) models.LastAgentError { t.Helper() - session, err := repo.GetTaskSession(context.Background(), sessionID) - if err != nil { - t.Fatalf("GetTaskSession: %v", err) - } - lastError, ok := models.LoadLastAgentError(session.Metadata) - if !ok { - t.Fatalf("session %q has no typed launch error: %#v", sessionID, session.Metadata) - } + var lastError models.LastAgentError + require.Eventually(t, func() bool { + session, err := repo.GetTaskSession(context.Background(), sessionID) + if err != nil { + return false + } + var ok bool + lastError, ok = models.LoadLastAgentError(session.Metadata) + return ok + }, time.Second, 10*time.Millisecond, "session %q has no typed launch error", sessionID) if lastError.Code == "" || lastError.Stamp() == "" { t.Fatalf("session %q has incomplete typed launch error: %#v", sessionID, lastError) } diff --git a/apps/backend/internal/orchestrator/workflow_e2e_test.go b/apps/backend/internal/orchestrator/workflow_e2e_test.go index c5ab5e9ff4..4b592cd331 100644 --- a/apps/backend/internal/orchestrator/workflow_e2e_test.go +++ b/apps/backend/internal/orchestrator/workflow_e2e_test.go @@ -11,7 +11,6 @@ import ( v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/internal/orchestrator/executor" - "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/task/models" sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" "github.com/kandev/kandev/internal/workflow/engine" @@ -314,7 +313,7 @@ func createEngineService(t *testing.T, repo *sqliterepo.Repository, sg *mockStep repo: repo, taskRepo: newMockTaskRepo(), agentManager: agentMgr, - messageQueue: messagequeue.NewServiceMemory(log), + messageQueue: newAuthoritativeMemoryQueue(repo, log), executor: executor.NewExecutor(agentMgr, repo, log, executor.ExecutorConfig{}), } svc.SetWorkflowStepGetter(sg) diff --git a/apps/backend/internal/orchestrator/workflow_store.go b/apps/backend/internal/orchestrator/workflow_store.go index bf5fe95009..8c9d9de6a0 100644 --- a/apps/backend/internal/orchestrator/workflow_store.go +++ b/apps/backend/internal/orchestrator/workflow_store.go @@ -11,6 +11,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/steptelemetry" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/workflow/engine" @@ -57,6 +58,21 @@ type workflowMoveAdmissionCASRepository interface { ) (applied bool, err error) } +type deferredMoveAdmissionRepository interface { + UpdateTaskWithWorkflowStepAdmissionForDeferredMove( + ctx context.Context, + task *models.Task, + expectedStepID, targetStepID string, + limit int, + record messagequeue.PendingMoveRecord, + ) (admitted, applied bool, err error) + MarkDeferredMoveAppliedForSession( + ctx context.Context, + taskID, moveID string, + record messagequeue.PendingMoveRecord, + ) (applied bool, err error) +} + type workflowQueuedTaskPromoter interface { PromoteQueuedTaskIfWorkflowStepHasCapacity(ctx context.Context, task *models.Task, fromStepID, destinationStepID string, limit int) (bool, error) } @@ -273,14 +289,42 @@ func (s *workflowStore) LoadPreviousStep(ctx context.Context, workflowID string, } func (s *workflowStore) ApplyTransition(ctx context.Context, taskID, sessionID, fromStepID, toStepID string, trigger engine.Trigger) error { - return s.applyTransition(ctx, taskID, sessionID, fromStepID, toStepID, trigger, "") + return s.applyTransition(ctx, taskID, sessionID, fromStepID, toStepID, trigger, "", nil) } -func (s *workflowStore) ApplyDeferredMoveTransition(ctx context.Context, taskID, sessionID, fromStepID, toStepID, moveID string) error { - return s.applyTransition(ctx, taskID, sessionID, fromStepID, toStepID, engine.TriggerOnEnter, moveID) +func (s *workflowStore) ApplyDeferredMoveTransition( + ctx context.Context, + taskID, sessionID, fromStepID, toStepID, moveID string, + record messagequeue.PendingMoveRecord, +) error { + return s.applyTransition( + ctx, taskID, sessionID, fromStepID, toStepID, engine.TriggerOnEnter, moveID, &record, + ) } -func (s *workflowStore) MarkDeferredMoveApplied(ctx context.Context, taskID, moveID string) error { +func (s *workflowStore) MarkDeferredMoveApplied( + ctx context.Context, + taskID, moveID string, + record messagequeue.PendingMoveRecord, +) error { + if moveID == "" { + return nil + } + repo, ok := s.repo.(deferredMoveAdmissionRepository) + if !ok { + return fmt.Errorf("deferred move repository unavailable") + } + applied, err := repo.MarkDeferredMoveAppliedForSession(ctx, taskID, moveID, record) + if err != nil { + return fmt.Errorf("persist deferred move identity: %w", err) + } + if !applied { + return errDeferredMoveAlreadyApplied + } + return nil +} + +func (s *workflowStore) markDeferredMoveAppliedUnfenced(ctx context.Context, taskID, moveID string) error { if moveID == "" { return nil } @@ -297,7 +341,13 @@ func (s *workflowStore) MarkDeferredMoveApplied(ctx context.Context, taskID, mov return nil } -func (s *workflowStore) applyTransition(ctx context.Context, taskID, sessionID, fromStepID, toStepID string, trigger engine.Trigger, moveID string) error { +func (s *workflowStore) applyTransition( + ctx context.Context, + taskID, sessionID, fromStepID, toStepID string, + trigger engine.Trigger, + moveID string, + deferredMove *messagequeue.PendingMoveRecord, +) error { task, err := s.repo.GetTask(ctx, taskID) if err != nil { return fmt.Errorf("load task for transition: %w", err) @@ -328,47 +378,21 @@ func (s *workflowStore) applyTransition(ctx context.Context, taskID, sessionID, delete(task.Metadata, models.MetaKeyQueuedMoveExitCompleted) delete(task.Metadata, models.MetaKeyQueuePromotionPending) } - // The on_turn_complete transition consumes the source session's signal. - // Put its carry token on this task snapshot before admission so a queued - // destination cannot be promoted between the transition commit and a later - // metadata write. - if trigger == engine.TriggerOnTurnComplete { - var consumedSignal *models.PendingStepCompletionSignal - session, sessionErr := s.repo.GetTaskSession(ctx, sessionID) - if sessionErr != nil { - s.logger.Debug("failed to load session for step handoff carry", - zap.String("session_id", sessionID), zap.Error(sessionErr)) - } else if session != nil { - if signal, has := models.LoadPendingStepSignal(session.Metadata); has && signal.StepID == fromStepID { - consumedSignal = &signal - } - setStepHandoffCarryMetadata(task, toStepID, consumedSignal) - } - } + // Publish the on_turn_complete signal's handoff on the same task snapshot + // as admission so a queued destination cannot promote in between. + s.carryStepHandoffForTransition(ctx, task, sessionID, fromStepID, toStepID, trigger) task.UpdatedAt = time.Now().UTC() // engine_transition applies only when no outer caller already declared a // trigger — applyPendingMove sets mcp_deferred_move before reaching this // path, and that must survive rather than be overwritten. - transitionCtx := ctx - if !steptelemetry.HasTrigger(transitionCtx) { - transitionCtx = engineTransitionAttribution(transitionCtx, sessionID, trigger) - } - // Only attach a PendingAllocation when the caller already opted in by - // wrapping ctx with a ResultHolder (applyEngineTransition does this for - // the engine-driven on_turn_complete path). Attaching it unconditionally - // would allocate workflow_step_entries rows for every transition into a - // step with engine-owned on_enter actions, including the callers (manual - // move, deferred move) that don't yet dispatch through them this Build - // round — see docs/specs/workflow-on-enter-action-dispatch/spec.md and - // the task plan's scope note for why those entry paths are deferred. - if targetStep != nil { - if _, wantsAllocation := stepentry.ResultHolderFromContext(ctx); wantsAllocation { - if pending, ok := stepentry.BuildPendingAllocation(targetStep.ID, targetStep.Events.OnEnter); ok { - transitionCtx = stepentry.WithPendingAllocation(transitionCtx, pending) - } + transitionCtx := transitionContext(ctx, sessionID, trigger, targetStep) + if deferredMove != nil { + if err := s.updateDeferredTransitionTask( + transitionCtx, task, fromStepID, targetStep, *deferredMove, + ); err != nil { + return fmt.Errorf("update task workflow step: %w", err) } - } - if err := s.updateTransitionTask(transitionCtx, task, targetStep); err != nil { + } else if err := s.updateTransitionTask(transitionCtx, task, targetStep); err != nil { return fmt.Errorf("update task workflow step: %w", err) } @@ -378,7 +402,7 @@ func (s *workflowStore) applyTransition(ctx context.Context, taskID, sessionID, // instead of leaving a stale duplicate until reload. s.publishTaskUpdated(ctx, task, oldWorkflowID) - if task.QueuedForStepID == "" { + if deferredMove == nil && task.QueuedForStepID == "" { if err := s.repo.UpdateSessionReviewStatus(ctx, sessionID, ""); err != nil { s.logger.Warn("failed to clear session review status", zap.String("session_id", sessionID), @@ -397,6 +421,74 @@ func (s *workflowStore) applyTransition(ctx context.Context, taskID, sessionID, return nil } +func (s *workflowStore) carryStepHandoffForTransition( + ctx context.Context, + task *models.Task, + sessionID, fromStepID, toStepID string, + trigger engine.Trigger, +) { + if trigger != engine.TriggerOnTurnComplete { + return + } + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + s.logger.Debug("failed to load session for step handoff carry", + zap.String("session_id", sessionID), zap.Error(err)) + return + } + if session == nil { + return + } + var consumedSignal *models.PendingStepCompletionSignal + if signal, has := models.LoadPendingStepSignal(session.Metadata); has && signal.StepID == fromStepID { + consumedSignal = &signal + } + setStepHandoffCarryMetadata(task, toStepID, consumedSignal) +} + +func transitionContext( + ctx context.Context, + sessionID string, + trigger engine.Trigger, + targetStep *wfmodels.WorkflowStep, +) context.Context { + transitionCtx := ctx + if !steptelemetry.HasTrigger(transitionCtx) { + transitionCtx = engineTransitionAttribution(transitionCtx, sessionID, trigger) + } + if targetStep != nil { + if _, wantsAllocation := stepentry.ResultHolderFromContext(ctx); wantsAllocation { + if pending, ok := stepentry.BuildPendingAllocation(targetStep.ID, targetStep.Events.OnEnter); ok { + transitionCtx = stepentry.WithPendingAllocation(transitionCtx, pending) + } + } + } + return transitionCtx +} + +func (s *workflowStore) updateDeferredTransitionTask( + ctx context.Context, + task *models.Task, + fromStepID string, + targetStep *wfmodels.WorkflowStep, + record messagequeue.PendingMoveRecord, +) error { + repo, ok := s.repo.(deferredMoveAdmissionRepository) + if !ok { + return fmt.Errorf("deferred move repository unavailable") + } + _, applied, err := repo.UpdateTaskWithWorkflowStepAdmissionForDeferredMove( + ctx, task, fromStepID, targetStep.ID, targetStep.WIPLimit, record, + ) + if err != nil { + return err + } + if !applied { + return fmt.Errorf("deferred move source changed") + } + return nil +} + // ApplyTransitionIfAtStep is the AC-46/48 compare-and-swap variant of // ApplyTransition used by the workflow engine's guarded-transition // re-evaluation apply path (Engine.reevaluateGuardedTransitions). The service diff --git a/apps/backend/internal/system/queuesettings/queuesettings_test.go b/apps/backend/internal/system/queuesettings/queuesettings_test.go index db26774ee8..f9cf41ed8d 100644 --- a/apps/backend/internal/system/queuesettings/queuesettings_test.go +++ b/apps/backend/internal/system/queuesettings/queuesettings_test.go @@ -96,7 +96,7 @@ func TestStoreRoundTripAndValidation(t *testing.T) { if err := json.Unmarshal(raw.raw, &saved); err != nil { t.Fatalf("decode saved JSON: %v", err) } - if saved["auto_merge_enabled"] != true || len(saved) != 3 { + if saved["auto_merge_enabled"] != true || saved["auto_merge_revision"] != float64(0) || len(saved) != 4 { t.Fatalf("saved settings are not normalized: %s", raw.raw) } loaded, err = store.Load(context.Background()) @@ -318,6 +318,80 @@ func TestServiceSerializesPersistenceAndLiveApply(t *testing.T) { } } +func TestServicesSerializeConcurrentSettingsUpdatesAcrossInstances(t *testing.T) { + raw := &compareAndSwapRawStore{ + raw: []byte(`{"max_per_session":10,"merge_enabled":true,"auto_merge_enabled":true,"auto_merge_revision":0}`), + found: true, + } + first := NewService( + NewStore(raw), &fakeTarget{max: 10, mergeEnabled: true, autoMergeEnabled: true}, + func() Environment { return Environment{} }, testLogger(t), + ) + second := NewService( + NewStore(raw), &fakeTarget{max: 10, mergeEnabled: true, autoMergeEnabled: true}, + func() Environment { return Environment{} }, testLogger(t), + ) + + start := make(chan struct{}) + results := make(chan error, 2) + go func() { + <-start + _, err := first.Update(context.Background(), SettingsPatch{AutoMergeEnabled: new(false)}) + results <- err + }() + go func() { + <-start + _, err := second.Update(context.Background(), SettingsPatch{MaxPerSession: new(20)}) + results <- err + }() + close(start) + if err := <-results; err != nil { + t.Fatalf("first concurrent update: %v", err) + } + if err := <-results; err != nil { + t.Fatalf("second concurrent update: %v", err) + } + + configured, err := NewStore(raw).Load(context.Background()) + if err != nil || configured == nil { + t.Fatalf("load concurrent result: configured=%+v err=%v", configured, err) + } + if configured.MaxPerSession != 20 || configured.AutoMergeEnabled || + configured.AutoMergeRevision != 1 { + t.Fatalf("concurrent result = %+v, want max=20 automatic=false revision=1", configured) + } +} + +func TestServiceInstallsConsistentAutoMergePolicyLoader(t *testing.T) { + raw := &compareAndSwapRawStore{} + writer := NewService( + NewStore(raw), &fakeTarget{max: 10, mergeEnabled: true, autoMergeEnabled: true}, + func() Environment { return Environment{} }, testLogger(t), + ) + readerTarget := &policyLoaderTarget{ + fakeTarget: fakeTarget{max: 10, mergeEnabled: true, autoMergeEnabled: true}, + } + _ = NewService( + NewStore(raw), readerTarget, func() Environment { return Environment{} }, testLogger(t), + ) + if readerTarget.load == nil { + t.Fatal("queue settings service did not install a shared policy loader") + } + if _, err := writer.Update( + context.Background(), + SettingsPatch{AutoMergeEnabled: new(false)}, + ); err != nil { + t.Fatalf("disable Auto-merge: %v", err) + } + enabled, revision, err := readerTarget.load(context.Background()) + if err != nil { + t.Fatalf("load shared policy: %v", err) + } + if enabled || revision != 1 { + t.Fatalf("shared policy = enabled=%v revision=%d, want false/1", enabled, revision) + } +} + func TestServiceEnvironmentLockRejectsUpdate(t *testing.T) { raw := &fakeRawStore{} target := &fakeTarget{max: 20} @@ -584,12 +658,65 @@ func (f *fakeRawStore) Save(_ context.Context, _ string, value []byte) error { return nil } +type compareAndSwapRawStore struct { + mu sync.Mutex + raw []byte + found bool +} + +func (s *compareAndSwapRawStore) Get(context.Context, string) ([]byte, bool, error) { + return s.GetConsistent(context.Background(), "") +} + +func (s *compareAndSwapRawStore) GetConsistent(context.Context, string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]byte(nil), s.raw...), s.found, nil +} + +func (s *compareAndSwapRawStore) Save(_ context.Context, _ string, value []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + s.raw = append([]byte(nil), value...) + s.found = true + return nil +} + +func (s *compareAndSwapRawStore) CompareAndSwap( + _ context.Context, + _ string, + expected, value []byte, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if expected == nil { + if s.found { + return false, nil + } + } else if !s.found || !bytes.Equal(s.raw, expected) { + return false, nil + } + s.raw = append([]byte(nil), value...) + s.found = true + return true, nil +} + type fakeTarget struct { max int mergeEnabled bool autoMergeEnabled bool } +type policyLoaderTarget struct { + fakeTarget + load func(context.Context) (bool, int64, error) +} + +func (t *policyLoaderTarget) SetAutoMergePolicyLoader( + load func(context.Context) (bool, int64, error), +) { + t.load = load +} func (f *fakeTarget) MaxPerSession() int { return f.max } func (f *fakeTarget) SetMaxPerSession(n int) { f.max = n } func (f *fakeTarget) MergeEnabled() bool { return f.mergeEnabled } diff --git a/apps/backend/internal/system/queuesettings/revision_test.go b/apps/backend/internal/system/queuesettings/revision_test.go new file mode 100644 index 0000000000..a661a4c9e4 --- /dev/null +++ b/apps/backend/internal/system/queuesettings/revision_test.go @@ -0,0 +1,49 @@ +package queuesettings + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.11 +func TestAutoMergeRevisionAdvancesOnlyWhenBooleanChangesAndStaysInternal(t *testing.T) { + raw := &fakeRawStore{} + target := &fakeTarget{max: DefaultMaxPerSession, mergeEnabled: true, autoMergeEnabled: true} + service := NewService(NewStore(raw), target, func() Environment { return Environment{} }, testLogger(t)) + ctx := context.Background() + + if _, err := service.Update(ctx, SettingsPatch{AutoMergeEnabled: new(false)}); err != nil { + t.Fatalf("disable Auto-merge: %v", err) + } + assertStoredAutoMergeRevision(t, raw.raw, 1) + if _, err := service.Update(ctx, SettingsPatch{MaxPerSession: new(4)}); err != nil { + t.Fatalf("update capacity: %v", err) + } + assertStoredAutoMergeRevision(t, raw.raw, 1) + response, err := service.Update(ctx, SettingsPatch{AutoMergeEnabled: new(true)}) + if err != nil { + t.Fatalf("enable Auto-merge: %v", err) + } + assertStoredAutoMergeRevision(t, raw.raw, 2) + + encoded, err := json.Marshal(response) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + if strings.Contains(string(encoded), "auto_merge_revision") { + t.Fatalf("public response leaked internal revision: %s", encoded) + } +} + +func assertStoredAutoMergeRevision(t *testing.T, raw []byte, want float64) { + t.Helper() + var stored map[string]interface{} + if err := json.Unmarshal(raw, &stored); err != nil { + t.Fatalf("decode stored settings: %v", err) + } + if got := stored["auto_merge_revision"]; got != want { + t.Fatalf("stored auto_merge_revision = %v, want %.0f", got, want) + } +} diff --git a/apps/backend/internal/system/queuesettings/service.go b/apps/backend/internal/system/queuesettings/service.go index c5967a2966..ae95c5f39a 100644 --- a/apps/backend/internal/system/queuesettings/service.go +++ b/apps/backend/internal/system/queuesettings/service.go @@ -18,6 +18,12 @@ type Target interface { AutoMergeEnabled() bool SetAutoMergeEnabled(bool) } +type revisionedAutoMergeTarget interface { + SetAutoMergePolicy(enabled bool, revision int64) +} +type autoMergePolicyLoaderTarget interface { + SetAutoMergePolicyLoader(func(context.Context) (bool, int64, error)) +} type EnvironmentReader func() Environment @@ -44,10 +50,14 @@ func NewService( if len(startup) > 0 { configuration = startup[0] } - return &Service{ + service := &Service{ store: store, target: target, readEnvironment: readEnvironment, configuration: configuration, logger: log, } + if target, ok := target.(autoMergePolicyLoaderTarget); ok { + target.SetAutoMergePolicyLoader(service.loadAutoMergePolicy) + } + return service } func ReadEnvironment() Environment { @@ -80,30 +90,32 @@ func (s *Service) Update(ctx context.Context, patch SettingsPatch) (Response, er s.mu.Lock() defer s.mu.Unlock() - environment := s.readEnvironment() - current, err := s.loadConfigured(ctx) - if err != nil { - return Response{}, err - } - resolution, err := Resolve(current, environment, s.configuration) - if err != nil { - return Response{}, err - } - s.warnInvalidEnvironment(resolution) - if patch.MaxPerSession != nil && resolution.Effective.Locked { - if resolution.Effective.Source == SourceConfiguration { - return Response{}, ErrConfigurationLocked - } - return Response{}, ErrEnvironmentLocked - } - settings := patch.Apply(resolution.Settings) - if err := Validate(settings); err != nil { - return Response{}, err - } if s.target == nil { return Response{}, ErrTargetUnavailable } - if err := s.store.Save(ctx, settings); err != nil { + environment := s.readEnvironment() + settings, err := s.store.Update(ctx, func(current *Settings) (Settings, error) { + resolution, resolveErr := Resolve(current, environment, s.configuration) + if resolveErr != nil { + return Settings{}, resolveErr + } + s.warnInvalidEnvironment(resolution) + if patch.MaxPerSession != nil && resolution.Effective.Locked { + if resolution.Effective.Source == SourceConfiguration { + return Settings{}, ErrConfigurationLocked + } + return Settings{}, ErrEnvironmentLocked + } + updated := patch.Apply(resolution.Settings) + if patch.AutoMergeEnabled != nil && *patch.AutoMergeEnabled != resolution.Settings.AutoMergeEnabled { + updated.AutoMergeRevision = resolution.Settings.AutoMergeRevision + 1 + } + if validateErr := Validate(updated); validateErr != nil { + return Settings{}, validateErr + } + return updated, nil + }) + if err != nil { return Response{}, err } updated, err := Resolve(&settings, environment, s.configuration) @@ -112,10 +124,26 @@ func (s *Service) Update(ctx context.Context, patch SettingsPatch) (Response, er } s.target.SetMaxPerSession(updated.Effective.MaxPerSession) s.target.SetMergeEnabled(updated.Effective.MergeEnabled) - s.target.SetAutoMergeEnabled(updated.Effective.AutoMergeEnabled) + if target, ok := s.target.(revisionedAutoMergeTarget); ok { + target.SetAutoMergePolicy(updated.Effective.AutoMergeEnabled, settings.AutoMergeRevision) + } else { + s.target.SetAutoMergeEnabled(updated.Effective.AutoMergeEnabled) + } return updated.Response, nil } +func (s *Service) loadAutoMergePolicy(ctx context.Context) (bool, int64, error) { + configured, err := s.loadConfiguredConsistent(ctx) + if err != nil { + return false, 0, err + } + resolution, err := Resolve(configured, s.readEnvironment(), s.configuration) + if err != nil { + return false, 0, err + } + return resolution.Effective.AutoMergeEnabled, resolution.Settings.AutoMergeRevision, nil +} + func (s *Service) loadConfigured(ctx context.Context) (*Settings, error) { configured, err := s.store.Load(ctx) if err == nil { @@ -130,6 +158,20 @@ func (s *Service) loadConfigured(ctx context.Context) (*Settings, error) { return nil, nil } +func (s *Service) loadConfiguredConsistent(ctx context.Context) (*Settings, error) { + configured, err := s.store.LoadConsistent(ctx) + if err == nil { + return configured, nil + } + if !errors.Is(err, ErrInvalidPersisted) { + return nil, err + } + if s.logger != nil { + s.logger.Warn("Ignoring invalid persisted message queue settings", zap.Error(err)) + } + return nil, nil +} + func (s *Service) warnInvalidEnvironment(resolution Resolution) { if !resolution.InvalidEnvironment || s.logger == nil { return diff --git a/apps/backend/internal/system/queuesettings/store.go b/apps/backend/internal/system/queuesettings/store.go index f2cbee7548..3806a966db 100644 --- a/apps/backend/internal/system/queuesettings/store.go +++ b/apps/backend/internal/system/queuesettings/store.go @@ -3,6 +3,7 @@ package queuesettings import ( "context" "encoding/json" + "errors" "fmt" ) @@ -11,6 +12,11 @@ type RawStore interface { Save(ctx context.Context, key string, value []byte) error } +type rawCompareAndSwapStore interface { + GetConsistent(ctx context.Context, key string) ([]byte, bool, error) + CompareAndSwap(ctx context.Context, key string, expected, value []byte) (bool, error) +} + type Store struct { raw RawStore } @@ -23,9 +29,10 @@ func NewStore(raw RawStore) *Store { // settings as pointers. Missing keys in records written by older versions // therefore default on, while explicit false values remain false. type storedSettings struct { - MaxPerSession int `json:"max_per_session"` - MergeEnabled *bool `json:"merge_enabled"` - AutoMergeEnabled *bool `json:"auto_merge_enabled"` + MaxPerSession int `json:"max_per_session"` + MergeEnabled *bool `json:"merge_enabled"` + AutoMergeEnabled *bool `json:"auto_merge_enabled"` + AutoMergeRevision int64 `json:"auto_merge_revision"` } // toSettings normalizes a decoded record, defaulting omitted merge settings on. @@ -39,7 +46,8 @@ func (s storedSettings) toSettings() Settings { autoMergeEnabled = *s.AutoMergeEnabled } return Settings{ - MaxPerSession: s.MaxPerSession, MergeEnabled: mergeEnabled, AutoMergeEnabled: autoMergeEnabled, + MaxPerSession: s.MaxPerSession, MergeEnabled: mergeEnabled, + AutoMergeEnabled: autoMergeEnabled, AutoMergeRevision: s.AutoMergeRevision, } } @@ -48,6 +56,22 @@ func (s *Store) Load(ctx context.Context) (*Settings, error) { if err != nil { return nil, fmt.Errorf("load message queue settings: %w", err) } + return decodeSettings(raw, found) +} + +func (s *Store) LoadConsistent(ctx context.Context) (*Settings, error) { + cas, ok := s.raw.(rawCompareAndSwapStore) + if !ok { + return s.Load(ctx) + } + raw, found, err := cas.GetConsistent(ctx, SettingsKey) + if err != nil { + return nil, fmt.Errorf("load message queue settings: %w", err) + } + return decodeSettings(raw, found) +} + +func decodeSettings(raw []byte, found bool) (*Settings, error) { if !found { return nil, nil } @@ -63,15 +87,98 @@ func (s *Store) Load(ctx context.Context) (*Settings, error) { } func (s *Store) Save(ctx context.Context, settings Settings) error { - if err := Validate(settings); err != nil { - return err - } - raw, err := json.Marshal(settings) + raw, err := encodeSettings(settings) if err != nil { - return fmt.Errorf("encode message queue settings: %w", err) + return err } if err := s.raw.Save(ctx, SettingsKey, raw); err != nil { return fmt.Errorf("save message queue settings: %w", err) } return nil } + +func encodeSettings(settings Settings) ([]byte, error) { + if err := Validate(settings); err != nil { + return nil, err + } + raw, err := json.Marshal(storedSettings{ + MaxPerSession: settings.MaxPerSession, MergeEnabled: &settings.MergeEnabled, + AutoMergeEnabled: &settings.AutoMergeEnabled, AutoMergeRevision: settings.AutoMergeRevision, + }) + if err != nil { + return nil, fmt.Errorf("encode message queue settings: %w", err) + } + return raw, nil +} + +func (s *Store) Update( + ctx context.Context, + apply func(current *Settings) (Settings, error), +) (Settings, error) { + cas, ok := s.raw.(rawCompareAndSwapStore) + if !ok { + return s.updateWithoutCompareAndSwap(ctx, apply) + } + return s.updateWithCompareAndSwap(ctx, cas, apply) +} + +func (s *Store) updateWithoutCompareAndSwap( + ctx context.Context, + apply func(current *Settings) (Settings, error), +) (Settings, error) { + current, err := settingsOrUnset(s.Load(ctx)) + if err != nil { + return Settings{}, err + } + updated, err := apply(current) + if err != nil { + return Settings{}, err + } + return updated, s.Save(ctx, updated) +} + +func (s *Store) updateWithCompareAndSwap( + ctx context.Context, + cas rawCompareAndSwapStore, + apply func(current *Settings) (Settings, error), +) (Settings, error) { + for { + raw, found, err := cas.GetConsistent(ctx, SettingsKey) + if err != nil { + return Settings{}, fmt.Errorf("load message queue settings: %w", err) + } + current, err := settingsOrUnset(decodeSettings(raw, found)) + if err != nil { + return Settings{}, err + } + updated, err := apply(current) + if err != nil { + return Settings{}, err + } + next, err := encodeSettings(updated) + if err != nil { + return Settings{}, err + } + var expected []byte + if found { + expected = raw + } + swapped, err := cas.CompareAndSwap(ctx, SettingsKey, expected, next) + if err != nil { + return Settings{}, fmt.Errorf("save message queue settings: %w", err) + } + if swapped { + return updated, nil + } + if err := ctx.Err(); err != nil { + return Settings{}, err + } + } +} + +func settingsOrUnset(settings *Settings, err error) (*Settings, error) { + if errors.Is(err, ErrInvalidPersisted) { + return nil, nil + } + return settings, err +} diff --git a/apps/backend/internal/system/queuesettings/types.go b/apps/backend/internal/system/queuesettings/types.go index a59f713af0..5ad904e053 100644 --- a/apps/backend/internal/system/queuesettings/types.go +++ b/apps/backend/internal/system/queuesettings/types.go @@ -31,9 +31,10 @@ var ( ) type Settings struct { - MaxPerSession int `json:"max_per_session"` - MergeEnabled bool `json:"merge_enabled"` - AutoMergeEnabled bool `json:"auto_merge_enabled"` + MaxPerSession int `json:"max_per_session"` + MergeEnabled bool `json:"merge_enabled"` + AutoMergeEnabled bool `json:"auto_merge_enabled"` + AutoMergeRevision int64 `json:"-"` } // SettingsPatch is a partial update to Settings: a nil field means "leave diff --git a/apps/backend/internal/system/settings/store.go b/apps/backend/internal/system/settings/store.go index 95cecd5107..126ef2e1f2 100644 --- a/apps/backend/internal/system/settings/store.go +++ b/apps/backend/internal/system/settings/store.go @@ -79,6 +79,50 @@ func (s *Store) Get(ctx context.Context, key string) ([]byte, bool, error) { return []byte(raw), true, nil } +func (s *Store) GetConsistent(ctx context.Context, key string) ([]byte, bool, error) { + var raw string + err := s.db.QueryRowContext(ctx, s.db.Rebind(`SELECT value FROM settings WHERE key = ?`), key).Scan(&raw) + if err != nil { + if err == sql.ErrNoRows { + return nil, false, nil + } + return nil, false, err + } + return []byte(raw), true, nil +} + +func (s *Store) CompareAndSwap( + ctx context.Context, + key string, + expected, value []byte, +) (bool, error) { + now := time.Now().UTC() + var ( + result sql.Result + err error + ) + if expected == nil { + result, err = s.db.ExecContext(ctx, s.db.Rebind(` + INSERT INTO settings (key, value, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(key) DO NOTHING + `), key, string(value), now) + } else { + result, err = s.db.ExecContext(ctx, s.db.Rebind(` + UPDATE settings SET value = ?, updated_at = ? + WHERE key = ? AND value = ? + `), string(value), now, key, string(expected)) + } + if err != nil { + return false, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, err + } + return rows > 0, nil +} + // GetEntry reads one setting row, including its persisted update timestamp. func (s *Store) GetEntry(ctx context.Context, key string) (Entry, bool, error) { var row struct { diff --git a/apps/backend/internal/system/settings/store_test.go b/apps/backend/internal/system/settings/store_test.go index 2d7fd81014..36c712ed72 100644 --- a/apps/backend/internal/system/settings/store_test.go +++ b/apps/backend/internal/system/settings/store_test.go @@ -35,6 +35,42 @@ func TestStoreSaveAndGet(t *testing.T) { } } +func TestStoreCompareAndSwap(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + swapped, err := store.CompareAndSwap(ctx, "message_queue", nil, []byte(`{"revision":1}`)) + if err != nil || !swapped { + t.Fatalf("insert compare-and-swap = %v, %v", swapped, err) + } + swapped, err = store.CompareAndSwap(ctx, "message_queue", nil, []byte(`{"revision":2}`)) + if err != nil || swapped { + t.Fatalf("duplicate insert compare-and-swap = %v, %v", swapped, err) + } + swapped, err = store.CompareAndSwap( + ctx, + "message_queue", + []byte(`{"revision":0}`), + []byte(`{"revision":2}`), + ) + if err != nil || swapped { + t.Fatalf("stale update compare-and-swap = %v, %v", swapped, err) + } + swapped, err = store.CompareAndSwap( + ctx, + "message_queue", + []byte(`{"revision":1}`), + []byte(`{"revision":2}`), + ) + if err != nil || !swapped { + t.Fatalf("current update compare-and-swap = %v, %v", swapped, err) + } + value, found, err := store.GetConsistent(ctx, "message_queue") + if err != nil || !found || string(value) != `{"revision":2}` { + t.Fatalf("consistent value = (%q, %v, %v)", value, found, err) + } +} + func TestStoreMigratesLegacySystemSettings(t *testing.T) { conn := newSQLite(t) if _, err := conn.Exec(` diff --git a/apps/backend/internal/task/dto/converters_test.go b/apps/backend/internal/task/dto/converters_test.go index 9a527136ab..170cb5ff0c 100644 --- a/apps/backend/internal/task/dto/converters_test.go +++ b/apps/backend/internal/task/dto/converters_test.go @@ -194,6 +194,18 @@ func TestFromTaskSession_IncludesAllWorktrees(t *testing.T) { } } +func TestTaskSessionDTOsProjectQueueIncarnation(t *testing.T) { + session := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", QueueIncarnationID: "incarnation-1", + } + if got := FromTaskSession(session).QueueIncarnationID; got != session.QueueIncarnationID { + t.Fatalf("full queue incarnation = %q, want %q", got, session.QueueIncarnationID) + } + if got := FromTaskSessionSummary(session).QueueIncarnationID; got != session.QueueIncarnationID { + t.Fatalf("summary queue incarnation = %q, want %q", got, session.QueueIncarnationID) + } +} + // TestFromTaskSession_CopiesRollupColumns pins docs/specs/task-cost-ledger/ // spec.md AC-28/AC-29: TaskSessionDTO (the full session detail shape) must // surface the four usage/cost rollup columns internal/task/usage's writer diff --git a/apps/backend/internal/task/dto/dto.go b/apps/backend/internal/task/dto/dto.go index 03a9cedc93..5b921acd68 100644 --- a/apps/backend/internal/task/dto/dto.go +++ b/apps/backend/internal/task/dto/dto.go @@ -329,8 +329,9 @@ type TaskWorkspaceFolderDTO struct { } type TaskSessionDTO struct { - ID string `json:"id"` - TaskID string `json:"task_id"` + ID string `json:"id"` + TaskID string `json:"task_id"` + QueueIncarnationID string `json:"queue_incarnation_id"` // Name is the user-supplied session tab label. Serialized without // omitempty so a cleared name ("") overwrites stale client state. Name string `json:"name"` @@ -422,8 +423,9 @@ type TaskSessionDTO struct { // TaskSessionSummaryDTO is a lightweight version of TaskSessionDTO without snapshot fields. // Used for list endpoints where snapshots are not needed, reducing response size by ~40-60%. type TaskSessionSummaryDTO struct { - ID string `json:"id"` - TaskID string `json:"task_id"` + ID string `json:"id"` + TaskID string `json:"task_id"` + QueueIncarnationID string `json:"queue_incarnation_id"` // Name is the user-supplied session tab label. Serialized without // omitempty so a cleared name ("") overwrites stale client state. Name string `json:"name"` @@ -995,30 +997,31 @@ func FromTaskWithSessionInfo( // FromTaskSessionSummary converts a session model to a summary DTO (no snapshot fields). func FromTaskSessionSummary(session *models.TaskSession) TaskSessionSummaryDTO { result := TaskSessionSummaryDTO{ - ID: session.ID, - TaskID: session.TaskID, - Name: session.Name, - AgentExecutionID: session.AgentExecutionID, - ContainerID: session.ContainerID, - AgentProfileID: session.AgentProfileID, - ExecutorID: session.ExecutorID, - ExecutorProfileID: session.ExecutorProfileID, - EnvironmentID: session.EnvironmentID, - RepositoryID: session.RepositoryID, - BaseBranch: session.BaseBranch, - BaseCommitSHA: session.BaseCommitSHA, - WorkspacePath: session.WorkspacePath, - State: session.State, - ErrorMessage: session.ErrorMessage, - Metadata: session.Metadata, - StartedAt: session.StartedAt, - CompletedAt: session.CompletedAt, - UpdatedAt: session.UpdatedAt, - IsPrimary: session.IsPrimary, - IsPassthrough: session.IsPassthrough, - ReviewStatus: session.ReviewStatus, - TaskEnvironmentID: session.TaskEnvironmentID, - LastReadMessageID: session.LastReadMessageID, + ID: session.ID, + TaskID: session.TaskID, + QueueIncarnationID: session.QueueIncarnationID, + Name: session.Name, + AgentExecutionID: session.AgentExecutionID, + ContainerID: session.ContainerID, + AgentProfileID: session.AgentProfileID, + ExecutorID: session.ExecutorID, + ExecutorProfileID: session.ExecutorProfileID, + EnvironmentID: session.EnvironmentID, + RepositoryID: session.RepositoryID, + BaseBranch: session.BaseBranch, + BaseCommitSHA: session.BaseCommitSHA, + WorkspacePath: session.WorkspacePath, + State: session.State, + ErrorMessage: session.ErrorMessage, + Metadata: session.Metadata, + StartedAt: session.StartedAt, + CompletedAt: session.CompletedAt, + UpdatedAt: session.UpdatedAt, + IsPrimary: session.IsPrimary, + IsPassthrough: session.IsPassthrough, + ReviewStatus: session.ReviewStatus, + TaskEnvironmentID: session.TaskEnvironmentID, + LastReadMessageID: session.LastReadMessageID, } if worktrees := session.WorktreesAPI(); len(worktrees) > 0 { result.WorktreeID = session.Worktrees[0].WorktreeID @@ -1033,6 +1036,7 @@ func FromTaskSession(session *models.TaskSession) TaskSessionDTO { result := TaskSessionDTO{ ID: session.ID, TaskID: session.TaskID, + QueueIncarnationID: session.QueueIncarnationID, Name: session.Name, AgentExecutionID: session.AgentExecutionID, ContainerID: session.ContainerID, diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index 2a84a16297..e6aa0fa8fa 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -468,7 +468,7 @@ func (m *mockRepository) CountActiveTaskSessionsByRepository(ctx context.Context func (m *mockRepository) DeleteEphemeralTasksByAgentProfile(ctx context.Context, agentProfileID string) (int64, error) { return 0, nil } -func (m *mockRepository) DeleteTaskSession(ctx context.Context, id string) error { +func (m *mockRepository) DeleteTaskSession(ctx context.Context, session *models.TaskSession) error { return nil } func (m *mockRepository) ListTaskSessionWorktrees(ctx context.Context, sessionID string) ([]*models.TaskEnvironmentRepo, error) { diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index 2583a6a215..d0b1650f8b 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -1598,6 +1598,7 @@ type SessionBranchInfo struct { type TaskSession struct { ID string `json:"id"` TaskID string `json:"task_id"` + QueueIncarnationID string `json:"queue_incarnation_id"` Name string `json:"name,omitempty"` // Optional user-supplied label shown on the session tab AgentExecutionID string `json:"agent_execution_id"` // Docker container/agent execution ContainerID string `json:"container_id"` // Docker container ID for cleanup @@ -1663,6 +1664,7 @@ func (s *TaskSession) ToAPI() map[string]interface{} { result := map[string]interface{}{ "id": s.ID, "task_id": s.TaskID, + "queue_incarnation_id": s.QueueIncarnationID, "agent_execution_id": s.AgentExecutionID, "container_id": s.ContainerID, "agent_profile_id": s.AgentProfileID, diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index 511ba198c4..4302154bb9 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -363,7 +363,7 @@ type SessionRepository interface { HasActiveTaskSessionsByRepository(ctx context.Context, repositoryID string) (bool, error) CountActiveTaskSessionsByRepository(ctx context.Context, repositoryID string) (int, error) DeleteEphemeralTasksByAgentProfile(ctx context.Context, agentProfileID string) (int64, error) - DeleteTaskSession(ctx context.Context, id string) error + DeleteTaskSession(ctx context.Context, session *models.TaskSession) error GetPrimarySessionByTaskID(ctx context.Context, taskID string) (*models.TaskSession, error) GetPrimarySessionIDsByTaskIDs(ctx context.Context, taskIDs []string) (map[string]string, error) GetSessionCountsByTaskIDs(ctx context.Context, taskIDs []string) (map[string]int, error) diff --git a/apps/backend/internal/task/repository/session_repository_test.go b/apps/backend/internal/task/repository/session_repository_test.go index 43c6a53a05..3ec4096ca5 100644 --- a/apps/backend/internal/task/repository/session_repository_test.go +++ b/apps/backend/internal/task/repository/session_repository_test.go @@ -75,7 +75,7 @@ func TestSQLiteRepository_TaskSessionCRUD(t *testing.T) { } // Delete agent session - if err := repo.DeleteTaskSession(ctx, session.ID); err != nil { + if err := repo.DeleteTaskSession(ctx, retrieved); err != nil { t.Fatalf("failed to delete agent session: %v", err) } _, err = repo.GetTaskSession(ctx, session.ID) @@ -99,7 +99,9 @@ func TestSQLiteRepository_TaskSessionNotFound(t *testing.T) { t.Error("expected error for updating nonexistent agent session") } - err = repo.DeleteTaskSession(ctx, "nonexistent") + err = repo.DeleteTaskSession(ctx, &models.TaskSession{ + ID: "nonexistent", TaskID: "task-123", QueueIncarnationID: "missing", + }) if err == nil { t.Error("expected error for deleting nonexistent agent session") } diff --git a/apps/backend/internal/task/repository/sqlite/attachment.go b/apps/backend/internal/task/repository/sqlite/attachment.go index 6746b29c0a..9f6fef2429 100644 --- a/apps/backend/internal/task/repository/sqlite/attachment.go +++ b/apps/backend/internal/task/repository/sqlite/attachment.go @@ -3,12 +3,14 @@ package sqlite import ( "context" "database/sql" + "encoding/json" "fmt" "strings" "time" "github.com/google/uuid" "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/db/dialect" "github.com/kandev/kandev/internal/task/models" ) @@ -105,7 +107,23 @@ func (r *Repository) ClaimMessageAttachments(ctx context.Context, ids []string, return fmt.Errorf("begin attachment claim: %w", err) } defer func() { _ = tx.Rollback() }() - + if taskID != "" { + result, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE tasks SET updated_at = updated_at WHERE id = ? + `), taskID) + if err != nil { + return fmt.Errorf("lock attachment task %q: %w", taskID, err) + } + rows, _ := result.RowsAffected() + if rows == 0 { + return fmt.Errorf("%w: %s", ErrTaskNotFound, taskID) + } + } + if sessionID != "" { + if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), sessionID); err != nil { + return err + } + } now := time.Now().UTC() selection, err := r.selectAttachmentsForClaim(ctx, tx, ids, ownerID, workspaceID, taskID, sessionID, now) if err != nil { @@ -219,14 +237,50 @@ func (r *Repository) DeleteClaimedMessageAttachments(ctx context.Context, ids [] if len(ids) == 0 { return nil, nil } - placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") - args := []interface{}{ownerID, taskID, sessionID, models.AttachmentStateClaimed} - args = append(args, idsToInterfaces(ids)...) + queueTablePresent, err := r.tableExists("queued_messages") + if err != nil { + return nil, fmt.Errorf("probe queue attachment references: %w", err) + } tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return nil, fmt.Errorf("begin claimed attachment release: %w", err) } defer func() { _ = tx.Rollback() }() + if sessionID != "" { + if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), sessionID); err != nil { + return nil, err + } + } + if queueTablePresent { + if err := lockAttachmentQueueSession(ctx, tx, r.db.DriverName(), sessionID); err != nil { + return nil, err + } + } + referenced, err := referencedAttachmentIDsTx(ctx, r, tx, taskID, sessionID, queueTablePresent) + if err != nil { + return nil, err + } + releasable := make([]string, 0, len(ids)) + seen := make(map[string]struct{}, len(ids)) + for _, id := range ids { + if id == "" { + continue + } + if _, ok := referenced[id]; ok { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + releasable = append(releasable, id) + } + if len(releasable) == 0 { + return nil, nil + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(releasable)), ",") + args := []interface{}{ownerID, taskID, sessionID, models.AttachmentStateClaimed} + args = append(args, idsToInterfaces(releasable)...) rows, err := tx.QueryxContext(ctx, tx.Rebind(` SELECT `+attachmentSelectColumns+` FROM task_message_attachments WHERE owner_id = ? AND task_id = ? AND session_id = ? AND state = ? @@ -273,6 +327,248 @@ func (r *Repository) DeleteClaimedMessageAttachments(ctx context.Context, ids [] return released, nil } +func lockAttachmentQueueSession( + ctx context.Context, + tx *sqlx.Tx, + driverName, sessionID string, +) error { + if sessionID == "" || !dialect.IsPostgres(driverName) { + return nil + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO queue_session_locks (session_id) VALUES ($1) + ON CONFLICT(session_id) DO NOTHING + `, sessionID); err != nil { + return fmt.Errorf("ensure attachment queue session lock %q: %w", sessionID, err) + } + var lockedSessionID string + if err := tx.GetContext( + ctx, + &lockedSessionID, + `SELECT session_id FROM queue_session_locks WHERE session_id = $1 FOR UPDATE`, + sessionID, + ); err != nil { + return fmt.Errorf("lock attachment queue session %q: %w", sessionID, err) + } + return nil +} + +type attachmentReference struct { + AttachmentID string `json:"attachment_id"` +} + +func referencedAttachmentIDsTx( + ctx context.Context, + r *Repository, + tx *sqlx.Tx, + taskID, sessionID string, + queueTablePresent bool, +) (map[string]struct{}, error) { + referenced := make(map[string]struct{}) + //nolint:nestif // durable queue and transcript references use separate optional tables. + if queueTablePresent { + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT attachments_json FROM queued_messages WHERE task_id = ? AND session_id = ? + `), taskID, sessionID) + if err != nil { + return nil, fmt.Errorf("list queued attachment references: %w", err) + } + for rows.Next() { + var encoded string + if err := rows.Scan(&encoded); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan queued attachment references: %w", err) + } + var attachments []attachmentReference + if err := json.Unmarshal([]byte(encoded), &attachments); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("decode queued attachment references: %w", err) + } + addAttachmentReferences(referenced, attachments) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("iterate queued attachment references: %w", err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close queued attachment references: %w", err) + } + } + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT metadata FROM task_session_messages WHERE task_id = ? AND task_session_id = ? + `), taskID, sessionID) + if err != nil { + return nil, fmt.Errorf("list transcript attachment references: %w", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var encoded string + if err := rows.Scan(&encoded); err != nil { + return nil, fmt.Errorf("scan transcript attachment references: %w", err) + } + var metadata struct { + Attachments []attachmentReference `json:"attachments"` + } + if err := json.Unmarshal([]byte(encoded), &metadata); err != nil { + return nil, fmt.Errorf("decode transcript attachment references: %w", err) + } + addAttachmentReferences(referenced, metadata.Attachments) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate transcript attachment references: %w", err) + } + return referenced, nil +} + +func (r *Repository) queuedAttachmentIDsForTaskTx( + ctx context.Context, + tx *sqlx.Tx, + taskID string, +) (map[string]struct{}, error) { + ids := make(map[string]struct{}) + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT attachments_json FROM queued_messages WHERE task_id = ? + `), taskID) + if err != nil { + return nil, fmt.Errorf("list queued attachment references for archive: %w", err) + } + for rows.Next() { + var encoded string + if err := rows.Scan(&encoded); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan queued attachment references for archive: %w", err) + } + var attachments []attachmentReference + if err := json.Unmarshal([]byte(encoded), &attachments); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("decode queued attachment references for archive: %w", err) + } + addAttachmentReferences(ids, attachments) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("iterate queued attachment references for archive: %w", err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close queued attachment references for archive: %w", err) + } + return ids, nil +} + +// releaseUnreferencedTaskAttachmentClaimsTx stages queue-owned claims that no +// transcript message references. Only IDs present in queuedIDs are eligible: +// a direct prompt Claim commits before its transcript row exists, so staging +// every unreferenced claimed row would release an in-flight direct claim whose +// message inserts after the archive commits. Queue admission claims atomically +// with its queue row, so the pre-purge queued set is the exact ownership +// boundary. Rows are collected before any UPDATE so no cursor stays open +// across statements on a single-connection Postgres transaction. +func (r *Repository) releaseUnreferencedTaskAttachmentClaimsTx( + ctx context.Context, + tx *sqlx.Tx, + taskID string, + queuedIDs map[string]struct{}, +) error { + if len(queuedIDs) == 0 { + return nil + } + referenced := make(map[string]struct{}) + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT metadata FROM task_session_messages WHERE task_id = ? + `), taskID) + if err != nil { + return fmt.Errorf("list task attachment references for archive: %w", err) + } + for rows.Next() { + var encoded string + if err := rows.Scan(&encoded); err != nil { + _ = rows.Close() + return fmt.Errorf("scan task attachment references for archive: %w", err) + } + var metadata struct { + Attachments []attachmentReference `json:"attachments"` + } + if err := json.Unmarshal([]byte(encoded), &metadata); err != nil { + _ = rows.Close() + return fmt.Errorf("decode task attachment references for archive: %w", err) + } + addAttachmentReferences(referenced, metadata.Attachments) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate task attachment references for archive: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close task attachment references for archive: %w", err) + } + var unreferenced []string + for id := range queuedIDs { + if _, ok := referenced[id]; !ok { + unreferenced = append(unreferenced, id) + } + } + now := time.Now().UTC() + for _, attachmentID := range unreferenced { + if _, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE task_message_attachments + SET state = ?, expires_at = ?, updated_at = ? + WHERE id = ? AND task_id = ? AND state = ? + `), models.AttachmentStateStaged, now, now, attachmentID, taskID, models.AttachmentStateClaimed); err != nil { + return fmt.Errorf("release task attachment claim for archive: %w", err) + } + } + return nil +} + +func (r *Repository) deleteUnreferencedSessionAttachmentClaimsTx( + ctx context.Context, + tx *sqlx.Tx, + taskID, sessionID string, +) ([]*models.TaskMessageAttachment, error) { + // The session, its transcript, and its queue rows are deleted by this + // transaction. Return descriptors before deleting their registry rows so + // the caller can remove private bytes after commit. + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT `+attachmentSelectColumns+` + FROM task_message_attachments + WHERE task_id = ? AND session_id = ? AND state = ? + `), taskID, sessionID, models.AttachmentStateClaimed) + if err != nil { + return nil, fmt.Errorf("list deleted-session attachment claims: %w", err) + } + var attachments []*models.TaskMessageAttachment + for rows.Next() { + attachment := &models.TaskMessageAttachment{} + if err := rows.StructScan(attachment); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan deleted-session attachment claim: %w", err) + } + attachments = append(attachments, attachment) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("iterate deleted-session attachment claims: %w", err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close deleted-session attachment claims: %w", err) + } + if _, err := tx.ExecContext(ctx, tx.Rebind(` + DELETE FROM task_message_attachments + WHERE task_id = ? AND session_id = ? AND state = ? + `), taskID, sessionID, models.AttachmentStateClaimed); err != nil { + return nil, fmt.Errorf("delete deleted-session attachment claims: %w", err) + } + return attachments, nil +} + +func addAttachmentReferences(target map[string]struct{}, attachments []attachmentReference) { + for _, attachment := range attachments { + if attachment.AttachmentID != "" { + target[attachment.AttachmentID] = struct{}{} + } + } +} + func (r *Repository) DeleteMessageAttachmentsByTask(ctx context.Context, taskID string) ([]*models.TaskMessageAttachment, error) { if taskID == "" { return nil, nil @@ -313,6 +609,42 @@ func (r *Repository) DeleteMessageAttachmentsByTask(ctx context.Context, taskID return attachments, nil } +func (r *Repository) DeleteMessageAttachmentsByWorkspaceTx( + ctx context.Context, + tx *sqlx.Tx, + workspaceID string, +) ([]*models.TaskMessageAttachment, error) { + if workspaceID == "" { + return nil, nil + } + rows, err := tx.QueryxContext(ctx, tx.Rebind(` + SELECT `+attachmentSelectColumns+` FROM task_message_attachments WHERE workspace_id = ? + `), workspaceID) + if err != nil { + return nil, fmt.Errorf("list workspace attachments for cleanup: %w", err) + } + var attachments []*models.TaskMessageAttachment + for rows.Next() { + attachment := &models.TaskMessageAttachment{} + if err := rows.StructScan(attachment); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan workspace attachment for cleanup: %w", err) + } + attachments = append(attachments, attachment) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("iterate workspace attachments for cleanup: %w", err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close workspace attachments for cleanup: %w", err) + } + if _, err := tx.ExecContext(ctx, tx.Rebind(`DELETE FROM task_message_attachments WHERE workspace_id = ?`), workspaceID); err != nil { + return nil, fmt.Errorf("delete workspace attachments: %w", err) + } + return attachments, nil +} + func (r *Repository) MarkExpiredMessageAttachments(ctx context.Context, now time.Time) ([]*models.TaskMessageAttachment, error) { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { diff --git a/apps/backend/internal/task/repository/sqlite/attachment_lifecycle_coverage_test.go b/apps/backend/internal/task/repository/sqlite/attachment_lifecycle_coverage_test.go index bf5c7ca429..995683e96c 100644 --- a/apps/backend/internal/task/repository/sqlite/attachment_lifecycle_coverage_test.go +++ b/apps/backend/internal/task/repository/sqlite/attachment_lifecycle_coverage_test.go @@ -57,6 +57,8 @@ func TestClaimedAttachmentCleanupReturnsOnlyMatchingRows(t *testing.T) { repo := newRepoForEntityTests(t) ctx := context.Background() seedWorkspace(t, repo, "workspace") + seedAttachmentTask(t, repo, "task", "workspace") + seedAttachmentTask(t, repo, "other-task", "workspace") expires := time.Now().UTC().Add(time.Hour) for _, attachment := range []*models.TaskMessageAttachment{ {ID: "attachment-matching", OwnerID: "owner", WorkspaceID: "workspace", Name: "matching", SizeBytes: 1, ExpiresAt: expires}, diff --git a/apps/backend/internal/task/repository/sqlite/attachment_postgres_test.go b/apps/backend/internal/task/repository/sqlite/attachment_postgres_test.go index 8b30d94740..60af02118f 100644 --- a/apps/backend/internal/task/repository/sqlite/attachment_postgres_test.go +++ b/apps/backend/internal/task/repository/sqlite/attachment_postgres_test.go @@ -19,6 +19,9 @@ func TestPostgresMessageAttachmentLifecycle(t *testing.T) { if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "workspace-attachments-pg", Name: "Attachments PG"}); err != nil { t.Fatalf("seed workspace: %v", err) } + if err := repo.CreateTask(ctx, &models.Task{ID: "task-pg", WorkspaceID: "workspace-attachments-pg", Title: "Attachment PG"}); err != nil { + t.Fatalf("seed task: %v", err) + } t.Run("claim and release", func(t *testing.T) { now := time.Now().UTC() diff --git a/apps/backend/internal/task/repository/sqlite/attachment_test.go b/apps/backend/internal/task/repository/sqlite/attachment_test.go index f3408ff8f6..b7bb8d87f5 100644 --- a/apps/backend/internal/task/repository/sqlite/attachment_test.go +++ b/apps/backend/internal/task/repository/sqlite/attachment_test.go @@ -2,16 +2,127 @@ package sqlite import ( "context" + "errors" "testing" "time" "github.com/kandev/kandev/internal/task/models" ) +func seedAttachmentTask(t *testing.T, repo *Repository, taskID, workspaceID string) { + t.Helper() + if err := repo.CreateTask(context.Background(), &models.Task{ + ID: taskID, WorkspaceID: workspaceID, Title: taskID, + }); err != nil { + t.Fatalf("CreateTask(%s): %v", taskID, err) + } +} + +func TestReleaseUnreferencedTaskAttachmentClaimsTxStagesQueueOnlyClaims(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-attachments") + attachment := &models.TaskMessageAttachment{ + ID: "attachment-archive-queue-only", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 16, StorageKey: "attachment-archive-queue-only", State: models.AttachmentStateClaimed, + TaskID: "task-archive", SessionID: "session-archive", CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + tx, err := repo.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := repo.releaseUnreferencedTaskAttachmentClaimsTx(ctx, tx, attachment.TaskID, map[string]struct{}{attachment.ID: {}}); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + got, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if got.State != models.AttachmentStateStaged || got.TaskID != attachment.TaskID || got.SessionID != attachment.SessionID { + t.Fatalf("released archive claim = %+v", got) + } +} + +func TestReleaseUnreferencedTaskAttachmentClaimsTxPreservesDirectClaims(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-attachments") + attachment := &models.TaskMessageAttachment{ + ID: "attachment-archive-direct", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 16, StorageKey: "attachment-archive-direct", State: models.AttachmentStateClaimed, + TaskID: "task-archive", SessionID: "session-archive", CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + tx, err := repo.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if err := repo.releaseUnreferencedTaskAttachmentClaimsTx(ctx, tx, attachment.TaskID, nil); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + got, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if got.State != models.AttachmentStateClaimed { + t.Fatalf("direct claim state = %q, want claimed", got.State) + } +} + +func TestDeleteMessageAttachmentsByWorkspaceTxRemovesRegistryRowsBeforeCascade(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-attachments") + attachment := &models.TaskMessageAttachment{ + ID: "attachment-workspace-delete", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 16, StorageKey: "attachment-workspace-delete", State: models.AttachmentStateClaimed, + TaskID: "task-1", SessionID: "session-1", CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + tx, err := repo.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatal(err) + } + removed, err := repo.DeleteMessageAttachmentsByWorkspaceTx(ctx, tx, "workspace-attachments") + if err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + if len(removed) != 1 || removed[0].ID != attachment.ID { + _ = tx.Rollback() + t.Fatalf("removed attachments = %+v", removed) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err := repo.GetMessageAttachment(ctx, attachment.ID); err == nil { + t.Fatal("workspace attachment registry row still exists") + } +} + func TestClaimMessageAttachments_IsIdempotentForSameTaskSession(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() seedWorkspace(t, repo, "workspace-attachments") + seedAttachmentTask(t, repo, "task-1", "workspace-attachments") now := time.Now().UTC() attachment := &models.TaskMessageAttachment{ ID: "attachment-idempotent", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", @@ -41,6 +152,7 @@ func TestClaimMessageAttachments_AllowsTaskScopedClaimForSameTaskSession(t *test repo := newRepoForSessionTests(t) ctx := context.Background() seedWorkspace(t, repo, "workspace-attachments") + seedAttachmentTask(t, repo, "task-1", "workspace-attachments") now := time.Now().UTC() attachment := &models.TaskMessageAttachment{ ID: "attachment-task-scoped", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", @@ -70,6 +182,7 @@ func TestClaimMessageAttachments_RejectsDifferentSessionAfterSessionClaim(t *tes repo := newRepoForSessionTests(t) ctx := context.Background() seedWorkspace(t, repo, "workspace-attachments") + seedAttachmentTask(t, repo, "task-1", "workspace-attachments") now := time.Now().UTC() attachment := &models.TaskMessageAttachment{ ID: "attachment-session-scoped", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", @@ -92,6 +205,7 @@ func TestClaimMessageAttachments_AllowsSeparateSubmissionsToReachTheLimit(t *tes repo := newRepoForSessionTests(t) ctx := context.Background() seedWorkspace(t, repo, "workspace-attachments") + seedAttachmentTask(t, repo, "task-1", "workspace-attachments") now := time.Now().UTC() first := &models.TaskMessageAttachment{ ID: "attachment-first", OwnerID: "owner-1", WorkspaceID: "workspace-attachments", @@ -120,6 +234,7 @@ func TestClaimMessageAttachments_AllowsSeparateSubmissionsToReachTheLimit(t *tes func TestClaimMessageAttachments_RejectsExpiredStagedDescriptor(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() + seedWorkspace(t, repo, "workspace-attachments") now := time.Now().UTC() attachment := &models.TaskMessageAttachment{ @@ -142,6 +257,35 @@ func TestClaimMessageAttachments_RejectsExpiredStagedDescriptor(t *testing.T) { t.Fatalf("expired attachment state = %q, want staged", got.State) } } +func TestDeleteTaskSessionWithAttachmentsReturnsDeletedClaimDescriptors(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-session-attachments") + seedAttachmentTask(t, repo, "task-session-attachments", "workspace-session-attachments") + session := &models.TaskSession{ID: "session-attachments", TaskID: "task-session-attachments"} + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + attachment := &models.TaskMessageAttachment{ + ID: "attachment-session-delete", OwnerID: "owner", WorkspaceID: "workspace-session-attachments", + Name: "session.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 4, StorageKey: "attachment-session-delete", State: models.AttachmentStateClaimed, + TaskID: session.TaskID, SessionID: session.ID, CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatalf("CreateMessageAttachment: %v", err) + } + deleted, err := repo.DeleteTaskSessionWithAttachments(ctx, session) + if err != nil { + t.Fatalf("DeleteTaskSessionWithAttachments: %v", err) + } + if len(deleted) != 1 || deleted[0].ID != attachment.ID { + t.Fatalf("deleted attachments = %+v, want %s", deleted, attachment.ID) + } + if _, err := repo.GetMessageAttachment(ctx, attachment.ID); !errors.Is(err, models.ErrAttachmentNotFound) { + t.Fatalf("attachment registry row still exists: %v", err) + } +} func TestDeleteClaimedMessageAttachments_RemovesOnlyMatchingClaims(t *testing.T) { repo := newRepoForSessionTests(t) diff --git a/apps/backend/internal/task/repository/sqlite/base.go b/apps/backend/internal/task/repository/sqlite/base.go index b0a1dcb6f4..6d6f59f875 100644 --- a/apps/backend/internal/task/repository/sqlite/base.go +++ b/apps/backend/internal/task/repository/sqlite/base.go @@ -16,14 +16,15 @@ import ( // Repository provides SQLite-based task storage operations. type Repository struct { - db *sqlx.DB // writer - ro *sqlx.DB // reader (read-only pool) - ownsDB bool - log *logger.Logger - migrate *db.MigrateLogger - queuePurgeMu sync.RWMutex - queuePurger func(context.Context, string) - queuePurgeNotify func(context.Context, string) + db *sqlx.DB // writer + ro *sqlx.DB // reader (read-only pool) + ownsDB bool + log *logger.Logger + migrate *db.MigrateLogger + queuePurgeMu sync.RWMutex + queuePurger func(context.Context, string) + queuePurgePrepare func(context.Context, string) + queuePurgeNotify func(context.Context, string) // clockNow is a test-only clock seam. Set it before any concurrent // repository call; it carries no synchronization. clockNow func() time.Time @@ -115,6 +116,14 @@ func (r *Repository) SetTaskQueuePurger(purger func(context.Context, string)) { r.queuePurger = purger } +// SetTaskQueuePurgePreparer registers a bounded in-process cancellation hook +// invoked before a task lifecycle transaction purges queue rows. +func (r *Repository) SetTaskQueuePurgePreparer(prepare func(context.Context, string)) { + r.queuePurgeMu.Lock() + defer r.queuePurgeMu.Unlock() + r.queuePurgePrepare = prepare +} + // SetTaskQueuePurgeNotifier registers a post-commit observer for every path // that purges a task's queued_messages (archive/delete/workspace cascade). // Callers must not purge the production SQLite queue again — that already @@ -127,6 +136,15 @@ func (r *Repository) SetTaskQueuePurgeNotifier(notifier func(context.Context, st r.queuePurgeNotify = notifier } +func (r *Repository) notifyTaskQueuePurging(ctx context.Context, taskID string) { + r.queuePurgeMu.RLock() + prepare := r.queuePurgePrepare + r.queuePurgeMu.RUnlock() + if prepare != nil { + prepare(ctx, taskID) + } +} + func (r *Repository) notifyTaskQueuePurged(ctx context.Context, taskID string) { r.queuePurgeMu.RLock() purger := r.queuePurger diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index bfef93aa6c..2bf16bb712 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/google/uuid" + "go.uber.org/zap" "github.com/kandev/kandev/internal/db/dialect" @@ -303,6 +305,10 @@ func (r *Repository) runMigrations() error { // frontend snapshots the prior value before the advance to position the // "New" divider (see models.TaskSession.LastReadMessageID). r.migrate.Apply("task_sessions.last_read_message_id", `ALTER TABLE task_sessions ADD COLUMN last_read_message_id TEXT DEFAULT ''`) + _ = r.migrate.Apply("task_sessions.queue_incarnation_id", `ALTER TABLE task_sessions ADD COLUMN queue_incarnation_id TEXT NOT NULL DEFAULT ''`) + if err := r.backfillTaskSessionQueueIncarnations(); err != nil { + return err + } r.migrate.Apply("task_session_turns.execution_profile_id", `ALTER TABLE task_session_turns ADD COLUMN execution_profile_id TEXT NOT NULL DEFAULT ''`) r.migrate.Apply("task_session_turns.route_generation", `ALTER TABLE task_session_turns ADD COLUMN route_generation BIGINT NOT NULL DEFAULT 0`) @@ -366,6 +372,37 @@ func (r *Repository) runMigrations() error { return nil } +func (r *Repository) backfillTaskSessionQueueIncarnations() error { + rows, err := r.db.Queryx(`SELECT id FROM task_sessions WHERE queue_incarnation_id = ''`) + if err != nil { + return fmt.Errorf("list sessions missing queue incarnation: %w", err) + } + var sessionIDs []string + for rows.Next() { + var sessionID string + if err := rows.Scan(&sessionID); err != nil { + _ = rows.Close() + return fmt.Errorf("scan session missing queue incarnation: %w", err) + } + sessionIDs = append(sessionIDs, sessionID) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close queue incarnation rows: %w", err) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate sessions missing queue incarnation: %w", err) + } + for _, sessionID := range sessionIDs { + if _, err := r.db.Exec(r.db.Rebind(` + UPDATE task_sessions + SET queue_incarnation_id = ? + WHERE id = ? AND queue_incarnation_id = '' + `), uuid.NewString(), sessionID); err != nil { + return fmt.Errorf("backfill queue incarnation for session %s: %w", sessionID, err) + } + } + return nil +} // backfillPromptSeq assigns existing user rows their previously derived // ordinal (the correlated count over the session's user rows using the same diff --git a/apps/backend/internal/task/repository/sqlite/base_schema.go b/apps/backend/internal/task/repository/sqlite/base_schema.go index 8b1406336f..e9399432f6 100644 --- a/apps/backend/internal/task/repository/sqlite/base_schema.go +++ b/apps/backend/internal/task/repository/sqlite/base_schema.go @@ -974,6 +974,7 @@ const sessionWorktreeSchemaDDL = ` CREATE TABLE IF NOT EXISTS task_sessions ( id TEXT PRIMARY KEY, task_id TEXT NOT NULL, + queue_incarnation_id TEXT NOT NULL DEFAULT '', agent_execution_id TEXT NOT NULL DEFAULT '', container_id TEXT NOT NULL DEFAULT '', agent_profile_id TEXT, diff --git a/apps/backend/internal/task/repository/sqlite/git_snapshots_environment_test.go b/apps/backend/internal/task/repository/sqlite/git_snapshots_environment_test.go index c075f43f98..50edd18efe 100644 --- a/apps/backend/internal/task/repository/sqlite/git_snapshots_environment_test.go +++ b/apps/backend/internal/task/repository/sqlite/git_snapshots_environment_test.go @@ -39,7 +39,7 @@ func TestGitSnapshotEnvironmentPreservesStatusAfterSessionDelete(t *testing.T) { t.Fatalf("CreateGitSnapshot: %v", err) } - if err := repo.DeleteTaskSession(ctx, sessionID); err != nil { + if err := deleteTaskSessionForTest(t, repo, ctx, sessionID); err != nil { t.Fatalf("DeleteTaskSession: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/git_snapshots_test.go b/apps/backend/internal/task/repository/sqlite/git_snapshots_test.go index e623512917..96440caacd 100644 --- a/apps/backend/internal/task/repository/sqlite/git_snapshots_test.go +++ b/apps/backend/internal/task/repository/sqlite/git_snapshots_test.go @@ -747,7 +747,7 @@ func TestGitSnapshotsAndCommitsCascadeOnSessionDelete(t *testing.T) { t.Fatalf("CreateSessionCommit: %v", err) } - if err := repo.DeleteTaskSession(ctx, "session-cascade"); err != nil { + if err := deleteTaskSessionForTest(t, repo, ctx, "session-cascade"); err != nil { t.Fatalf("DeleteTaskSession: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/prompt_history_test.go b/apps/backend/internal/task/repository/sqlite/prompt_history_test.go index 35dca0b30d..7e1bc95bc5 100644 --- a/apps/backend/internal/task/repository/sqlite/prompt_history_test.go +++ b/apps/backend/internal/task/repository/sqlite/prompt_history_test.go @@ -174,7 +174,7 @@ func TestDeleteTaskSessionRemovesPromptHistoryClaim(t *testing.T) { if err != nil || !claimed { t.Fatalf("initial fallback claim = %t, %v; want claimed", claimed, err) } - if err := repo.DeleteTaskSession(ctx, "session-session-reuse"); err != nil { + if err := deleteTaskSessionForTest(t, repo, ctx, "session-session-reuse"); err != nil { t.Fatalf("delete session: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/queue_attachment_atomic_test.go b/apps/backend/internal/task/repository/sqlite/queue_attachment_atomic_test.go new file mode 100644 index 0000000000..136f660aaf --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/queue_attachment_atomic_test.go @@ -0,0 +1,287 @@ +package sqlite + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + "github.com/kandev/kandev/internal/task/models" +) + +func TestQueueAdmissionClaimsAttachmentsAtomically(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-queue-claim") + if err := repo.CreateTask(ctx, &models.Task{ID: "task-queue-claim", WorkspaceID: "workspace-queue-claim", Title: "Queue claim"}); err != nil { + t.Fatal(err) + } + session := &models.TaskSession{ID: "session-queue-claim", TaskID: "task-queue-claim"} + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatal(err) + } + attachment := &models.TaskMessageAttachment{ + ID: "attachment-queue-claim", OwnerID: "owner-queue-claim", WorkspaceID: "workspace-queue-claim", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 5, StorageKey: "attachment-queue-claim", State: models.AttachmentStateStaged, + ExpiresAt: time.Now().UTC().Add(time.Hour), CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + + db := sqlx.NewDb(repo.DB(), "sqlite3") + queueRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatal(err) + } + queue := messagequeue.NewService(queueRepo, messagequeue.DefaultMaxPerSession, logger.Default()) + identity := messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, SessionID: session.ID, SessionIncarnationID: session.QueueIncarnationID, + } + queued, err := queue.QueueMessageWithMetadataForSessionWithClaim( + ctx, identity, "review", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{{AttachmentID: attachment.ID, Name: attachment.Name, DeliveryMode: attachment.DeliveryMode}}, + nil, + messagequeue.QueueAttachmentClaim{OwnerID: attachment.OwnerID, WorkspaceID: attachment.WorkspaceID, IDs: []string{attachment.ID}}, + ) + if err != nil { + t.Fatalf("queue with attachment claim: %v", err) + } + if queued == nil { + t.Fatal("queued message is nil") + } + stored, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != models.AttachmentStateClaimed || stored.TaskID != session.TaskID || stored.SessionID != session.ID { + t.Fatalf("attachment claim = %+v", stored) + } +} + +func TestQueueAdmissionRollsBackAttachmentClaimWhenInsertFails(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-queue-rollback") + if err := repo.CreateTask(ctx, &models.Task{ID: "task-queue-rollback", WorkspaceID: "workspace-queue-rollback", Title: "Queue rollback"}); err != nil { + t.Fatal(err) + } + session := &models.TaskSession{ID: "session-queue-rollback", TaskID: "task-queue-rollback"} + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatal(err) + } + attachment := &models.TaskMessageAttachment{ + ID: "attachment-queue-rollback", OwnerID: "owner-queue-rollback", WorkspaceID: "workspace-queue-rollback", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 5, StorageKey: "attachment-queue-rollback", State: models.AttachmentStateStaged, + ExpiresAt: time.Now().UTC().Add(time.Hour), CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + + db := sqlx.NewDb(repo.DB(), "sqlite3") + queueRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatal(err) + } + if _, err := repo.DB().Exec(` + CREATE TRIGGER fail_queue_insert + BEFORE INSERT ON queued_messages + BEGIN + SELECT RAISE(ABORT, 'forced queue insert failure'); + END + `); err != nil { + t.Fatal(err) + } + queue := messagequeue.NewService(queueRepo, messagequeue.DefaultMaxPerSession, logger.Default()) + identity := messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, SessionID: session.ID, SessionIncarnationID: session.QueueIncarnationID, + } + _, err = queue.QueueMessageWithMetadataForSessionWithClaim( + ctx, identity, "review", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{{AttachmentID: attachment.ID, Name: attachment.Name, DeliveryMode: attachment.DeliveryMode}}, + nil, + messagequeue.QueueAttachmentClaim{OwnerID: attachment.OwnerID, WorkspaceID: attachment.WorkspaceID, IDs: []string{attachment.ID}}, + ) + if err == nil { + t.Fatal("queue admission with forced insert failure succeeded") + } + status, snapshotErr := queue.Snapshot(ctx, identity) + if snapshotErr != nil { + t.Fatal(snapshotErr) + } + if status.Count != 0 { + t.Fatalf("queue count = %d, want 0", status.Count) + } + stored, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != models.AttachmentStateStaged || stored.TaskID != "" || stored.SessionID != "" { + t.Fatalf("attachment changed after rollback = %+v", stored) + } +} + +func TestQueueAdmissionAtCapacityDoesNotClaimAttachments(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-queue-full-claim") + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-queue-full-claim", WorkspaceID: "workspace-queue-full-claim", Title: "Queue full claim", + }); err != nil { + t.Fatal(err) + } + session := &models.TaskSession{ID: "session-queue-full-claim", TaskID: "task-queue-full-claim"} + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatal(err) + } + attachment := &models.TaskMessageAttachment{ + ID: "attachment-queue-full-claim", OwnerID: "owner-queue-full-claim", WorkspaceID: "workspace-queue-full-claim", + Name: "notes.txt", MimeType: "text/plain", Kind: "resource", DeliveryMode: "path", + SizeBytes: 5, StorageKey: "attachment-queue-full-claim", State: models.AttachmentStateStaged, + ExpiresAt: time.Now().UTC().Add(time.Hour), CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + + db := sqlx.NewDb(repo.DB(), "sqlite3") + queueRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatal(err) + } + queue := messagequeue.NewService(queueRepo, 1, logger.Default()) + identity := messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, SessionID: session.ID, SessionIncarnationID: session.QueueIncarnationID, + } + first, err := queue.QueueMessageWithMetadataForSession( + ctx, identity, "first", "", messagequeue.QueuedByUser, false, nil, nil, + ) + if err != nil { + t.Fatal(err) + } + + queued, err := queue.QueueMessageWithMetadataForSessionWithClaim( + ctx, identity, "second", "", messagequeue.QueuedByUser, false, + []messagequeue.MessageAttachment{{ + AttachmentID: attachment.ID, Name: attachment.Name, DeliveryMode: attachment.DeliveryMode, + }}, + nil, + messagequeue.QueueAttachmentClaim{ + OwnerID: attachment.OwnerID, WorkspaceID: attachment.WorkspaceID, IDs: []string{attachment.ID}, + }, + ) + if !errors.Is(err, messagequeue.ErrQueueFull) || queued != nil { + t.Fatalf("full queue attachment admission = %+v, err=%v, want queue full", queued, err) + } + status, err := queue.Snapshot(ctx, identity) + if err != nil { + t.Fatal(err) + } + if status.Count != 1 || status.Entries[0].ID != first.ID || status.Entries[0].Content != "first" { + t.Fatalf("queue changed after rejected attachment admission: %+v", status) + } + stored, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != models.AttachmentStateStaged || stored.TaskID != "" || stored.SessionID != "" { + t.Fatalf("attachment changed after full queue rejection = %+v", stored) + } +} + +func TestQueueTransferMovesAttachmentClaimToDestinationSession(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-queue-transfer-claim") + task := &models.Task{ + ID: "task-queue-transfer-claim", WorkspaceID: "workspace-queue-transfer-claim", Title: "Queue transfer claim", + } + if err := repo.CreateTask(ctx, task); err != nil { + t.Fatal(err) + } + source := &models.TaskSession{ID: "session-queue-transfer-source", TaskID: task.ID} + destination := &models.TaskSession{ID: "session-queue-transfer-destination", TaskID: task.ID} + if err := repo.CreateTaskSession(ctx, source); err != nil { + t.Fatal(err) + } + if err := repo.CreateTaskSession(ctx, destination); err != nil { + t.Fatal(err) + } + attachment := &models.TaskMessageAttachment{ + ID: "attachment-queue-transfer", OwnerID: "owner-queue-transfer", WorkspaceID: task.WorkspaceID, + TaskID: task.ID, SessionID: source.ID, Name: "notes.txt", MimeType: "text/plain", + Kind: "resource", DeliveryMode: "path", SizeBytes: 5, StorageKey: "attachment-queue-transfer", + State: models.AttachmentStateClaimed, ExpiresAt: time.Now().UTC().Add(time.Hour), CreatedAt: time.Now().UTC(), + } + if err := repo.CreateMessageAttachment(ctx, attachment); err != nil { + t.Fatal(err) + } + + db := sqlx.NewDb(repo.DB(), "sqlite3") + queueRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatal(err) + } + sourceIdentity := messagequeue.QueueSessionIdentity{ + TaskID: task.ID, SessionID: source.ID, SessionIncarnationID: source.QueueIncarnationID, + } + destinationIdentity := messagequeue.QueueSessionIdentity{ + TaskID: task.ID, SessionID: destination.ID, SessionIncarnationID: destination.QueueIncarnationID, + } + if err := queueRepo.InsertForSession(ctx, sourceIdentity, &messagequeue.QueuedMessage{ + ID: "queue-transfer-attachment", TaskID: task.ID, SessionID: source.ID, Content: "review", + QueuedBy: messagequeue.QueuedByUser, QueuedAt: time.Now().UTC(), + Attachments: []messagequeue.MessageAttachment{{ + AttachmentID: attachment.ID, Name: attachment.Name, DeliveryMode: attachment.DeliveryMode, + }}, + }, messagequeue.DefaultMaxPerSession); err != nil { + t.Fatalf("queue source attachment: %v", err) + } + retained, err := repo.DeleteClaimedMessageAttachments( + ctx, []string{attachment.ID}, attachment.OwnerID, task.ID, source.ID, + ) + if err != nil { + t.Fatal(err) + } + if len(retained) != 0 { + t.Fatalf("released attachment still referenced by queue: %+v", retained) + } + if _, err := repo.GetMessageAttachment(ctx, attachment.ID); err != nil { + t.Fatalf("queued attachment was removed: %v", err) + } + if err := queueRepo.TransferSessionIdentities(ctx, sourceIdentity, destinationIdentity); err != nil { + t.Fatalf("transfer queue: %v", err) + } + stored, err := repo.GetMessageAttachment(ctx, attachment.ID) + if err != nil { + t.Fatal(err) + } + if stored.SessionID != destination.ID || stored.TaskID != task.ID || stored.State != models.AttachmentStateClaimed { + t.Fatalf("transferred attachment claim = %+v", stored) + } + removed, err := queueRepo.DeleteByIDForSession(ctx, destinationIdentity, "queue-transfer-attachment") + if err != nil { + t.Fatal(err) + } + if len(removed.Removed) != 1 || removed.Removed[0].ID != "queue-transfer-attachment" { + t.Fatalf("removed queue rows = %+v", removed) + } + released, err := repo.DeleteClaimedMessageAttachments( + ctx, []string{attachment.ID}, attachment.OwnerID, task.ID, destination.ID, + ) + if err != nil { + t.Fatal(err) + } + if len(released) != 1 || released[0].StorageKey != attachment.StorageKey { + t.Fatalf("released attachments = %+v", released) + } + if _, err := repo.GetMessageAttachment(ctx, attachment.ID); !errors.Is(err, models.ErrAttachmentNotFound) { + t.Fatalf("attachment remains after destination cleanup: %v", err) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/session.go b/apps/backend/internal/task/repository/sqlite/session.go index 3b7f2bc280..9daba39a99 100644 --- a/apps/backend/internal/task/repository/sqlite/session.go +++ b/apps/backend/internal/task/repository/sqlite/session.go @@ -17,6 +17,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/tracing" "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/task/models" ) @@ -471,7 +472,7 @@ func (r *Repository) ListTurnsBySession(ctx context.Context, sessionID string) ( // ADR 0005: agent_profile_id is the single column for both kanban (FK to a // shallow profile) and office (FK to a per-workspace rich profile) sessions — // the two column names that used to live here have collapsed into one. -const taskSessionSelectCols = `ts.id, ts.task_id, +const taskSessionSelectCols = `ts.id, ts.task_id, ts.queue_incarnation_id, COALESCE(er.agent_execution_id, ''), COALESCE(er.container_id, ''), ts.agent_profile_id, ts.execution_profile_id, ts.route_generation, ts.route_state, ts.route_reason, ts.downstream_acp_session_id, ts.executor_id, ts.executor_profile_id, ts.environment_id, @@ -899,6 +900,9 @@ func (r *Repository) createTaskSession(ctx context.Context, exec taskSessionExec if session.ID == "" { session.ID = uuid.New().String() } + if session.QueueIncarnationID == "" { + session.QueueIncarnationID = uuid.New().String() + } now := time.Now().UTC() // Only default StartedAt / UpdatedAt when the caller hasn't supplied // one. The test harness backdates StartedAt so completed sessions @@ -946,7 +950,7 @@ func (r *Repository) createTaskSession(ctx context.Context, exec taskSessionExec } _, err = exec.ExecContext(ctx, r.db.Rebind(` INSERT INTO task_sessions ( - id, task_id, agent_profile_id, execution_profile_id, route_generation, route_state, route_reason, downstream_acp_session_id, + id, task_id, queue_incarnation_id, agent_profile_id, execution_profile_id, route_generation, route_state, route_reason, downstream_acp_session_id, executor_id, executor_profile_id, environment_id, repository_id, base_branch, base_commit_sha, workspace_path, agent_profile_snapshot, executor_snapshot, environment_snapshot, repository_snapshot, @@ -955,9 +959,10 @@ func (r *Repository) createTaskSession(ctx context.Context, exec taskSessionExec ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ? ) - `), session.ID, session.TaskID, agentProfileID, + `), session.ID, session.TaskID, session.QueueIncarnationID, agentProfileID, session.ExecutionProfileID, session.RouteGeneration, session.RouteState, session.RouteReason, session.DownstreamACPSessionID, session.ExecutorID, session.ExecutorProfileID, session.EnvironmentID, session.RepositoryID, session.BaseBranch, session.BaseCommitSHA, session.WorkspacePath, string(agentProfileSnapshotJSON), string(executorSnapshotJSON), string(environmentSnapshotJSON), string(repositorySnapshotJSON), @@ -999,7 +1004,8 @@ func (r *Repository) scanTaskSession(ctx context.Context, row *sql.Row, noRowsEr var lastReadMessageID sql.NullString err := row.Scan( - &session.ID, &session.TaskID, &session.AgentExecutionID, &session.ContainerID, &agentProfileID, + &session.ID, &session.TaskID, &session.QueueIncarnationID, + &session.AgentExecutionID, &session.ContainerID, &agentProfileID, &session.ExecutionProfileID, &session.RouteGeneration, &session.RouteState, &session.RouteReason, &session.DownstreamACPSessionID, &session.ExecutorID, &session.ExecutorProfileID, &session.EnvironmentID, &session.RepositoryID, &session.BaseBranch, &session.BaseCommitSHA, &session.WorkspacePath, @@ -1115,6 +1121,23 @@ func (r *Repository) GetTaskSession(ctx context.Context, id string) (*models.Tas // prompt while its task is still active. It intentionally performs no agent // I/O; callers dispatch only after this bounded database claim commits. func (r *Repository) ClaimPromptableTaskSessionIfActive(ctx context.Context, id string) (models.PromptableTaskSessionClaim, error) { + return r.claimPromptableTaskSessionIfActive(ctx, id, "", "") +} + +func (r *Repository) ClaimPromptableTaskSessionIfActiveForIdentity( + ctx context.Context, + taskID, id, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { + if taskID == "" || id == "" || incarnationID == "" { + return models.PromptableTaskSessionClaim{Status: models.PromptableTaskSessionInactive}, nil + } + return r.claimPromptableTaskSessionIfActive(ctx, id, taskID, incarnationID) +} + +func (r *Repository) claimPromptableTaskSessionIfActive( + ctx context.Context, + id, taskID, incarnationID string, +) (models.PromptableTaskSessionClaim, error) { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return models.PromptableTaskSessionClaim{}, err @@ -1127,7 +1150,9 @@ func (r *Repository) ClaimPromptableTaskSessionIfActive(ctx context.Context, id SELECT ts.state, t.archived_at IS NULL FROM task_sessions ts JOIN tasks t ON t.id = ts.task_id WHERE ts.id = ? - `), id).Scan(&state, &active) + AND (? = '' OR ts.task_id = ?) + AND (? = '' OR ts.queue_incarnation_id = ?) + `), id, taskID, taskID, incarnationID, incarnationID).Scan(&state, &active) if errors.Is(err, sql.ErrNoRows) { return models.PromptableTaskSessionClaim{Status: models.PromptableTaskSessionInactive}, nil } @@ -1144,8 +1169,11 @@ func (r *Repository) ClaimPromptableTaskSessionIfActive(ctx context.Context, id result, err := tx.ExecContext(ctx, r.db.Rebind(` UPDATE task_sessions SET state = ?, completed_at = NULL, updated_at = ? WHERE id = ? AND state = ? + AND (? = '' OR task_id = ?) + AND (? = '' OR queue_incarnation_id = ?) AND EXISTS (SELECT 1 FROM tasks WHERE tasks.id = task_sessions.task_id AND tasks.archived_at IS NULL) - `), models.TaskSessionStateRunning, time.Now().UTC(), id, state) + `), models.TaskSessionStateRunning, time.Now().UTC(), id, state, + taskID, taskID, incarnationID, incarnationID) if err != nil { return models.PromptableTaskSessionClaim{}, err } @@ -1154,7 +1182,7 @@ func (r *Repository) ClaimPromptableTaskSessionIfActive(ctx context.Context, id return models.PromptableTaskSessionClaim{}, err } if changed == 0 { - claim, err := r.classifyPromptableTaskSessionClaim(ctx, tx, id) + claim, err := r.classifyPromptableTaskSessionClaimForIdentity(ctx, tx, id, taskID, incarnationID) if err != nil { return models.PromptableTaskSessionClaim{}, err } @@ -1176,7 +1204,17 @@ func (r *Repository) ClaimPromptableTaskSessionIfActive(ctx context.Context, id // It must run in the claim transaction so the result describes the same // ownership window as the failed UPDATE. func (r *Repository) classifyPromptableTaskSessionClaim( - ctx context.Context, tx *sqlx.Tx, id string, + ctx context.Context, + tx *sqlx.Tx, + id string, +) (models.PromptableTaskSessionClaim, error) { + return r.classifyPromptableTaskSessionClaimForIdentity(ctx, tx, id, "", "") +} + +func (r *Repository) classifyPromptableTaskSessionClaimForIdentity( + ctx context.Context, + tx *sqlx.Tx, + id, taskID, incarnationID string, ) (models.PromptableTaskSessionClaim, error) { var state models.TaskSessionState var active bool @@ -1184,7 +1222,9 @@ func (r *Repository) classifyPromptableTaskSessionClaim( SELECT ts.state, t.archived_at IS NULL FROM task_sessions ts JOIN tasks t ON t.id = ts.task_id WHERE ts.id = ? - `), id).Scan(&state, &active) + AND (? = '' OR ts.task_id = ?) + AND (? = '' OR ts.queue_incarnation_id = ?) + `), id, taskID, taskID, incarnationID, incarnationID).Scan(&state, &active) if errors.Is(err, sql.ErrNoRows) { return models.PromptableTaskSessionClaim{Status: models.PromptableTaskSessionInactive}, nil } @@ -1568,6 +1608,30 @@ func (r *Repository) UpdateTaskSessionStateIfCurrent( return rows > 0, now, nil } +func (r *Repository) UpdateTaskSessionStateIfCurrentIdentity( + ctx context.Context, + taskID, id, incarnationID string, + expected, status models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + now := time.Now().UTC() + completedAt := completedAtForTaskSessionState(status, now) + result, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE task_sessions + SET state = ?, error_message = ?, completed_at = ?, updated_at = ? + WHERE id = ? AND task_id = ? AND queue_incarnation_id = ? AND state = ? + `), string(status), errorMessage, completedAt, now, + id, taskID, incarnationID, string(expected)) + if err != nil { + return false, time.Time{}, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, time.Time{}, err + } + return rows > 0, now, nil +} + // UpdateTaskSessionDynamicRouteIfCurrent changes only the route projection // while the session generation and projected route state still match the // caller's observation. It avoids writing a stale full session row after an @@ -2790,7 +2854,8 @@ func scanTaskSessionRow(rows *sql.Rows) (*models.TaskSession, error) { var lastReadMessageID sql.NullString err := rows.Scan( - &session.ID, &session.TaskID, &session.AgentExecutionID, &session.ContainerID, &agentProfileID, + &session.ID, &session.TaskID, &session.QueueIncarnationID, + &session.AgentExecutionID, &session.ContainerID, &agentProfileID, &session.ExecutionProfileID, &session.RouteGeneration, &session.RouteState, &session.RouteReason, &session.DownstreamACPSessionID, &session.ExecutorID, &session.ExecutorProfileID, &session.EnvironmentID, &session.RepositoryID, &session.BaseBranch, &session.BaseCommitSHA, &session.WorkspacePath, @@ -2850,45 +2915,107 @@ func unmarshalSessionSnapshots( return unmarshalSessionJSON(repositorySnapshotJSON, &session.RepositorySnapshot, "repository snapshot") } -// DeleteTaskSession deletes an agent session by ID and any pending queue rows -// keyed to that session. Without the queue purge, orphan rows keep inflating -// task-scoped queued_prompt_count after the session is gone. -func (r *Repository) DeleteTaskSession(ctx context.Context, id string) error { +// DeleteTaskSession deletes the exact session incarnation and its pending queue +// rows. The session identity fences delayed deletion from removing a replacement +// that reuses the same textual session ID. +func (r *Repository) DeleteTaskSession(ctx context.Context, session *models.TaskSession) error { + _, err := r.DeleteTaskSessionWithAttachments(ctx, session) + return err +} + +// DeleteTaskSessionWithAttachments also returns claimed attachment descriptors +// whose registry rows were removed, allowing the service layer to remove their +// private bytes after the transaction commits. +func (r *Repository) DeleteTaskSessionWithAttachments( + ctx context.Context, + session *models.TaskSession, +) ([]*models.TaskMessageAttachment, error) { + if session == nil || session.ID == "" || session.TaskID == "" || session.QueueIncarnationID == "" { + return nil, errors.New("task session identity is required") + } tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return err + return nil, err } defer func() { _ = tx.Rollback() }() - result, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM task_sessions WHERE id = ?`), id) + if err := r.confirmTaskSessionDeletionIdentityTx(ctx, tx, session); err != nil { + return nil, err + } + deletedAttachments, err := r.purgeTaskSessionStateTx(ctx, tx, session) if err != nil { - return err + return nil, err } - rows, _ := result.RowsAffected() - if rows == 0 { - return fmt.Errorf("agent session not found: %s", id) + if err := tx.Commit(); err != nil { + return nil, err } - // task_session_prompt_seq intentionally has no foreign key because it was - // added by a replay-safe migration. Remove its session-scoped admission - // state explicitly so a later session that reuses this ID starts fresh. - if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM task_session_prompt_seq WHERE task_session_id = ?`), id); err != nil { - // Isolated unit tests may omit the prompt-sequence schema. Production - // always has it; treat a missing table as already-cleaned. - if !db.IsMissingTableError(err) { - return fmt.Errorf("purge prompt history for session %s: %w", id, err) - } - } - if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM queued_messages WHERE session_id = ?`), id); err != nil { - // Isolated unit tests may omit the messagequeue schema. Production - // always has queued_messages; treat a missing table as already-purged. - if !db.IsMissingTableError(err) { - return fmt.Errorf("purge queued messages for session %s: %w", id, err) - } - } - return tx.Commit() + return deletedAttachments, nil } // Task Session Worktree operations +func (r *Repository) confirmTaskSessionDeletionIdentityTx( + ctx context.Context, + tx *sqlx.Tx, + session *models.TaskSession, +) error { + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE tasks SET updated_at = updated_at WHERE id = ? + `), session.TaskID); err != nil { + return fmt.Errorf("lock deleted session task: %w", err) + } + var found string + err := tx.QueryRowxContext(ctx, r.db.Rebind(` + SELECT id + FROM task_sessions + WHERE id = ? AND task_id = ? AND queue_incarnation_id = ? + `), session.ID, session.TaskID, session.QueueIncarnationID).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("agent session not found: %s", session.ID) + } + if err != nil { + return fmt.Errorf("confirm deleted session identity: %w", err) + } + return nil +} + +func (r *Repository) purgeTaskSessionStateTx( + ctx context.Context, + tx *sqlx.Tx, + session *models.TaskSession, +) ([]*models.TaskMessageAttachment, error) { + identity := messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, + SessionID: session.ID, + SessionIncarnationID: session.QueueIncarnationID, + } + if err := messagequeue.DeleteSessionInTransaction(ctx, tx, r.db, identity); err != nil { + if !db.IsMissingTableError(err) { + return nil, fmt.Errorf("purge deleted session queue: %w", err) + } + } + deletedAttachments, err := r.deleteUnreferencedSessionAttachmentClaimsTx(ctx, tx, session.TaskID, session.ID) + if err != nil { + return nil, fmt.Errorf("purge deleted session attachment claims: %w", err) + } + result, err := tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM task_sessions + WHERE id = ? AND task_id = ? AND queue_incarnation_id = ? + `), session.ID, session.TaskID, session.QueueIncarnationID) + if err != nil { + return nil, err + } + rows, _ := result.RowsAffected() + if rows == 0 { + return nil, fmt.Errorf("agent session not found: %s", session.ID) + } + if _, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM task_session_prompt_seq WHERE task_session_id = ?`), session.ID); err != nil { + if !db.IsMissingTableError(err) { + return nil, fmt.Errorf("purge prompt history for session %s: %w", session.ID, err) + } + } + return deletedAttachments, nil +} + // // Sessions reference worktrees only through task_sessions.task_environment_id; // the physical worktree records live on task_environment_repos (owned by the diff --git a/apps/backend/internal/task/repository/sqlite/session_delete_identity_test.go b/apps/backend/internal/task/repository/sqlite/session_delete_identity_test.go new file mode 100644 index 0000000000..708d006077 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/session_delete_identity_test.go @@ -0,0 +1,59 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/kandev/kandev/internal/task/models" +) + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 +func deleteTaskSessionForTest(t *testing.T, repo *Repository, ctx context.Context, id string) error { + t.Helper() + session, err := repo.GetTaskSession(ctx, id) + if err != nil { + return err + } + if session.QueueIncarnationID == "" { + session.QueueIncarnationID = uuid.NewString() + if _, err := repo.db.Exec(repo.db.Rebind(` + UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ? + `), session.QueueIncarnationID, session.ID); err != nil { + return err + } + } + return repo.DeleteTaskSession(ctx, session) +} + +func TestDeleteTaskSessionDoesNotDeleteReplacementIncarnation(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-delete-identity" + sessionID = "session-delete-identity" + ) + if err := repo.CreateTask(ctx, &models.Task{ID: taskID, Title: "Identity fence"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: taskID}); err != nil { + t.Fatalf("CreateTaskSession initial: %v", err) + } + original, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("GetTaskSession initial: %v", err) + } + if err := repo.DeleteTaskSession(ctx, original); err != nil { + t.Fatalf("DeleteTaskSession initial: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: taskID}); err != nil { + t.Fatalf("CreateTaskSession replacement: %v", err) + } + + if err := repo.DeleteTaskSession(ctx, original); err == nil { + t.Fatal("DeleteTaskSession delayed original succeeded") + } + if _, err := repo.GetTaskSession(ctx, sessionID); err != nil { + t.Fatalf("replacement session was deleted: %v", err) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/session_incarnation_migration_test.go b/apps/backend/internal/task/repository/sqlite/session_incarnation_migration_test.go new file mode 100644 index 0000000000..3a0cba9117 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/session_incarnation_migration_test.go @@ -0,0 +1,87 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 +func TestTaskSessionQueueIncarnationIsGeneratedProjectedAndRenewed(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-queue-incarnation" + sessionID = "session-reused" + ) + + if err := repo.CreateTask(ctx, &models.Task{ID: taskID, Title: "Queue incarnation"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: taskID}); err != nil { + t.Fatalf("CreateTaskSession first: %v", err) + } + + var first string + if err := repo.db.GetContext(ctx, &first, `SELECT queue_incarnation_id FROM task_sessions WHERE id = ?`, sessionID); err != nil { + t.Fatalf("read first queue incarnation: %v", err) + } + if first == "" { + t.Fatal("first queue incarnation is empty") + } + stored, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("GetTaskSession first: %v", err) + } + if got := stored.ToAPI()["queue_incarnation_id"]; got != first { + t.Fatalf("projected queue incarnation = %v, want %q", got, first) + } + + if err := repo.DeleteTaskSession(ctx, stored); err != nil { + t.Fatalf("DeleteTaskSession: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: taskID}); err != nil { + t.Fatalf("CreateTaskSession replacement: %v", err) + } + var replacement string + if err := repo.db.GetContext(ctx, &replacement, `SELECT queue_incarnation_id FROM task_sessions WHERE id = ?`, sessionID); err != nil { + t.Fatalf("read replacement queue incarnation: %v", err) + } + if replacement == "" || replacement == first { + t.Fatalf("replacement queue incarnation = %q, want non-empty value different from %q", replacement, first) + } +} + +// @covers AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 +func TestTaskSessionQueueIncarnationMigrationBackfillsDistinctValuesOnReplay(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "workspace-incarnation-backfill", Name: "Backfill"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-incarnation-backfill", WorkspaceID: "workspace-incarnation-backfill", Title: "Backfill", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + for _, sessionID := range []string{"session-backfill-a", "session-backfill-b"} { + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: "task-incarnation-backfill"}); err != nil { + t.Fatalf("CreateTaskSession %s: %v", sessionID, err) + } + } + if _, err := repo.db.ExecContext(ctx, `UPDATE task_sessions SET queue_incarnation_id = ''`); err != nil { + t.Fatalf("clear queue incarnations: %v", err) + } + + if _, err := NewWithDB(repo.db, repo.ro, nil); err != nil { + t.Fatalf("replay repository migrations: %v", err) + } + var incarnations []string + if err := repo.db.SelectContext(ctx, &incarnations, `SELECT queue_incarnation_id FROM task_sessions ORDER BY id`); err != nil { + t.Fatalf("list queue incarnations: %v", err) + } + if len(incarnations) != 2 || incarnations[0] == "" || incarnations[1] == "" || incarnations[0] == incarnations[1] { + t.Fatalf("backfilled queue incarnations = %#v, want two distinct non-empty values", incarnations) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/session_test.go b/apps/backend/internal/task/repository/sqlite/session_test.go index 6922c648ec..111ee3ffa1 100644 --- a/apps/backend/internal/task/repository/sqlite/session_test.go +++ b/apps/backend/internal/task/repository/sqlite/session_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/jmoiron/sqlx" - "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/db/dialect" "github.com/kandev/kandev/internal/task/models" @@ -232,7 +231,9 @@ func TestCreateTaskSessionWithInitialRuntimeSeedConsumesOnceAcrossConcurrentAndR t.Fatalf("initial runtime seed profile remained in task metadata: %#v", task.Metadata) } - require.NoError(t, repo.DeleteTaskSession(ctx, initialSessionID)) + initialSession, err := repo.GetTaskSession(ctx, initialSessionID) + require.NoError(t, err) + require.NoError(t, repo.DeleteTaskSession(ctx, initialSession)) replacement := &models.TaskSession{ ID: "initial-runtime-session-replacement", TaskID: taskID, @@ -684,6 +685,54 @@ func TestClaimPromptableTaskSessionIfActive(t *testing.T) { }) } +func TestPromptableSessionClaimAndRestoreRequireExactIdentity(t *testing.T) { + ctx := context.Background() + repo := newRepoForSessionTests(t) + seedForMsgTest(t, repo, "task-identity", "session-identity", "turn-identity") + _, err := repo.db.ExecContext( + ctx, + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`, + "current-incarnation", + "session-identity", + ) + require.NoError(t, err) + require.NoError(t, repo.UpdateTaskSessionState( + ctx, "session-identity", models.TaskSessionStateWaitingForInput, "", + )) + session, err := repo.GetTaskSession(ctx, "session-identity") + require.NoError(t, err) + require.NotEmpty(t, session.QueueIncarnationID) + + stale, err := repo.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, session.TaskID, session.ID, "replaced-incarnation", + ) + require.NoError(t, err) + require.Equal(t, models.PromptableTaskSessionInactive, stale.Status) + persisted, err := repo.GetTaskSession(ctx, session.ID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateWaitingForInput, persisted.State) + + claim, err := repo.ClaimPromptableTaskSessionIfActiveForIdentity( + ctx, session.TaskID, session.ID, session.QueueIncarnationID, + ) + require.NoError(t, err) + require.Equal(t, models.PromptableTaskSessionClaimed, claim.Status) + restored, _, err := repo.UpdateTaskSessionStateIfCurrentIdentity( + ctx, + session.TaskID, + session.ID, + "replaced-incarnation", + models.TaskSessionStateRunning, + models.TaskSessionStateWaitingForInput, + "", + ) + require.NoError(t, err) + require.False(t, restored) + persisted, err = repo.GetTaskSession(ctx, session.ID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, persisted.State) +} + func TestClaimPromptableTaskSessionIfActive_ZeroRowAfterConcurrentBusyTransition(t *testing.T) { ctx := context.Background() repo := newRepoForSessionTests(t) diff --git a/apps/backend/internal/task/repository/sqlite/subagent_context_test.go b/apps/backend/internal/task/repository/sqlite/subagent_context_test.go index 609490b6ba..d188039b3c 100644 --- a/apps/backend/internal/task/repository/sqlite/subagent_context_test.go +++ b/apps/backend/internal/task/repository/sqlite/subagent_context_test.go @@ -468,7 +468,7 @@ func TestUpsertSubagentContextCascadeDeletesWithSession(t *testing.T) { }); err != nil { t.Fatalf("upsert: %v", err) } - if err := repo.DeleteTaskSession(ctx, "session-12"); err != nil { + if err := deleteTaskSessionForTest(t, repo, ctx, "session-12"); err != nil { t.Fatalf("DeleteTaskSession: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index f2df8ea415..d6ad3078d9 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -704,7 +704,7 @@ func (r *Repository) UpdateTaskWithWorkflowStepAdmission( targetStepID string, limit int, ) (bool, error) { - admitted, _, err := r.updateTaskWithWorkflowStepAdmission(ctx, task, targetStepID, limit, nil, false, "", "") + admitted, _, err := r.updateTaskWithWorkflowStepAdmission(ctx, task, targetStepID, limit, nil, false, "", "", nil) return admitted, err } @@ -728,7 +728,7 @@ func (r *Repository) UpdateTaskWithWorkflowStepAdmissionAndState( expectedWorkflowID string, ) (bool, error) { admitted, _, err := r.updateTaskWithWorkflowStepAdmission( - ctx, task, targetStepID, limit, admittedState, queueExitPending, "", expectedWorkflowID, + ctx, task, targetStepID, limit, admittedState, queueExitPending, "", expectedWorkflowID, nil, ) return admitted, err } @@ -749,10 +749,141 @@ func (r *Repository) UpdateTaskWithWorkflowStepAdmissionIfAtStep( targetStepID string, limit int, ) (applied bool, err error) { - _, applied, err = r.updateTaskWithWorkflowStepAdmission(ctx, task, targetStepID, limit, nil, false, expectedStepID, "") + _, applied, err = r.updateTaskWithWorkflowStepAdmission(ctx, task, targetStepID, limit, nil, false, expectedStepID, "", nil) return applied, err } +func (r *Repository) UpdateTaskWithWorkflowStepAdmissionForDeferredMove( + ctx context.Context, + task *models.Task, + expectedStepID, targetStepID string, + limit int, + record messagequeue.PendingMoveRecord, +) (admitted, applied bool, err error) { + return r.updateTaskWithWorkflowStepAdmission( + ctx, task, targetStepID, limit, nil, false, expectedStepID, "", &record, + ) +} + +func (r *Repository) MarkDeferredMoveAppliedForSession( + ctx context.Context, + taskID, moveID string, + record messagequeue.PendingMoveRecord, +) (bool, error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + if err := r.validateDeferredMoveGuardTx(ctx, tx, record); err != nil { + return false, err + } + if _, _, found, err := r.readTaskStepInTx(ctx, tx, taskID); err != nil { + return false, err + } else if !found { + return false, sql.ErrNoRows + } + task, err := r.scanSingleTask(tx.QueryRowContext(ctx, r.db.Rebind( + `SELECT `+taskSelectColumns("t")+` FROM tasks t WHERE t.id = ?`), taskID)) + if err != nil { + return false, err + } + if task.Metadata == nil { + task.Metadata = map[string]interface{}{} + } + applied, _ := task.Metadata[models.MetaKeyAppliedDeferredMoves].(map[string]interface{}) + if _, exists := applied[moveID]; exists { + return false, nil + } + if applied == nil { + applied = map[string]interface{}{} + } + applied[moveID] = true + task.Metadata[models.MetaKeyAppliedDeferredMoves] = applied + task.UpdatedAt = time.Now().UTC() + metadata, err := json.Marshal(task.Metadata) + if err != nil { + return false, err + } + if _, err := r.updateTaskTx(ctx, tx, task, metadata, ""); err != nil { + return false, err + } + if err := r.deleteDeferredMoveGuardTx(ctx, tx, record); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + return true, nil +} + +func (r *Repository) validateDeferredMoveGuardTx( + ctx context.Context, + tx *sql.Tx, + record messagequeue.PendingMoveRecord, +) error { + move := record.Move + if record.SessionID == "" || move.SessionIncarnationID == "" || move.TaskID == "" { + return messagequeue.ErrSessionIdentityMismatch + } + if _, err := tx.ExecContext(ctx, r.db.Rebind(` + INSERT INTO queue_session_locks (session_id) VALUES (?) + ON CONFLICT(session_id) DO UPDATE SET session_id = excluded.session_id + `), record.SessionID); err != nil { + return fmt.Errorf("lock deferred move session: %w", err) + } + var matched int + err := tx.QueryRowContext(ctx, r.db.Rebind(` + SELECT COUNT(*) + FROM task_sessions s + JOIN pending_moves p ON p.session_id = s.id + WHERE s.id = ? AND s.task_id = ? AND s.queue_incarnation_id = ? + AND p.move_id = ? AND p.session_incarnation_id = ? + AND p.task_id = ? AND p.workflow_id = ? AND p.workflow_step_id = ? + AND p.step_position = ? AND p.queued_at = ? + AND p.actor = ? AND p.sender_session_id = ? + `), record.SessionID, move.TaskID, move.SessionIncarnationID, + move.MoveID, move.SessionIncarnationID, move.TaskID, move.WorkflowID, + move.WorkflowStepID, move.Position, move.QueuedAt, move.Actor, move.SenderSessionID, + ).Scan(&matched) + if err != nil { + return fmt.Errorf("validate deferred move identity: %w", err) + } + if matched != 1 { + return messagequeue.ErrSessionIdentityMismatch + } + return nil +} + +func (r *Repository) deleteDeferredMoveGuardTx( + ctx context.Context, + tx *sql.Tx, + record messagequeue.PendingMoveRecord, +) error { + move := record.Move + result, err := tx.ExecContext(ctx, r.db.Rebind(` + DELETE FROM pending_moves + WHERE session_id = ? AND move_id = ? AND session_incarnation_id = ? + AND task_id = ? AND workflow_id = ? AND workflow_step_id = ? + AND step_position = ? AND queued_at = ? + AND actor = ? AND sender_session_id = ? + `), record.SessionID, move.MoveID, move.SessionIncarnationID, move.TaskID, + move.WorkflowID, move.WorkflowStepID, move.Position, move.QueuedAt, + move.Actor, move.SenderSessionID, + ) + if err != nil { + return fmt.Errorf("consume deferred move: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected != 1 { + return messagequeue.ErrSessionIdentityMismatch + } + return nil +} + // rebaseTaskForStepAdmissionCAS applies the AC-46/48 compare-and-swap // precondition for the CAS-guarded admission path: it reads the task's // current step inside the transaction, after the workspace lock, via @@ -848,6 +979,7 @@ func (r *Repository) updateTaskWithWorkflowStepAdmission( queueExitPending bool, expectedStepID string, expectedWorkflowID string, + deferredMove *messagequeue.PendingMoveRecord, ) (admitted bool, applied bool, err error) { now := time.Now().UTC() task.UpdatedAt = now @@ -879,6 +1011,11 @@ func (r *Repository) updateTaskWithWorkflowStepAdmission( if err := r.lockWorkspaceRowStdTx(ctx, tx, workspaceID); err != nil { return false, false, err } + if deferredMove != nil { + if err := r.validateDeferredMoveGuardTx(ctx, tx, *deferredMove); err != nil { + return false, false, err + } + } // AC-46/48 compare-and-swap precondition, only for CAS callers (see // rebaseTaskForStepAdmissionCAS for the full rationale). A mismatch means @@ -933,6 +1070,26 @@ func (r *Repository) updateTaskWithWorkflowStepAdmission( if err != nil { return false, false, err } + if deferredMove != nil { + result, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE task_sessions SET review_status = '' + WHERE id = ? AND task_id = ? AND queue_incarnation_id = ? + `), deferredMove.SessionID, deferredMove.Move.TaskID, deferredMove.Move.SessionIncarnationID) + if err != nil { + return false, false, err + } + if affected, rowsErr := result.RowsAffected(); rowsErr != nil || affected != 1 { + if rowsErr != nil { + return false, false, rowsErr + } + return false, false, messagequeue.ErrSessionIdentityMismatch + } + } + if deferredMove != nil { + if err := r.deleteDeferredMoveGuardTx(ctx, tx, *deferredMove); err != nil { + return false, false, err + } + } if err := tx.Commit(); err != nil { return false, false, err } @@ -1910,7 +2067,10 @@ func (r *Repository) DeleteTaskWithVacatedStep(ctx context.Context, id string) ( if rows == 0 { return "", fmt.Errorf("%w: %s", ErrTaskNotFound, id) } - if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions); err != nil { + if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions, true); err != nil { + return "", err + } + if err := r.purgeQueueSessionPoliciesInTx(ctx, tx, sessions); err != nil { return "", err } if err := tx.Commit(); err != nil { @@ -2640,7 +2800,7 @@ func (r *Repository) ArchiveTask(ctx context.Context, id string) error { if err != nil { return err } - if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions); err != nil { + if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions, false); err != nil { return err } if err := tx.Commit(); err != nil { @@ -2701,7 +2861,7 @@ func (r *Repository) ArchiveTaskIfActiveWithVacatedStep( if err != nil { return "", false, err } - if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions); err != nil { + if err := r.purgeTaskQueueInTx(ctx, tx, id, sessions, false); err != nil { return "", false, err } if err := tx.Commit(); err != nil { @@ -2740,12 +2900,119 @@ func (r *Repository) taskQueueSessionsInTx(ctx context.Context, tx *sqlx.Tx, tas return sessions, nil } -func (r *Repository) purgeTaskQueueInTx(ctx context.Context, tx *sqlx.Tx, taskID string, sessions []string) error { - _, err := messagequeue.PurgeTaskInTransaction(ctx, tx, r.db, taskID, sessions) - if internaldb.IsMissingTableError(err) { +func (r *Repository) purgeTaskQueueInTx( + ctx context.Context, + tx *sqlx.Tx, + taskID string, + sessions []string, + deleteTask bool, +) error { + r.notifyTaskQueuePurging(ctx, taskID) + if dialect.IsPostgres(r.db.DriverName()) { + return r.purgeTaskQueuePostgres(ctx, tx, taskID, sessions, deleteTask) + } + queuedIDs, err := r.archiveQueuedIDsOrNil(ctx, tx, taskID, deleteTask) + if err != nil { + if internaldb.IsMissingTableError(err) { + return nil + } + return err + } + if _, err := messagequeue.PurgeTaskInTransaction(ctx, tx, r.db, taskID, sessions); err != nil { + if internaldb.IsMissingTableError(err) { + return nil + } + return err + } + if deleteTask { return nil } - return err + return r.releaseUnreferencedTaskAttachmentClaimsTx(ctx, tx, taskID, queuedIDs) +} + +// purgeTaskQueuePostgres purges under a savepoint so a missing queue schema +// does not abort the enclosing archive/delete transaction. The attachment +// release runs after the savepoint is released, with no cursor open across +// statements on the single-connection transaction. +func (r *Repository) purgeTaskQueuePostgres( + ctx context.Context, + tx *sqlx.Tx, + taskID string, + sessions []string, + deleteTask bool, +) error { + if _, err := tx.ExecContext(ctx, "SAVEPOINT purge_task_queue"); err != nil { + return err + } + queuedIDs, err := r.archiveQueuedIDsOrNil(ctx, tx, taskID, deleteTask) + if err != nil { + return r.abortPurgeSavepoint(ctx, tx, err) + } + if _, err := messagequeue.PurgeTaskInTransaction(ctx, tx, r.db, taskID, sessions); err != nil { + return r.abortPurgeSavepoint(ctx, tx, err) + } + if _, err := tx.ExecContext(ctx, "RELEASE SAVEPOINT purge_task_queue"); err != nil { + return err + } + if deleteTask { + return nil + } + return r.releaseUnreferencedTaskAttachmentClaimsTx(ctx, tx, taskID, queuedIDs) +} + +// archiveQueuedIDsOrNil collects the pre-purge queued attachment set for the +// archive release. The delete path needs no ownership set. A missing queue +// schema means bare task storage with no queue-owned claims. +func (r *Repository) archiveQueuedIDsOrNil( + ctx context.Context, + tx *sqlx.Tx, + taskID string, + deleteTask bool, +) (map[string]struct{}, error) { + if deleteTask { + return nil, nil + } + queuedIDs, err := r.queuedAttachmentIDsForTaskTx(ctx, tx, taskID) + if err != nil { + return nil, err + } + return queuedIDs, nil +} + +// abortPurgeSavepoint rolls back the purge savepoint. A missing queue schema +// is not an archive failure: bare task repositories never had queue rows, so +// there is nothing queue-owned to release. +func (r *Repository) abortPurgeSavepoint(ctx context.Context, tx *sqlx.Tx, cause error) error { + if _, rbErr := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT purge_task_queue"); rbErr != nil { + return cause + } + if _, relErr := tx.ExecContext(ctx, "RELEASE SAVEPOINT purge_task_queue"); relErr != nil { + return cause + } + if internaldb.IsMissingTableError(cause) { + return nil + } + return cause +} + +func (r *Repository) purgeQueueSessionPoliciesInTx( + ctx context.Context, + tx *sqlx.Tx, + sessionIDs []string, +) error { + for _, sessionID := range sessionIDs { + if _, err := tx.ExecContext( + ctx, + r.db.Rebind(`DELETE FROM queue_session_state WHERE session_id = ?`), + sessionID, + ); err != nil { + if internaldb.IsMissingTableError(err) { + return nil + } + return fmt.Errorf("purge queue session policy for %s: %w", sessionID, err) + } + } + return nil } // UnarchiveTaskByCascade clears archived_at + archived_by_cascade_id only diff --git a/apps/backend/internal/task/repository/sqlite/task_deferred_move_test.go b/apps/backend/internal/task/repository/sqlite/task_deferred_move_test.go new file mode 100644 index 0000000000..3c7766404a --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/task_deferred_move_test.go @@ -0,0 +1,79 @@ +package sqlite + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + "github.com/kandev/kandev/internal/task/models" +) + +func TestUpdateTaskWithWorkflowStepAdmissionForDeferredMoveRejectsReplacementSession(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedWorkspace(t, repo, "workspace-deferred") + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "workflow-deferred", WorkspaceID: "workspace-deferred", Name: "Workflow"}); err != nil { + t.Fatal(err) + } + seedCASWorkflowStep(t, repo, "workflow-deferred", "step-source", 0) + seedCASWorkflowStep(t, repo, "workflow-deferred", "step-target", 1) + task := &models.Task{ + ID: "task-deferred", WorkspaceID: "workspace-deferred", WorkflowID: "workflow-deferred", + WorkflowStepID: "step-source", Title: "Deferred candidate", WIPAdmitted: true, + } + if err := repo.CreateTask(ctx, task); err != nil { + t.Fatal(err) + } + session := &models.TaskSession{ + ID: "session-deferred", TaskID: task.ID, QueueIncarnationID: "incarnation-original", + State: models.TaskSessionStateWaitingForInput, StartedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC(), + } + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatal(err) + } + queueRepo, err := messagequeue.NewSQLiteRepository(repo.db, repo.ro) + if err != nil { + t.Fatal(err) + } + move := messagequeue.PendingMove{ + MoveID: "move-deferred", SessionIncarnationID: session.QueueIncarnationID, + TaskID: task.ID, WorkflowID: task.WorkflowID, WorkflowStepID: "step-target", + QueuedAt: time.Now().UTC(), + } + if err := queueRepo.SetPendingMove(ctx, session.ID, &move); err != nil { + t.Fatal(err) + } + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_sessions SET queue_incarnation_id = ? WHERE id = ?`), + "incarnation-replacement", session.ID, + ); err != nil { + t.Fatal(err) + } + + _, applied, err := repo.UpdateTaskWithWorkflowStepAdmissionForDeferredMove( + ctx, task, "step-source", "step-target", 0, + messagequeue.PendingMoveRecord{SessionID: session.ID, Move: move}, + ) + if !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + t.Fatalf("expected session identity mismatch, got %v", err) + } + if applied { + t.Fatal("replacement session must not apply the deferred move") + } + stored, err := repo.GetTask(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + if stored.WorkflowStepID != "step-source" { + t.Fatalf("task moved to %q despite replacement session", stored.WorkflowStepID) + } + pending, err := queueRepo.GetPendingMove(ctx, session.ID) + if err != nil { + t.Fatal(err) + } + if pending == nil || pending.MoveID != move.MoveID { + t.Fatalf("pending move was not preserved: %+v", pending) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/task_purge_race_postgres_test.go b/apps/backend/internal/task/repository/sqlite/task_purge_race_postgres_test.go index 71d7067b45..ea8cd15c43 100644 --- a/apps/backend/internal/task/repository/sqlite/task_purge_race_postgres_test.go +++ b/apps/backend/internal/task/repository/sqlite/task_purge_race_postgres_test.go @@ -540,3 +540,80 @@ func TestPostgresRepository_WorkspaceCascade_SerializesWithTaskCreation(t *testi t.Fatalf("workspace survived cascade: count=%d", count) } } + +// TestPostgresClaimedAttachmentReleaseWaitsForQueueAdmission proves cleanup +// acquires the queue repository's cross-process session lock and rechecks +// references after the winning admission commits. +func TestPostgresClaimedAttachmentReleaseWaitsForQueueAdmission(t *testing.T) { + repoA, repoB, dbC := newTaskPostgresRepoPair(t) + ctx := context.Background() + const ( + taskID = "task-attachment-release-race" + workspaceID = "ws-attachment-release-race" + sessionID = "session-attachment-release-race" + attachmentID = "attachment-release-race" + ownerID = "owner-attachment-release-race" + ) + seedTaskWithSession(t, repoA, taskID, workspaceID, sessionID) + if err := repoA.CreateMessageAttachment(ctx, &models.TaskMessageAttachment{ + ID: attachmentID, OwnerID: ownerID, WorkspaceID: workspaceID, + TaskID: taskID, SessionID: sessionID, Name: "notes.txt", MimeType: "text/plain", + Kind: "resource", DeliveryMode: "path", SizeBytes: 5, StorageKey: attachmentID, + State: models.AttachmentStateClaimed, ExpiresAt: time.Now().UTC().Add(time.Hour), + }); err != nil { + t.Fatalf("create claimed attachment: %v", err) + } + if _, err := repoA.db.ExecContext(ctx, ` + INSERT INTO queue_session_locks (session_id) VALUES ($1) + ON CONFLICT(session_id) DO NOTHING + `, sessionID); err != nil { + t.Fatalf("seed queue session lock: %v", err) + } + admissionTx, err := repoA.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin admission transaction: %v", err) + } + defer func() { _ = admissionTx.Rollback() }() + if _, err := admissionTx.ExecContext(ctx, ` + SELECT session_id FROM queue_session_locks WHERE session_id = $1 FOR UPDATE + `, sessionID); err != nil { + t.Fatalf("lock queue admission: %v", err) + } + if _, err := admissionTx.ExecContext(ctx, ` + INSERT INTO queued_messages ( + id, session_id, task_id, position, content, attachments_json, queued_at, queued_by + ) VALUES ($1, $2, $3, 1, 'new reference', $4, now(), 'user') + `, "queue-attachment-release-race", sessionID, taskID, + `[{"attachment_id":"attachment-release-race","name":"notes.txt","delivery_mode":"path"}]`); err != nil { + t.Fatalf("insert uncommitted queue admission: %v", err) + } + + releasePID := pgBackendPID(t, repoB.db) + releaseDone := make(chan struct { + released []*models.TaskMessageAttachment + err error + }, 1) + go func() { + released, releaseErr := repoB.DeleteClaimedMessageAttachments( + ctx, []string{attachmentID}, ownerID, taskID, sessionID, + ) + releaseDone <- struct { + released []*models.TaskMessageAttachment + err error + }{released: released, err: releaseErr} + }() + waitForWaitingLocks(t, dbC, releasePID, 1, "attachment cleanup on queue admission") + if err := admissionTx.Commit(); err != nil { + t.Fatalf("commit queue admission: %v", err) + } + result := <-releaseDone + if result.err != nil { + t.Fatalf("release claimed attachment: %v", result.err) + } + if len(result.released) != 0 { + t.Fatalf("release removed newly referenced attachment: %+v", result.released) + } + if _, err := repoA.GetMessageAttachment(ctx, attachmentID); err != nil { + t.Fatalf("newly referenced attachment was deleted: %v", err) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/task_queue_purge_notify_test.go b/apps/backend/internal/task/repository/sqlite/task_queue_purge_notify_test.go index 926af49d1e..2a1de7e7e4 100644 --- a/apps/backend/internal/task/repository/sqlite/task_queue_purge_notify_test.go +++ b/apps/backend/internal/task/repository/sqlite/task_queue_purge_notify_test.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "errors" "sync/atomic" "testing" "time" @@ -28,6 +29,22 @@ func TestArchiveTaskNotifiesQueuePurgeAfterCommit(t *testing.T) { t.Fatalf("logger: %v", err) } queue := messagequeue.NewService(mqRepo, messagequeue.DefaultMaxPerSession, log) + identity := queueIdentityForSession(t, repo, "session-1") + if _, err := mqRepo.SetAutoMergeOverride(ctx, identity, false); err != nil { + t.Fatalf("SetAutoMergeOverride: %v", err) + } + if err := queue.SetPendingMove(ctx, identity.SessionID, &messagequeue.PendingMove{ + MoveID: "archive-pending-move", + SessionIncarnationID: identity.SessionIncarnationID, + TaskID: identity.TaskID, + }); err != nil { + t.Fatalf("SetPendingMove: %v", err) + } + staleIdentity := identity + staleIdentity.SessionIncarnationID = "stale-incarnation" + if _, err := mqRepo.SetAutoMergeOverride(ctx, staleIdentity, true); !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + t.Fatalf("stale SetAutoMergeOverride error = %v, want identity mismatch", err) + } if _, err := queue.QueueMessage(ctx, "session-1", "task-queue-purge-notify", "follow up", "", "user", false, nil); err != nil { t.Fatalf("QueueMessage: %v", err) } @@ -55,6 +72,12 @@ func TestArchiveTaskNotifiesQueuePurgeAfterCommit(t *testing.T) { if got, err := queue.CountPendingByTask(ctx, "task-queue-purge-notify"); err != nil || got != 0 { t.Fatalf("pending after archive = %d err=%v, want 0", got, err) } + if override, err := mqRepo.GetAutoMergeOverride(ctx, identity); err != nil || override == nil || override.Enabled { + t.Fatalf("archive override = %+v err=%v, want preserved OFF", override, err) + } + if move, ok := queue.GetPendingMove(ctx, identity.SessionID); ok || move != nil { + t.Fatalf("archive pending move = %+v exists=%t, want absent", move, ok) + } task, err := repo.GetTask(ctx, "task-queue-purge-notify") if err != nil { @@ -79,6 +102,10 @@ func TestDeleteTaskNotifiesQueuePurgeAfterCommit(t *testing.T) { t.Fatalf("logger: %v", err) } queue := messagequeue.NewService(mqRepo, messagequeue.DefaultMaxPerSession, log) + identity := queueIdentityForSession(t, repo, "session-1") + if _, err := mqRepo.SetAutoMergeOverride(ctx, identity, false); err != nil { + t.Fatalf("SetAutoMergeOverride: %v", err) + } if _, err := queue.QueueMessage(ctx, "session-1", "task-delete-purge-notify", "follow up", "", "user", false, nil); err != nil { t.Fatalf("QueueMessage: %v", err) } @@ -99,6 +126,10 @@ func TestDeleteTaskNotifiesQueuePurgeAfterCommit(t *testing.T) { if _, err := repo.GetTask(ctx, "task-delete-purge-notify"); err == nil { t.Fatal("expected task gone after DeleteTask") } + override, err := mqRepo.GetAutoMergeOverride(ctx, identity) + if override != nil || !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) { + t.Fatalf("deleted task override = %+v err=%v, want fail-closed identity mismatch", override, err) + } } func TestDeleteTaskSessionPurgesQueuedMessages(t *testing.T) { @@ -119,6 +150,17 @@ func TestDeleteTaskSessionPurgesQueuedMessages(t *testing.T) { t.Fatalf("logger: %v", err) } queue := messagequeue.NewService(mqRepo, messagequeue.DefaultMaxPerSession, log) + dropIdentity := queueIdentityForSession(t, repo, "session-drop") + if _, err := mqRepo.SetAutoMergeOverride(ctx, dropIdentity, false); err != nil { + t.Fatalf("SetAutoMergeOverride: %v", err) + } + if err := queue.SetPendingMove(ctx, dropIdentity.SessionID, &messagequeue.PendingMove{ + MoveID: "session-delete-pending-move", + SessionIncarnationID: dropIdentity.SessionIncarnationID, + TaskID: dropIdentity.TaskID, + }); err != nil { + t.Fatalf("SetPendingMove: %v", err) + } if _, err := queue.QueueMessage(ctx, "session-drop", "task-session-queue-purge", "orphan me", "", "user", false, nil); err != nil { t.Fatalf("QueueMessage drop: %v", err) } @@ -126,7 +168,7 @@ func TestDeleteTaskSessionPurgesQueuedMessages(t *testing.T) { t.Fatalf("QueueMessage keep: %v", err) } - if err := repo.DeleteTaskSession(ctx, "session-drop"); err != nil { + if err := deleteTaskSessionForTest(t, repo, ctx, "session-drop"); err != nil { t.Fatalf("DeleteTaskSession: %v", err) } if got := queue.GetStatus(ctx, "session-drop").Count; got != 0 { @@ -135,6 +177,12 @@ func TestDeleteTaskSessionPurgesQueuedMessages(t *testing.T) { if got, err := queue.CountPendingByTask(ctx, "task-session-queue-purge"); err != nil || got != 1 { t.Fatalf("pending after session delete = %d err=%v, want 1 (kept session)", got, err) } + if override, err := mqRepo.GetAutoMergeOverride(ctx, dropIdentity); !errors.Is(err, messagequeue.ErrSessionIdentityMismatch) || override != nil { + t.Fatalf("deleted session override = %+v err=%v, want identity mismatch", override, err) + } + if move, ok := queue.GetPendingMove(ctx, dropIdentity.SessionID); ok || move != nil { + t.Fatalf("deleted session pending move = %+v exists=%t, want absent", move, ok) + } } func seedLiveSessionForQueue(t *testing.T, repo *Repository, sessionID, taskID string) { @@ -147,3 +195,20 @@ func seedLiveSessionForQueue(t *testing.T, repo *Repository, sessionID, taskID s t.Fatalf("CreateTaskSession(%s): %v", sessionID, err) } } + +func queueIdentityForSession( + t *testing.T, + repo *Repository, + sessionID string, +) messagequeue.QueueSessionIdentity { + t.Helper() + session, err := repo.GetTaskSession(context.Background(), sessionID) + if err != nil { + t.Fatalf("GetTaskSession(%s): %v", sessionID, err) + } + return messagequeue.QueueSessionIdentity{ + TaskID: session.TaskID, + SessionID: session.ID, + SessionIncarnationID: session.QueueIncarnationID, + } +} diff --git a/apps/backend/internal/task/repository/sqlite/usage_events_schema_test.go b/apps/backend/internal/task/repository/sqlite/usage_events_schema_test.go index 1cf213176b..367a03142b 100644 --- a/apps/backend/internal/task/repository/sqlite/usage_events_schema_test.go +++ b/apps/backend/internal/task/repository/sqlite/usage_events_schema_test.go @@ -232,7 +232,7 @@ func TestTaskUsageEventsSchema_SessionDeleteSetsNull(t *testing.T) { createUsageEventsTestSession(t, repo, "session-setnull", "task-setnull") insertUsageEventRow(t, repo, "evt-setnull", "task-setnull", "session-setnull") - if err := repo.DeleteTaskSession(context.Background(), "session-setnull"); err != nil { + if err := deleteTaskSessionForTest(t, repo, context.Background(), "session-setnull"); err != nil { t.Fatalf("DeleteTaskSession: %v", err) } diff --git a/apps/backend/internal/task/repository/sqlite/workspace.go b/apps/backend/internal/task/repository/sqlite/workspace.go index 538732d2e9..0e28a258d6 100644 --- a/apps/backend/internal/task/repository/sqlite/workspace.go +++ b/apps/backend/internal/task/repository/sqlite/workspace.go @@ -237,6 +237,11 @@ func (r *Repository) deleteWorkspaceCascade( if err := r.purgeWorkspaceTaskQueuesInTx(ctx, tx, tasks); err != nil { return nil, nil, err } + if cleanup != nil { + if err := cleanup(ctx, tx); err != nil { + return nil, nil, fmt.Errorf("workspace secret cleanup: %w", err) + } + } rows, err := r.deleteWorkspaceCascadeRow(ctx, tx, id, expectedName) if err != nil { @@ -271,11 +276,6 @@ func (r *Repository) deleteWorkspaceCascade( `), id); err != nil { return nil, nil, err } - if cleanup != nil { - if err := cleanup(ctx, tx); err != nil { - return nil, nil, fmt.Errorf("workspace secret cleanup: %w", err) - } - } if err := tx.Commit(); err != nil { return nil, nil, err } @@ -293,9 +293,12 @@ func (r *Repository) purgeWorkspaceTaskQueuesInTx(ctx context.Context, tx *sqlx. if err != nil { return fmt.Errorf("task queue sessions for cascade task %s: %w", task.ID, err) } - if err := r.purgeTaskQueueInTx(ctx, tx, task.ID, sessions); err != nil { + if err := r.purgeTaskQueueInTx(ctx, tx, task.ID, sessions, true); err != nil { return fmt.Errorf("purge task queue for workspace cascade task %s: %w", task.ID, err) } + if err := r.purgeQueueSessionPoliciesInTx(ctx, tx, sessions); err != nil { + return fmt.Errorf("purge queue session policies for workspace cascade task %s: %w", task.ID, err) + } } return nil } diff --git a/apps/backend/internal/task/service/attachment_service.go b/apps/backend/internal/task/service/attachment_service.go index 29523b9d35..e3302c2248 100644 --- a/apps/backend/internal/task/service/attachment_service.go +++ b/apps/backend/internal/task/service/attachment_service.go @@ -11,6 +11,7 @@ import ( "time" "github.com/google/uuid" + "github.com/jmoiron/sqlx" "go.uber.org/zap" "github.com/kandev/kandev/internal/common/logger" @@ -306,6 +307,32 @@ func (s *AttachmentService) DeleteByTask(ctx context.Context, taskID string) err return nil } +type transactionalWorkspaceAttachmentRepository interface { + DeleteMessageAttachmentsByWorkspaceTx( + context.Context, + *sqlx.Tx, + string, + ) ([]*models.TaskMessageAttachment, error) +} + +func (s *AttachmentService) DeleteWorkspaceAttachmentsTx( + ctx context.Context, + tx *sqlx.Tx, + workspaceID string, +) ([]*models.TaskMessageAttachment, error) { + repo, ok := s.repo.(transactionalWorkspaceAttachmentRepository) + if !ok { + return nil, nil + } + return repo.DeleteMessageAttachmentsByWorkspaceTx(ctx, tx, workspaceID) +} + +func (s *AttachmentService) RemoveBytes(attachments []*models.TaskMessageAttachment) { + for _, attachment := range attachments { + s.removeBytes(attachment) + } +} + func (s *AttachmentService) Descriptor(attachment *models.TaskMessageAttachment) AttachmentDescriptor { return AttachmentDescriptor{ ID: attachment.ID, Name: attachment.Name, MimeType: attachment.MimeType, diff --git a/apps/backend/internal/task/service/attachment_service_lifecycle_test.go b/apps/backend/internal/task/service/attachment_service_lifecycle_test.go index 47e0e35074..1c1c617893 100644 --- a/apps/backend/internal/task/service/attachment_service_lifecycle_test.go +++ b/apps/backend/internal/task/service/attachment_service_lifecycle_test.go @@ -24,6 +24,12 @@ func newAttachmentTestService(t *testing.T) (*AttachmentService, *sqliterepo.Rep if err := repo.CreateWorkspace(context.Background(), &models.Workspace{ID: "ws-att", Name: "attachments"}); err != nil { t.Fatalf("create workspace: %v", err) } + if err := repo.CreateTask(context.Background(), &models.Task{ID: "task-1", WorkspaceID: "ws-att", Title: "task one"}); err != nil { + t.Fatalf("create task: %v", err) + } + if err := repo.CreateTask(context.Background(), &models.Task{ID: "task-2", WorkspaceID: "ws-att", Title: "task two"}); err != nil { + t.Fatalf("create task: %v", err) + } root := t.TempDir() auth := &recordingWorkspaceAuthorizer{} svc, err := NewAttachmentService(repo, root, auth.authorize, accessTestLogger(t)) diff --git a/apps/backend/internal/task/service/resource_cleanup_task_owned_test.go b/apps/backend/internal/task/service/resource_cleanup_task_owned_test.go index 761cbcd568..ee43560fd3 100644 --- a/apps/backend/internal/task/service/resource_cleanup_task_owned_test.go +++ b/apps/backend/internal/task/service/resource_cleanup_task_owned_test.go @@ -84,7 +84,7 @@ func TestDeleteTaskCleanupFindsWorktreeAfterLastSessionDeletedAndRestart(t *test worktreeID := created.ID // Delete the only session through the orchestrator-style deletion. - if err := repo.DeleteTaskSession(ctx, "session-zero-session"); err != nil { + if err := repo.DeleteTaskSession(ctx, session); err != nil { t.Fatalf("delete session: %v", err) } @@ -179,7 +179,7 @@ func TestDeleteTaskCleanupRemovesEveryWorktreeAfterLastSessionDeletedAndRestart( if err := repo.UpdateTaskSession(ctx, session); err != nil { t.Fatalf("link task session: %v", err) } - if err := repo.DeleteTaskSession(ctx, sessionID); err != nil { + if err := repo.DeleteTaskSession(ctx, session); err != nil { t.Fatalf("delete task session: %v", err) } @@ -262,7 +262,7 @@ func TestArchiveCleanupFindsWorktreeAfterLastSessionDeleted(t *testing.T) { if err := repo.UpdateTaskSession(ctx, session); err != nil { t.Fatalf("link session: %v", err) } - if err := repo.DeleteTaskSession(ctx, "session-archive-zero"); err != nil { + if err := repo.DeleteTaskSession(ctx, session); err != nil { t.Fatalf("delete session: %v", err) } diff --git a/apps/backend/internal/task/service/service_access.go b/apps/backend/internal/task/service/service_access.go index 037b4a4b71..b63ebe0790 100644 --- a/apps/backend/internal/task/service/service_access.go +++ b/apps/backend/internal/task/service/service_access.go @@ -309,9 +309,28 @@ func (s *Service) AuthorizeTaskSessionAccess(ctx context.Context, taskID, sessio return nil } +// AuthorizeTaskSessionIncarnationAccess additionally rejects a stale queue +// identity after a textual session ID has been deleted and recreated. +func (s *Service) AuthorizeTaskSessionIncarnationAccess( + ctx context.Context, + taskID, sessionID, incarnationID string, +) error { + if err := s.AuthorizeTaskAccess(ctx, taskID); err != nil { + return err + } + session, err := s.sessions.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if session == nil || session.TaskID != taskID || session.QueueIncarnationID != incarnationID { + return repoerrors.ErrTaskNotFound + } + return nil +} + // AuthorizeEnvironmentAccess checks reach of a task environment via its task's -// workspace. Used by the terminal environment-shell route, which resolves -// executions by environment ID rather than session ID. +// workspace. Used by the terminal environment-shell route, which +// resolves executions by environment ID rather than session ID. func (s *Service) AuthorizeEnvironmentAccess(ctx context.Context, taskEnvironmentID string) error { return s.AuthorizeEnvironmentScope(ctx, taskEnvironmentID, authz.ScopeWorkspaceRead) } diff --git a/apps/backend/internal/task/service/service_attachments.go b/apps/backend/internal/task/service/service_attachments.go index 80aff4affc..3a14fcb6fc 100644 --- a/apps/backend/internal/task/service/service_attachments.go +++ b/apps/backend/internal/task/service/service_attachments.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) @@ -43,8 +44,38 @@ func (s *Service) ClaimMessageAttachments(ctx context.Context, taskID, sessionID return s.attachmentSvc.Claim(ctx, identity.UserID, task.WorkspaceID, taskID, sessionID, ids) } -// ReleaseMessageAttachments removes claimed descriptors that a queue edit no -// longer references. The caller has already authorized the task session. +// PrepareQueueAttachmentClaim authenticates staged attachment ownership +// without mutating it. The queue repository applies the returned claim in the +// same transaction as queue admission. +func (s *Service) PrepareQueueAttachmentClaim(ctx context.Context, taskID string, attachments []v1.MessageAttachment) (messagequeue.QueueAttachmentClaim, error) { + claim := messagequeue.QueueAttachmentClaim{} + for _, attachment := range attachments { + if attachment.AttachmentID != "" { + claim.IDs = append(claim.IDs, attachment.AttachmentID) + } + } + if len(claim.IDs) == 0 { + return claim, nil + } + if s.attachmentSvc == nil { + return messagequeue.QueueAttachmentClaim{}, errors.New("file-backed attachments are unavailable") + } + task, err := s.GetTask(ctx, taskID) + if err != nil { + return messagequeue.QueueAttachmentClaim{}, err + } + identity, ok := authn.IdentityFromContext(ctx) + if !ok || identity.UserID == "" { + return messagequeue.QueueAttachmentClaim{}, models.ErrAttachmentForbidden + } + claim.OwnerID = identity.UserID + claim.WorkspaceID = task.WorkspaceID + return claim, nil +} + +// ReleaseMessageAttachments asks the attachment repository to remove candidate +// claims. The repository locks the session and rechecks durable queue and +// transcript references before deleting any descriptor. func (s *Service) ReleaseMessageAttachments(ctx context.Context, taskID, sessionID string, attachments []v1.MessageAttachment) error { if len(attachments) == 0 || s.attachmentSvc == nil { return nil diff --git a/apps/backend/internal/task/service/service_attachments_test.go b/apps/backend/internal/task/service/service_attachments_test.go new file mode 100644 index 0000000000..3cf81fb703 --- /dev/null +++ b/apps/backend/internal/task/service/service_attachments_test.go @@ -0,0 +1,63 @@ +package service + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +func TestReleaseMessageAttachmentsKeepsTranscriptReferences(t *testing.T) { + taskService, _, repo := createTestService(t) + ctx := authn.WithIdentity(context.Background(), authn.Identity{UserID: "owner"}) + seedTask(t, ctx, repo, "task-attachments") + seedSession(t, ctx, repo, "task-attachments", "session-attachments") + seedTurn(t, repo, "turn-attachments", "session-attachments", "task-attachments") + + attachmentService, err := NewAttachmentService(repo, t.TempDir(), nil, logger.Default()) + if err != nil { + t.Fatalf("create attachment service: %v", err) + } + taskService.SetAttachmentService(attachmentService) + + stage := func(name string) *models.TaskMessageAttachment { + t.Helper() + attachment, stageErr := attachmentService.Stage( + ctx, "owner", "ws-plan", name, "text/plain", "resource", "path", strings.NewReader(name), + ) + if stageErr != nil { + t.Fatalf("stage %s: %v", name, stageErr) + } + return attachment + } + referenced := stage("referenced.txt") + orphaned := stage("orphaned.txt") + if err := attachmentService.Claim( + ctx, "owner", "ws-plan", "task-attachments", "session-attachments", []string{referenced.ID, orphaned.ID}, + ); err != nil { + t.Fatalf("claim attachments: %v", err) + } + if err := repo.CreateMessage(ctx, &models.Message{ + ID: "message-attachments", TaskSessionID: "session-attachments", TaskID: "task-attachments", + TurnID: "turn-attachments", AuthorType: models.MessageAuthorUser, Content: "kept", + Metadata: map[string]interface{}{"attachments": []v1.MessageAttachment{{AttachmentID: referenced.ID}}}, + }); err != nil { + t.Fatalf("create transcript message: %v", err) + } + + candidates := []v1.MessageAttachment{{AttachmentID: referenced.ID}, {AttachmentID: orphaned.ID}} + if err := taskService.ReleaseMessageAttachments(ctx, "task-attachments", "session-attachments", candidates); err != nil { + t.Fatalf("release attachments: %v", err) + } + if _, err := repo.GetMessageAttachment(ctx, referenced.ID); err != nil { + t.Fatalf("transcript attachment was released: %v", err) + } + if _, err := repo.GetMessageAttachment(ctx, orphaned.ID); !errors.Is(err, models.ErrAttachmentNotFound) { + t.Fatalf("orphaned attachment error = %v, want not found", err) + } +} diff --git a/apps/backend/internal/task/service/service_resources.go b/apps/backend/internal/task/service/service_resources.go index 3790593cfc..e70f26b029 100644 --- a/apps/backend/internal/task/service/service_resources.go +++ b/apps/backend/internal/task/service/service_resources.go @@ -283,14 +283,31 @@ func (s *Service) deleteWorkspace(ctx context.Context, workspace *models.Workspa } } + var deletedWorkspaceAttachments []*models.TaskMessageAttachment var deletedTasks []*models.Task var deletedWorkflows []*models.Workflow transactionalCleanup, hasTransactionalCleanup := s.workspaceSecretDeleter.(secrets.WorkspaceSecretTransactionalDeleter) transactionalCascade, hasTransactionalCascade := s.workspaces.(transactionalWorkspaceCascade) + useTransactionalCleanup := hasTransactionalCascade && + (hasTransactionalCleanup || s.attachmentSvc != nil) switch { - case hasTransactionalCleanup && hasTransactionalCascade: + case useTransactionalCleanup: cleanup := func(cleanupCtx context.Context, tx *sqlx.Tx) error { - return transactionalCleanup.DeleteWorkspaceSecretsTx(cleanupCtx, tx, workspace.ID) + if hasTransactionalCleanup { + if err := transactionalCleanup.DeleteWorkspaceSecretsTx(cleanupCtx, tx, workspace.ID); err != nil { + return err + } + } + if s.attachmentSvc != nil { + var err error + deletedWorkspaceAttachments, err = s.attachmentSvc.DeleteWorkspaceAttachmentsTx( + cleanupCtx, tx, workspace.ID, + ) + if err != nil { + return err + } + } + return nil } if confirmedName == nil { deletedTasks, deletedWorkflows, err = transactionalCascade.DeleteWorkspaceCascadeWithSecretCleanup(ctx, workspace.ID, cleanup) @@ -306,6 +323,9 @@ func (s *Service) deleteWorkspace(ctx context.Context, workspace *models.Workspa s.cancelWorkspaceDeleteTaskCleanupJobs(ctx, cleanups) return s.mapWorkspaceDeleteError(workspace.ID, err) } + if s.attachmentSvc != nil { + s.attachmentSvc.RemoveBytes(deletedWorkspaceAttachments) + } if s.workspaceSecretDeleter != nil && (!hasTransactionalCleanup || !hasTransactionalCascade) { if err := s.workspaceSecretDeleter.DeleteWorkspaceSecrets(ctx, workspace.ID); err != nil { s.cancelWorkspaceDeleteTaskCleanupJobs(ctx, cleanups) diff --git a/apps/backend/internal/task/service/service_resources_test.go b/apps/backend/internal/task/service/service_resources_test.go index b43eaf9257..e81a135907 100644 --- a/apps/backend/internal/task/service/service_resources_test.go +++ b/apps/backend/internal/task/service/service_resources_test.go @@ -1002,6 +1002,52 @@ func TestService_DeleteWorkspaceDeletesWorkspaceOwnedTasksAndWorkflows(t *testin } } +func TestService_DeleteWorkspaceRemovesStagedAndClaimedAttachmentBytes(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-delete", Name: "Delete Me"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ID: "task-delete", WorkspaceID: "ws-delete", Title: "Delete attachments"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + attachmentRoot := t.TempDir() + attachmentSvc, err := NewAttachmentService(repo, attachmentRoot, nil, commonlogger.Default()) + if err != nil { + t.Fatalf("NewAttachmentService: %v", err) + } + svc.SetAttachmentService(attachmentSvc) + + stage := func(name string) *models.TaskMessageAttachment { + t.Helper() + attachment, stageErr := attachmentSvc.Stage( + ctx, "owner", "ws-delete", name, "text/plain", "resource", "path", strings.NewReader(name), + ) + if stageErr != nil { + t.Fatalf("Stage %s: %v", name, stageErr) + } + return attachment + } + staged := stage("staged.txt") + claimed := stage("claimed.txt") + if err := attachmentSvc.Claim(ctx, "owner", "ws-delete", "task-delete", "session-delete", []string{claimed.ID}); err != nil { + t.Fatalf("Claim: %v", err) + } + + if err := svc.DeleteWorkspace(ctx, "ws-delete"); err != nil { + t.Fatalf("DeleteWorkspace: %v", err) + } + + for _, attachment := range []*models.TaskMessageAttachment{staged, claimed} { + if _, err := os.Stat(filepath.Join(attachmentRoot, "attachments", attachment.StorageKey)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("attachment %s bytes still exist: %v", attachment.ID, err) + } + if _, err := repo.GetMessageAttachment(ctx, attachment.ID); !errors.Is(err, models.ErrAttachmentNotFound) { + t.Fatalf("attachment %s registry row still exists: %v", attachment.ID, err) + } + } +} + func TestService_DeleteWorkspaceRollsBackCascadeWhenSecretCleanupFails(t *testing.T) { svc, eventBus, repo := createTestService(t) ctx := context.Background() diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index 66a213a2d2..bd10d227a0 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -146,6 +146,7 @@ const ( ActionMessageQueueDrain = "message.queue.drain" // Dispatch one queued entry now when the session is promptable ActionMessageQueueSendNow = "message.queue.send_now" // Interrupt and replace the active turn with an exact queue selection ActionMessageQueueAutoRunSet = "message.queue.auto_run.set" // Persist automatic queue processing and optionally dispatch the head + ActionMessageQueueAutoMergeSet = "message.queue.auto_merge.set" // Persist a per-session automatic-merge override ActionMessageQueueRemove = "message.queue.remove" // Delete a single entry by id ActionMessageQueueMerge = "message.queue.merge" // Fold an entry into the entry above it ActionMessageQueueReorder = "message.queue.reorder" // Rewrite the visible pending order for a session diff --git a/apps/web/components/settings/system/message-queue-settings.test.tsx b/apps/web/components/settings/system/message-queue-settings.test.tsx index 60dc2457ea..38c5c088bb 100644 --- a/apps/web/components/settings/system/message-queue-settings.test.tsx +++ b/apps/web/components/settings/system/message-queue-settings.test.tsx @@ -373,6 +373,7 @@ describe("MessageQueueSettings — automatic merge toggle", () => { const toggle = await screen.findByRole("button", { name: AUTO_MERGE_TOGGLE_LABEL }); expect(toggle.getAttribute(ARIA_PRESSED)).toBe("true"); expect(screen.getByText(/stays as a separate queued message/)).toBeTruthy(); + expect(screen.getByText(/follows this value until you change Auto-merge/)).toBeTruthy(); const touchTarget = screen.getByTestId("message-queue-auto-merge-touch-target"); expect(touchTarget.className).toContain("min-h-11"); expect(touchTarget.className).toContain("min-w-11"); @@ -416,7 +417,9 @@ describe("MessageQueueSettings — automatic merge toggle", () => { auto_merge_enabled: false, }); }); +}); +describe("MessageQueueSettings — automatic merge permissions and recovery", () => { it("stays editable for admins under a capacity environment lock", async () => { fetchSettingsMock.mockResolvedValueOnce( response({ configured: 25, effective: 50, source: "environment", locked: true }), diff --git a/apps/web/components/task/chat/queued-ghost-list.test.tsx b/apps/web/components/task/chat/queued-ghost-list.test.tsx index 9c57e1a370..c29a4d68da 100644 --- a/apps/web/components/task/chat/queued-ghost-list.test.tsx +++ b/apps/web/components/task/chat/queued-ghost-list.test.tsx @@ -10,6 +10,7 @@ import { toast } from "sonner"; import { simulateReorderDrag } from "./queued-ghost-list.test-helpers"; const useQueueMock = vi.fn(); +const STATE_ATTRIBUTE = "data-state"; vi.mock("@/hooks/domains/session/use-queue", () => ({ useQueue: (sessionId: string | null) => useQueueMock(sessionId), @@ -92,10 +93,13 @@ function baseState(entries: QueuedMessage[]) { isFull: false, mergeEnabled: true, autoRun: true, + autoMerge: true, + autoMergeAvailable: true, isLoading: false, queue: vi.fn(async () => {}), clearAll: vi.fn(async () => {}), setAutoRun: vi.fn(async () => {}), + setAutoMerge: vi.fn(async () => {}), editEntry: vi.fn(async () => {}), removeEntry: vi.fn(async () => {}), mergeEntry: vi.fn(async () => {}), @@ -214,20 +218,20 @@ describe("QueueAffordance", () => { expect(state.clearAll).toHaveBeenCalledTimes(1); }); - it("shows Auto-run and removes legacy header dispatch actions", () => { + it("shows compact Auto-run and Auto-merge controls without legacy dispatch actions", () => { const state = queueState([entry()]); useQueueMock.mockReturnValue(state); render({CHILD}); fireEvent.click(screen.getByTestId(CHIP_ID)); const autoRun = screen.getByTestId(AUTO_RUN_BUTTON_ID); - expect(autoRun.getAttribute("data-state")).toBe("checked"); - const autoRunHelpId = autoRun.getAttribute("aria-describedby"); - expect(autoRunHelpId).toBeTruthy(); - expect(document.getElementById(autoRunHelpId!)).toBeTruthy(); - expect(screen.getByText("Runs queued messages one at a time.")).toBeTruthy(); + const autoMerge = screen.getByTestId("queue-auto-merge"); + expect(autoRun.getAttribute(STATE_ATTRIBUTE)).toBe("checked"); + expect(autoMerge.getAttribute(STATE_ATTRIBUTE)).toBe("checked"); + expect(autoRun.parentElement?.className).toContain("rounded-full"); + expect(autoMerge.parentElement?.className).toContain("rounded-full"); expect(autoRun.className).toContain("[@media(pointer:coarse)]:after:-inset-y-3.5"); - expect(autoRun.parentElement?.className).toContain("[@media(pointer:coarse)]:min-h-11"); + expect(autoMerge.className).toContain("[@media(pointer:coarse)]:after:-inset-y-3.5"); expect(screen.queryByTestId("queue-drain-next")).toBeNull(); expect(screen.queryByTestId("queue-send-now")).toBeNull(); }); @@ -253,11 +257,12 @@ describe("QueueAffordance Send Now", () => { it.each([ ["queue mutation", { isLoading: true }], ["cancellation", { cancellationPending: true }], - ])("disables row Send Now and Auto-run during %s", (_name, extra) => { + ])("disables row Send Now and both policy controls during %s", (_name, extra) => { useQueueMock.mockReturnValue(queueState([entry()], extra)); render({CHILD}); fireEvent.click(screen.getByTestId(CHIP_ID)); expect((screen.getByTestId(AUTO_RUN_BUTTON_ID) as HTMLButtonElement).disabled).toBe(true); + expect((screen.getByTestId("queue-auto-merge") as HTMLButtonElement).disabled).toBe(true); expect((screen.getByTestId(SEND_NOW_BUTTON_ID) as HTMLButtonElement).disabled).toBe(true); }); @@ -267,13 +272,20 @@ describe("QueueAffordance Send Now", () => { render({CHILD}); fireEvent.click(screen.getByTestId(CHIP_ID)); const autoRun = screen.getByTestId(AUTO_RUN_BUTTON_ID); - expect(autoRun.getAttribute("data-state")).toBe("unchecked"); - expect( - screen.getByText("Finishes the current response, then queued messages wait."), - ).toBeTruthy(); + expect(autoRun.getAttribute(STATE_ATTRIBUTE)).toBe("unchecked"); fireEvent.click(autoRun); expect(state.setAutoRun).toHaveBeenCalledWith(true); }); + it("changes the backend-owned Auto-merge policy", () => { + const state = queueState([entry()], { autoMerge: false }); + useQueueMock.mockReturnValue(state); + render({CHILD}); + fireEvent.click(screen.getByTestId(CHIP_ID)); + const autoMerge = screen.getByTestId("queue-auto-merge"); + expect(autoMerge.getAttribute(STATE_ATTRIBUTE)).toBe("unchecked"); + fireEvent.click(autoMerge); + expect(state.setAutoMerge).toHaveBeenCalledWith(true); + }); it("reports an Auto-run update failure", async () => { const state = queueState([entry()]); diff --git a/apps/web/components/task/chat/queued-ghost-list.tsx b/apps/web/components/task/chat/queued-ghost-list.tsx index 1c181b9c09..4480e72865 100644 --- a/apps/web/components/task/chat/queued-ghost-list.tsx +++ b/apps/web/components/task/chat/queued-ghost-list.tsx @@ -1,6 +1,6 @@ "use client"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useMemo, useRef, useState } from "react"; import type { ReactNode } from "react"; import { IconLayoutList } from "@tabler/icons-react"; import { @@ -35,6 +35,7 @@ import { useQueue } from "@/hooks/domains/session/use-queue"; import { useQueuePinned } from "@/hooks/use-queue-pinned"; import { canMergeWithAbove, QueuedGhostMessage } from "./queued-ghost-message"; import { useQueuePanelOpenState } from "./use-queue-panel-open-state"; +import { useQueuePanelEscape } from "./use-queue-panel-escape"; import { QueuePanelHeader } from "./queued-ghost-panel-header"; import type { QueuedMessage } from "@/lib/state/slices/session/types"; import type { EntityReference } from "@/lib/types/entity-reference"; @@ -54,34 +55,6 @@ function headPreviewText(entries: QueuedMessage[]): string { return clean.slice(0, HEAD_PREVIEW_MAX).trimEnd() + "…"; } -function isEditableTarget(el: EventTarget | null): boolean { - if (!(el instanceof HTMLElement)) return false; - const tag = el.tagName; - if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return true; - // contentEditable covers the TipTap chat editor and any rich-text surface. - // `el.isContentEditable` walks up the contenteditable inheritance chain. - return el.isContentEditable; -} - -function useEscToClose(open: boolean, onClose: () => void): void { - useEffect(() => { - if (!open) return; - const onKey = (e: KeyboardEvent) => { - if (e.key !== "Escape") return; - // Don't hijack Esc while the user is editing inside any input control on - // the page (queue textarea, TipTap chat editor, native input/select, or - // the clarification overlay). - if (isEditableTarget(e.target)) return; - e.preventDefault(); - e.stopPropagation(); - onClose(); - }; - // Capture Escape before an enclosing Radix dialog handles it as a dismissal. - window.addEventListener("keydown", onKey, true); - return () => window.removeEventListener("keydown", onKey, true); - }, [open, onClose]); -} - type QueueAffordanceProps = { sessionId: string | null; children: ReactNode; @@ -107,6 +80,7 @@ type QueuePanelHandlerArgs = { reorderEntries: (orderedIds: string[]) => Promise; sendEntryNow: (entryId: string) => Promise; setAutoRun: (enabled: boolean) => Promise; + setAutoMerge: (enabled: boolean) => Promise; }; function useSendNowPanelHandlers(sendEntryNow: (entryId: string) => Promise) { @@ -150,6 +124,7 @@ function useQueuePanelHandlers({ reorderEntries, sendEntryNow, setAutoRun, + setAutoMerge, }: QueuePanelHandlerArgs) { // Tracks merge requests still in flight so a rapid second click on the same // row cannot fire a second request for an entry that is already gone — the @@ -213,6 +188,15 @@ function useQueuePanelHandlers({ }, [setAutoRun, t], ); + const handleAutoMergeChange = useCallback( + (enabled: boolean) => { + setAutoMerge(enabled).catch((err) => { + console.error("Failed to update queue Auto-merge:", err); + toast.error(t("chat:failedToSetQueueAutoMerge")); + }); + }, + [setAutoMerge, t], + ); const handleReorder = useCallback( async (orderedIds: string[]) => { try { @@ -234,6 +218,7 @@ function useQueuePanelHandlers({ handleMerge, handleClear, handleAutoRunChange, + handleAutoMergeChange, handleReorder, handleSendEntryNow, }; @@ -247,6 +232,8 @@ type QueuePanelDisclosureProps = { max: number; isFull: boolean; autoRun: boolean; + autoMerge: boolean; + autoMergeAvailable: boolean; isLoading: boolean; cancellationPending: boolean; mergeEnabled: boolean; @@ -254,6 +241,7 @@ type QueuePanelDisclosureProps = { onClose: () => void; onClear: () => void; onAutoRunChange: (enabled: boolean) => void; + onAutoMergeChange: (enabled: boolean) => void; onTogglePin: () => void; onSave: (entryId: string, content: string, refs: EntityReference[]) => Promise; onRemove: (entryId: string) => Promise; @@ -271,6 +259,8 @@ function QueuePanelDisclosure({ max, isFull, autoRun, + autoMerge, + autoMergeAvailable, isLoading, cancellationPending, mergeEnabled, @@ -278,6 +268,7 @@ function QueuePanelDisclosure({ onClose, onClear, onAutoRunChange, + onAutoMergeChange, onTogglePin, onSave, onRemove, @@ -299,6 +290,8 @@ function QueuePanelDisclosure({ max={max} isFull={isFull} autoRun={autoRun} + autoMerge={autoMerge} + autoMergeAvailable={autoMergeAvailable} isLoading={isLoading} cancellationPending={cancellationPending} mergeEnabled={mergeEnabled} @@ -306,6 +299,7 @@ function QueuePanelDisclosure({ onClose={onClose} onClear={onClear} onAutoRunChange={onAutoRunChange} + onAutoMergeChange={onAutoMergeChange} onTogglePin={onTogglePin} onSave={onSave} onRemove={onRemove} @@ -334,9 +328,12 @@ export function QueueAffordance({ sessionId, children, renderStatusBar }: QueueA isFull, mergeEnabled, autoRun, + autoMerge, + autoMergeAvailable, isLoading, clearAll, setAutoRun, + setAutoMerge, editEntry, removeEntry, mergeEntry, @@ -352,6 +349,7 @@ export function QueueAffordance({ sessionId, children, renderStatusBar }: QueueA handleMerge, handleClear, handleAutoRunChange, + handleAutoMergeChange, handleReorder, handleSendEntryNow, } = useQueuePanelHandlers({ @@ -362,13 +360,14 @@ export function QueueAffordance({ sessionId, children, renderStatusBar }: QueueA reorderEntries, sendEntryNow, setAutoRun, + setAutoMerge, }); // Reset disclosure on session switch or full drain using render-phase state // adjustment (React docs: "Adjusting some state when a prop changes"). This // avoids the cascading-render anti-pattern of doing it inside useEffect. const close = useCallback(() => setIsOpen(false), []); - useEscToClose(isOpen, close); + useQueuePanelEscape(isOpen, close); const hasEntries = !!sessionId && entries.length > 0; const chipNode = @@ -402,6 +401,8 @@ export function QueueAffordance({ sessionId, children, renderStatusBar }: QueueA max={max} isFull={isFull} autoRun={autoRun} + autoMerge={autoMerge} + autoMergeAvailable={autoMergeAvailable} isLoading={isLoading} cancellationPending={cancellationPending} mergeEnabled={mergeEnabled} @@ -409,6 +410,7 @@ export function QueueAffordance({ sessionId, children, renderStatusBar }: QueueA onClose={close} onClear={handleClear} onAutoRunChange={handleAutoRunChange} + onAutoMergeChange={handleAutoMergeChange} onTogglePin={togglePin} onSave={handleSave} onRemove={handleRemove} @@ -483,6 +485,8 @@ type QueuePanelProps = { max: number; isFull: boolean; autoRun: boolean; + autoMerge: boolean; + autoMergeAvailable: boolean; isLoading: boolean; cancellationPending: boolean; mergeEnabled: boolean; @@ -490,6 +494,7 @@ type QueuePanelProps = { onClose: () => void; onClear: () => void; onAutoRunChange: (enabled: boolean) => void; + onAutoMergeChange: (enabled: boolean) => void; onTogglePin: () => void; onSave: (entryId: string, content: string, entityReferences: EntityReference[]) => Promise; onRemove: (entryId: string) => Promise; @@ -552,6 +557,8 @@ function QueuePanel({ max, isFull, autoRun, + autoMerge, + autoMergeAvailable, isLoading, cancellationPending, mergeEnabled, @@ -559,6 +566,7 @@ function QueuePanel({ onClose, onClear, onAutoRunChange, + onAutoMergeChange, onTogglePin, onSave, onRemove, @@ -594,11 +602,14 @@ function QueuePanel({ max={max} isFull={isFull} autoRun={autoRun} + autoMerge={autoMerge} + autoMergeAvailable={autoMergeAvailable} isLoading={isLoading} cancellationPending={cancellationPending} pinned={pinned} onClear={onClear} onAutoRunChange={onAutoRunChange} + onAutoMergeChange={onAutoMergeChange} onTogglePin={onTogglePin} onClose={onClose} /> diff --git a/apps/web/components/task/chat/queued-ghost-panel-header.test.tsx b/apps/web/components/task/chat/queued-ghost-panel-header.test.tsx index e25db90658..86a0ae82d7 100644 --- a/apps/web/components/task/chat/queued-ghost-panel-header.test.tsx +++ b/apps/web/components/task/chat/queued-ghost-panel-header.test.tsx @@ -1,45 +1,79 @@ -import { render, screen } from "@testing-library/react"; -import { describe, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@kandev/ui/tooltip"; import { QueuePanelHeader, type QueuePanelHeaderProps } from "./queued-ghost-panel-header"; - vi.mock("@/hooks/use-responsive-breakpoint", () => ({ useResponsiveBreakpoint: () => ({ isMobile: false }), })); +afterEach(cleanup); const props: QueuePanelHeaderProps = { count: 1, max: 10, isFull: false, autoRun: false, + autoMerge: true, + autoMergeAvailable: true, isLoading: false, cancellationPending: false, pinned: false, onClear: vi.fn(), onAutoRunChange: vi.fn(), + onAutoMergeChange: vi.fn(), onTogglePin: vi.fn(), onClose: vi.fn(), }; describe("QueuePanelHeader", () => { - it("connects each rendered Auto-run switch to unique label and help IDs", () => { + it("renders compact Auto-run and Auto-merge pills with unique accessible labels", () => { const { container } = render( - <> + - , + , ); - const switches = screen.getAllByTestId("queue-auto-run"); - const labels = Array.from(container.querySelectorAll("label")); - expect(new Set(switches.map((control) => control.id)).size).toBe(2); - expect(new Set(switches.map((control) => control.getAttribute("aria-describedby"))).size).toBe( - 2, + const autoRunSwitches = screen.getAllByTestId("queue-auto-run"); + const autoMergeSwitches = screen.getAllByTestId("queue-auto-merge"); + expect(autoRunSwitches).toHaveLength(2); + expect(autoMergeSwitches).toHaveLength(2); + for (const controls of [autoRunSwitches, autoMergeSwitches]) { + expect(new Set(controls.map((control) => control.id)).size).toBe(2); + controls.forEach((control) => { + const label = container.querySelector(`label[for="${control.id}"]`); + expect(label).not.toBeNull(); + expect(control.className).toContain("[@media(pointer:coarse)]:after:-inset-y-3.5"); + }); + } + const descriptionId = autoRunSwitches[0].getAttribute("aria-describedby"); + expect(descriptionId).not.toBeNull(); + expect(document.getElementById(descriptionId ?? "")?.textContent).toBe( + "Finishes the current response, then queued messages wait.", ); - switches.forEach((control, index) => { - expect(labels[index].htmlFor).toBe(control.id); - const helpId = control.getAttribute("aria-describedby"); - expect(helpId).toBeTruthy(); - expect(container.querySelectorAll(`[id="${helpId}"]`)).toHaveLength(1); - }); + }); + + it("disables both policies through the same conflicting-operation state", () => { + render( + + + , + ); + expect((screen.getByTestId("queue-auto-run") as HTMLButtonElement).disabled).toBe(true); + expect((screen.getByTestId("queue-auto-merge") as HTMLButtonElement).disabled).toBe(true); + }); + it("keeps the unavailable Auto-merge tooltip keyboard reachable", () => { + const { container } = render( + + + , + ); + + const autoMergeSwitch = screen.getByTestId("queue-auto-merge") as HTMLButtonElement; + expect(autoMergeSwitch.disabled).toBe(true); + const label = container.querySelector(`label[for="${autoMergeSwitch.id}"]`); + expect(label?.className).toContain("opacity-50"); + expect(label?.className).toContain("pointer-events-none"); + expect(label?.parentElement?.tagName).toBe("SPAN"); + expect(label?.parentElement?.getAttribute("tabindex")).toBe("0"); }); }); diff --git a/apps/web/components/task/chat/queued-ghost-panel-header.tsx b/apps/web/components/task/chat/queued-ghost-panel-header.tsx index 02454b5975..9e4a9fbc72 100644 --- a/apps/web/components/task/chat/queued-ghost-panel-header.tsx +++ b/apps/web/components/task/chat/queued-ghost-panel-header.tsx @@ -3,6 +3,7 @@ import { useId } from "react"; import { useTranslation } from "react-i18next"; import { Button } from "@kandev/ui"; import { Switch } from "@kandev/ui/switch"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@kandev/ui/tooltip"; import { cn } from "@/lib/utils"; import { useResponsiveBreakpoint } from "@/hooks/use-responsive-breakpoint"; @@ -11,107 +12,176 @@ export type QueuePanelHeaderProps = { max: number; isFull: boolean; autoRun: boolean; + autoMerge: boolean; + autoMergeAvailable: boolean; isLoading: boolean; cancellationPending: boolean; pinned: boolean; onClear: () => void; onAutoRunChange: (enabled: boolean) => void; + onAutoMergeChange: (enabled: boolean) => void; onTogglePin: () => void; onClose: () => void; }; +type QueueAutomationPillsProps = Pick< + QueuePanelHeaderProps, + | "autoRun" + | "autoMerge" + | "autoMergeAvailable" + | "isLoading" + | "cancellationPending" + | "onAutoRunChange" + | "onAutoMergeChange" +>; + +function QueueAutomationPills({ + autoRun, + autoMerge, + autoMergeAvailable, + isLoading, + cancellationPending, + onAutoRunChange, + onAutoMergeChange, +}: QueueAutomationPillsProps) { + const { t } = useTranslation(); + const autoRunId = useId(); + const autoRunDescriptionId = useId(); + const autoMergeId = useId(); + const controlsDisabled = isLoading || cancellationPending; + const pillClassName = + "flex min-h-7 items-center gap-1.5 rounded-full border border-border/70 bg-muted/40 px-2 text-xs font-medium text-foreground [@media(pointer:coarse)]:min-h-11 [@media(pointer:coarse)]:px-3"; + return ( +
+ + + + + + {t(autoRun ? "chat:queueAutoRunOnHelp" : "chat:queueAutoRunOffHelp")} + + + + {t(autoRun ? "chat:queueAutoRunOnHelp" : "chat:queueAutoRunOffHelp")} + + + + + + + + + {t("system:messageQueueAutoMergeNotice")} + + +
+ ); +} + export function QueuePanelHeader({ count, max, isFull, autoRun, + autoMerge, + autoMergeAvailable, isLoading, cancellationPending, pinned, onClear, onAutoRunChange, + onAutoMergeChange, onTogglePin, onClose, }: QueuePanelHeaderProps) { const { t } = useTranslation(); - const autoRunId = useId(); - const autoRunHelpId = `${autoRunId}-help`; - // The pin is a desktop-only convenience: on phone viewports the queue panel - // keeps its existing controls and the pin is not rendered. const { isMobile } = useResponsiveBreakpoint(); const capacityText = max > 0 ? t(isFull ? "chat:queueCapacityFull" : "chat:queueCapacity", { count, max }) : t("chat:queueCount", { count }); + return ( -
-
-
- - {t("chat:queued")} - - {capacityText} - -
-
- - {!isMobile && ( - - )} +
+
+ + {t("chat:queued")} + + {capacityText} + +
+ +
+ + {!isMobile && ( -
-
-
- - + )} +
); diff --git a/apps/web/components/task/chat/use-queue-panel-escape.ts b/apps/web/components/task/chat/use-queue-panel-escape.ts new file mode 100644 index 0000000000..47e4c28350 --- /dev/null +++ b/apps/web/components/task/chat/use-queue-panel-escape.ts @@ -0,0 +1,22 @@ +import { useEffect } from "react"; + +function isEditableTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false; + const tag = target.tagName; + if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") return true; + return target.isContentEditable; +} + +export function useQueuePanelEscape(open: boolean, onClose: () => void): void { + useEffect(() => { + if (!open) return; + const onKey = (event: KeyboardEvent) => { + if (event.key !== "Escape" || isEditableTarget(event.target)) return; + event.preventDefault(); + event.stopPropagation(); + onClose(); + }; + window.addEventListener("keydown", onKey, true); + return () => window.removeEventListener("keydown", onKey, true); + }, [open, onClose]); +} diff --git a/apps/web/e2e/helpers/api-client.ts b/apps/web/e2e/helpers/api-client.ts index c2b0c08e65..8041e34835 100644 --- a/apps/web/e2e/helpers/api-client.ts +++ b/apps/web/e2e/helpers/api-client.ts @@ -54,6 +54,12 @@ import type { import { loadInterimSettingsInterlockToken } from "./interim-settings-interlock"; import { dwell } from "./causal-waits"; +export type QueueSessionIdentityInput = { + taskId: string; + sessionId: string; + sessionIncarnationId: string; +}; + // --- GitHub Mock Types --- export type MockPR = { @@ -2357,6 +2363,7 @@ export class ApiClient { sessions: Array<{ id: string; task_id: string; + queue_incarnation_id: string; agent_profile_id?: string; executor_id?: string; executor_profile_id?: string; @@ -2378,6 +2385,22 @@ export class ApiClient { return this.request("GET", `/api/v1/tasks/${taskId}/sessions`); } + async getQueueSessionIdentity( + taskId: string, + sessionId: string, + ): Promise { + const { sessions } = await this.listTaskSessions(taskId); + const session = sessions.find((candidate) => candidate.id === sessionId); + if (!session?.queue_incarnation_id) { + throw new Error(`Queue identity is unavailable for session ${sessionId}`); + } + return { + taskId, + sessionId, + sessionIncarnationId: session.queue_incarnation_id, + }; + } + /** * Read a task's dependency projection. `blocked_reason` is `pending`, * `failed`, or `unknown`; the last one means the store could not be read and @@ -2720,34 +2743,46 @@ export class ApiClient { } async queueMessage( - taskId: string, - sessionId: string, + identity: QueueSessionIdentityInput, content: string, attachments?: MessageAttachmentInput[], ): Promise { await this.wsRequest("message.queue.add", { - task_id: taskId, - session_id: sessionId, + task_id: identity.taskId, + session_id: identity.sessionId, + session_incarnation_id: identity.sessionIncarnationId, content, attachments, }); } - /** Removes every pending queued message for a session (message.queue.cancel). */ - async clearQueue(sessionId: string): Promise { - await this.wsRequest("message.queue.cancel", { session_id: sessionId }); + /** Removes every pending queued message for an immutable session identity. */ + async clearQueue(identity: QueueSessionIdentityInput): Promise { + await this.wsRequest("message.queue.cancel", { + task_id: identity.taskId, + session_id: identity.sessionId, + session_incarnation_id: identity.sessionIncarnationId, + }); } - async getQueueStatus(sessionId: string): Promise<{ count: number; auto_run: boolean }> { - return this.wsRequest("message.queue.get", { session_id: sessionId }); + async getQueueStatus( + identity: QueueSessionIdentityInput, + ): Promise<{ count: number; auto_run: boolean; auto_merge_enabled: boolean }> { + return this.wsRequest("message.queue.get", { + task_id: identity.taskId, + session_id: identity.sessionId, + session_incarnation_id: identity.sessionIncarnationId, + }); } async setQueueAutoRun( - sessionId: string, + identity: QueueSessionIdentityInput, enabled: boolean, ): Promise<{ session_id: string; auto_run: boolean; dispatched: boolean }> { return this.wsRequest("message.queue.auto_run.set", { - session_id: sessionId, + task_id: identity.taskId, + session_id: identity.sessionId, + session_incarnation_id: identity.sessionIncarnationId, enabled, }); } diff --git a/apps/web/e2e/tests/chat/message-queue-scroll-helpers.ts b/apps/web/e2e/tests/chat/message-queue-scroll-helpers.ts index f4f19885a0..eb7fdf97ca 100644 --- a/apps/web/e2e/tests/chat/message-queue-scroll-helpers.ts +++ b/apps/web/e2e/tests/chat/message-queue-scroll-helpers.ts @@ -28,11 +28,11 @@ export async function seedFullQueueTask( state: "IDLE", agentProfileId: seedData.agentProfileId, }); + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, sessionId); for (let index = 0; index < FULL_QUEUE_SIZE; index++) { await apiClient.queueMessage( - task.id, - sessionId, + queueIdentity, `Queued item ${index + 1}\nSecond line keeps every queue row realistically tall.`, ); } diff --git a/apps/web/e2e/tests/chat/message-queue.spec.ts b/apps/web/e2e/tests/chat/message-queue.spec.ts index bacbf80464..54a54eb843 100644 --- a/apps/web/e2e/tests/chat/message-queue.spec.ts +++ b/apps/web/e2e/tests/chat/message-queue.spec.ts @@ -196,8 +196,9 @@ async function queueMessages( sessionId: string, messages: string[], ): Promise { + const identity = await apiClient.getQueueSessionIdentity(taskId, sessionId); for (const message of messages) { - await apiClient.queueMessage(taskId, sessionId, message); + await apiClient.queueMessage(identity, message); } } @@ -257,10 +258,11 @@ test.describe("Task session queue", () => { state: "RUNNING", agentProfileId: seedData.agentProfileId, }); + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, sessionId); await requestMessageQueueSettings(apiClient, "PATCH", { auto_merge_enabled: true }); - await apiClient.queueMessage(task.id, sessionId, "automatic first"); - await apiClient.queueMessage(task.id, sessionId, "automatic second"); + await apiClient.queueMessage(queueIdentity, "automatic first"); + await apiClient.queueMessage(queueIdentity, "automatic second"); await testPage.goto(`/t/${task.id}`); const session = new SessionPage(testPage); await session.waitForLoad(); @@ -268,12 +270,19 @@ test.describe("Task session queue", () => { const panel = session.activeChat().getByTestId("queued-ghost-list"); const entries = panel.getByTestId("queue-entry-text"); await expect(entries).toHaveCount(1); + const autoMerge = panel.getByTestId("queue-auto-merge"); + await expect(autoMerge).toHaveAttribute("data-state", "checked"); + await autoMerge.click(); + await expect(autoMerge).toHaveAttribute("data-state", "unchecked"); + await expect(entries.first()).toContainText("automatic first"); await expect(entries.first()).toContainText("automatic second"); await requestMessageQueueSettings(apiClient, "PATCH", { auto_merge_enabled: false }); - await apiClient.queueMessage(task.id, sessionId, "separate third"); - await apiClient.queueMessage(task.id, sessionId, "separate fourth"); + await requestMessageQueueSettings(apiClient, "PATCH", { auto_merge_enabled: true }); + await expect(autoMerge).toHaveAttribute("data-state", "unchecked"); + await apiClient.queueMessage(queueIdentity, "separate third"); + await apiClient.queueMessage(queueIdentity, "separate fourth"); await expect(entries).toHaveCount(3, { timeout: 10_000 }); await expect(entries.nth(1)).toHaveText("separate third"); await expect(entries.nth(2)).toHaveText("separate fourth"); @@ -295,7 +304,8 @@ test.describe("Task session queue", () => { // isolation). Enabling automatic merge must fold the next compatible // message into the tail instead of rejecting it as "queue full". await requestMessageQueueSettings(apiClient, "PATCH", { auto_merge_enabled: true }); - await apiClient.queueMessage(taskId, sessionId, "folded while full"); + const queueIdentity = await apiClient.getQueueSessionIdentity(taskId, sessionId); + await apiClient.queueMessage(queueIdentity, "folded while full"); const chat = session.activeChat(); const chip = chat.getByTestId("queue-chip"); @@ -789,12 +799,13 @@ test.describe("Queued row controls", () => { .first(); await expect(chatMaximize).toBeVisible(); await chatMaximize.click(); - const autoRunResponse = await apiClient.setQueueAutoRun(sessionId, false); + const identity = await apiClient.getQueueSessionIdentity(taskId, sessionId); + const autoRunResponse = await apiClient.setQueueAutoRun(identity, false); expect(autoRunResponse).toMatchObject({ session_id: sessionId, auto_run: false }); await waitForComposerQueueMode(testPage); - await apiClient.queueMessage(taskId, sessionId, fixture); + await apiClient.queueMessage(identity, fixture); await expect - .poll(() => apiClient.getQueueStatus(sessionId)) + .poll(() => apiClient.getQueueStatus(identity)) .toMatchObject({ count: 1, auto_run: false }); const chat = session.activeChat(); diff --git a/apps/web/e2e/tests/chat/mid-turn-steering.spec.ts b/apps/web/e2e/tests/chat/mid-turn-steering.spec.ts index 60978f2e78..0710b7e26c 100644 --- a/apps/web/e2e/tests/chat/mid-turn-steering.spec.ts +++ b/apps/web/e2e/tests/chat/mid-turn-steering.spec.ts @@ -88,7 +88,8 @@ test.describe.serial("Claude mid-turn steering experiment", () => { ); await waitForActiveSessionSupportsSteering(testPage, true); - await apiClient.queueMessage(taskId, sessionId, "already queued"); + const queueIdentity = await apiClient.getQueueSessionIdentity(taskId, sessionId); + await apiClient.queueMessage(queueIdentity, "already queued"); const chat = session.activeChat(); await expect(chat.getByTestId("queue-chip")).toBeVisible({ timeout: 10_000 }); diff --git a/apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts b/apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts index 783c366ea9..5248647338 100644 --- a/apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts +++ b/apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts @@ -141,12 +141,13 @@ test("mobile Send Now resumes Auto-run in targeted order without overflow", asyn const markerA = "mobile targeted A response"; const markerB = "mobile targeted B response"; const markerC = "mobile targeted C response"; + const queueIdentity = await apiClient.getQueueSessionIdentity(taskId, sessionId); for (const message of [ scriptedQueueMessage(markerA), scriptedQueueMessage(markerB, 1_000), scriptedQueueMessage(markerC), ]) { - await apiClient.queueMessage(taskId, sessionId, message); + await apiClient.queueMessage(queueIdentity, message); } await chat.getByTestId("queue-chip").tap(); @@ -154,11 +155,23 @@ test("mobile Send Now resumes Auto-run in targeted order without overflow", asyn await expect(panel.getByTestId("queue-entry-text")).toHaveCount(3); const rowSendNow = panel.getByTestId("queue-entry-send-now").nth(1); const autoRun = panel.getByTestId("queue-auto-run"); + const autoMerge = panel.getByTestId("queue-auto-merge"); await expectTouchTarget(rowSendNow); await expectEffectiveTouchTarget(autoRun); + await expectEffectiveTouchTarget(autoMerge); await expect(autoRun).toHaveAttribute("data-state", "checked"); await autoRun.tap(); await expect(autoRun).toHaveAttribute("data-state", "unchecked"); + await expect(autoMerge).toHaveAttribute("data-state", "unchecked"); + await expect(autoMerge).toBeEnabled(); + await autoMerge.tap(); + await expect + .poll(async () => (await apiClient.getQueueStatus(queueIdentity)).auto_merge_enabled) + .toBe(true); + await expect(autoMerge).toHaveAttribute("data-state", "checked"); + await expect(autoMerge).toBeEnabled(); + await autoMerge.tap(); + await expect(autoMerge).toHaveAttribute("data-state", "unchecked"); await assertNoDocumentHorizontalOverflow(testPage); @@ -198,6 +211,7 @@ test("mobile queue panel hides the desktop-only pin and keeps its controls", asy // queue panel, while the other header controls stay touch-sized. await expect(panel.getByTestId("queue-pin")).toHaveCount(0); await expectEffectiveTouchTarget(panel.getByTestId("queue-auto-run")); + await expectEffectiveTouchTarget(panel.getByTestId("queue-auto-merge")); await expectTouchTarget(panel.getByTestId("queue-clear-all")); await expectTouchTarget(panel.getByTestId("queue-close")); }); @@ -233,15 +247,16 @@ test.describe("Mobile queued row controls", () => { await session.sendMessageViaButton("/sleep 60"); await expect(session.agentStatus()).toBeVisible({ timeout: 15_000 }); await waitForActiveSessionForegroundActivity(testPage, "generating"); - const autoRunResponse = await apiClient.setQueueAutoRun(task.session_id, false); + const identity = await apiClient.getQueueSessionIdentity(task.id, task.session_id); + const autoRunResponse = await apiClient.setQueueAutoRun(identity, false); expect(autoRunResponse).toMatchObject({ session_id: task.session_id, auto_run: false, }); await waitForComposerQueueMode(testPage); - await apiClient.queueMessage(task.id, task.session_id, fixture); + await apiClient.queueMessage(identity, fixture); await expect - .poll(() => apiClient.getQueueStatus(task.session_id!)) + .poll(() => apiClient.getQueueStatus(identity)) .toMatchObject({ count: 1, auto_run: false }); const chat = session.activeChat(); diff --git a/apps/web/e2e/tests/ssh/launch-task.spec.ts b/apps/web/e2e/tests/ssh/launch-task.spec.ts index 59dba48866..d4f10bdf5a 100644 --- a/apps/web/e2e/tests/ssh/launch-task.spec.ts +++ b/apps/web/e2e/tests/ssh/launch-task.spec.ts @@ -347,7 +347,8 @@ printf 'prepared\\n' > "$workspace/custom-prepare-marker" ); if (!task.session_id) throw new Error("createTaskWithAgent did not return a session_id"); - await apiClient.queueMessage(task.id, task.session_id, "Inspect the follow-up file.", [ + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, task.session_id); + await apiClient.queueMessage(queueIdentity, "Inspect the follow-up file.", [ { type: "resource", data: Buffer.from(content, "utf8").toString("base64"), diff --git a/apps/web/e2e/tests/task/mobile-sidebar-queued-count.spec.ts b/apps/web/e2e/tests/task/mobile-sidebar-queued-count.spec.ts index a25c669d2d..837ade2931 100644 --- a/apps/web/e2e/tests/task/mobile-sidebar-queued-count.spec.ts +++ b/apps/web/e2e/tests/task/mobile-sidebar-queued-count.spec.ts @@ -24,7 +24,8 @@ test.describe("Mobile sidebar — queued prompt count", () => { state: "IDLE", agentProfileId: seedData.agentProfileId, }); - await apiClient.queueMessage(task.id, sessionId, "Mobile queued prompt"); + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, sessionId); + await apiClient.queueMessage(queueIdentity, "Mobile queued prompt"); await testPage.goto(`/t/${task.id}`); const session = new SessionPage(testPage); diff --git a/apps/web/e2e/tests/task/sidebar-queued-count.spec.ts b/apps/web/e2e/tests/task/sidebar-queued-count.spec.ts index ebe39b9e47..fb3c5f68f5 100644 --- a/apps/web/e2e/tests/task/sidebar-queued-count.spec.ts +++ b/apps/web/e2e/tests/task/sidebar-queued-count.spec.ts @@ -39,8 +39,9 @@ test.describe("Sidebar queued prompt count", () => { state: "IDLE", agentProfileId: seedData.agentProfileId, }); + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, sessionId); for (let index = 0; index < 3; index++) { - await apiClient.queueMessage(task.id, sessionId, `Queued prompt ${index + 1}`); + await apiClient.queueMessage(queueIdentity, `Queued prompt ${index + 1}`); } await testPage.goto(`/t/${task.id}`); @@ -60,7 +61,7 @@ test.describe("Sidebar queued prompt count", () => { // Clearing the queue must remove the badge without a reload (live path // through the status-summary broadcast). - await apiClient.clearQueue(sessionId); + await apiClient.clearQueue(queueIdentity); await expect(badge).not.toBeVisible({ timeout: 10_000 }); }); @@ -95,7 +96,8 @@ test.describe("Sidebar queued prompt count", () => { state: "IDLE", agentProfileId: seedData.agentProfileId, }); - await apiClient.queueMessage(child.id, sessionId, "Subtask queued prompt"); + const queueIdentity = await apiClient.getQueueSessionIdentity(child.id, sessionId); + await apiClient.queueMessage(queueIdentity, "Subtask queued prompt"); await testPage.goto(`/t/${parent.id}`); const session = new SessionPage(testPage); diff --git a/apps/web/e2e/tests/workflow/workflow-manual-move-queue.spec.ts b/apps/web/e2e/tests/workflow/workflow-manual-move-queue.spec.ts index 02bed95a2e..cc1d4fcf1e 100644 --- a/apps/web/e2e/tests/workflow/workflow-manual-move-queue.spec.ts +++ b/apps/web/e2e/tests/workflow/workflow-manual-move-queue.spec.ts @@ -11,7 +11,7 @@ test.describe("Workflow queued messages", () => { apiClient, seedData, }) => { - const { session, sessionId } = await seedQueuedWorkflowMessageScenario( + const { session, queueIdentity } = await seedQueuedWorkflowMessageScenario( testPage, apiClient, seedData, @@ -19,6 +19,6 @@ test.describe("Workflow queued messages", () => { ); await expectWorkflowQueueBadge(session); - await expectDeliveredWorkflowMessage(apiClient, session, sessionId); + await expectDeliveredWorkflowMessage(apiClient, session, queueIdentity); }); }); diff --git a/apps/web/e2e/tests/workflow/workflow-queue-helpers.ts b/apps/web/e2e/tests/workflow/workflow-queue-helpers.ts index 3fb4d72a81..929c1d94e2 100644 --- a/apps/web/e2e/tests/workflow/workflow-queue-helpers.ts +++ b/apps/web/e2e/tests/workflow/workflow-queue-helpers.ts @@ -1,11 +1,12 @@ import { type Page, expect } from "@playwright/test"; -import type { ApiClient } from "../../helpers/api-client"; +import type { ApiClient, QueueSessionIdentityInput } from "../../helpers/api-client"; import type { SeedData } from "../../fixtures/test-base"; import { SessionPage } from "../../pages/session-page"; type QueuedWorkflowScenario = { session: SessionPage; sessionId: string; + queueIdentity: QueueSessionIdentityInput; }; const WORKFLOW_REVIEW_STEP = "Review"; @@ -61,7 +62,8 @@ export async function seedQueuedWorkflowMessageScenario( await apiClient.moveTask(task.id, workflow.id, reviewStep.id); - return { session, sessionId: task.session_id }; + const queueIdentity = await apiClient.getQueueSessionIdentity(task.id, task.session_id); + return { session, sessionId: task.session_id, queueIdentity }; } export async function expectWorkflowQueueBadge(session: SessionPage) { @@ -79,15 +81,15 @@ export async function expectWorkflowQueueBadge(session: SessionPage) { export async function expectDeliveredWorkflowMessage( apiClient: ApiClient, session: SessionPage, - sessionId: string, + queueIdentity: QueueSessionIdentityInput, ) { const chat = session.activeChat(); await expect .poll( async () => { const [queue, { messages }] = await Promise.all([ - apiClient.getQueueStatus(sessionId), - apiClient.listSessionMessages(sessionId), + apiClient.getQueueStatus(queueIdentity), + apiClient.listSessionMessages(queueIdentity.sessionId), ]); const workflowMessageExists = messages.some( (message) => diff --git a/apps/web/hooks/domains/comments/use-run-comment.test.ts b/apps/web/hooks/domains/comments/use-run-comment.test.ts index 8666bdcccf..876b1d8297 100644 --- a/apps/web/hooks/domains/comments/use-run-comment.test.ts +++ b/apps/web/hooks/domains/comments/use-run-comment.test.ts @@ -55,8 +55,11 @@ function makeStoreState(sessionState: string, planMode = false, foregroundActivi return { taskSessions: { items: { - "sess-1": { state: sessionState, foreground_activity: foregroundActivity }, - "other-session": { state: "RUNNING", foreground_activity: "generating" }, + "sess-1": { + state: sessionState, + foreground_activity: foregroundActivity, + queue_incarnation_id: "inc-1", + }, }, }, chatInput: { diff --git a/apps/web/hooks/domains/comments/use-run-comment.ts b/apps/web/hooks/domains/comments/use-run-comment.ts index fde457be7d..e212060ffa 100644 --- a/apps/web/hooks/domains/comments/use-run-comment.ts +++ b/apps/web/hooks/domains/comments/use-run-comment.ts @@ -68,6 +68,7 @@ type UseRunCommentParams = { type QueuePayload = { session_id: string; task_id: string; + session_incarnation_id: string; content: string; plan_mode?: boolean; }; @@ -84,10 +85,16 @@ type MessagePayload = { function buildQueuePayload( sessionId: string, taskId: string, + sessionIncarnationId: string, content: string, planModeEnabled: boolean, ): QueuePayload { - const payload: QueuePayload = { session_id: sessionId, task_id: taskId, content }; + const payload: QueuePayload = { + session_id: sessionId, + session_incarnation_id: sessionIncarnationId, + task_id: taskId, + content, + }; if (planModeEnabled) payload.plan_mode = true; return payload; } @@ -141,7 +148,18 @@ export function useRunComment({ sessionId, taskId }: UseRunCommentParams) { throw new Error("Session is not available for input"); } if (inputMode === "queue") { - await appendToQueue(buildQueuePayload(sessionId, taskId, content, planModeEnabled)); + if (!activeSession?.queue_incarnation_id) { + throw new Error("Session is not available for input"); + } + await appendToQueue( + buildQueuePayload( + sessionId, + taskId, + activeSession.queue_incarnation_id, + content, + planModeEnabled, + ), + ); } else { const client = getWebSocketClient(); if (!client) throw new Error("WebSocket client unavailable"); diff --git a/apps/web/hooks/domains/review/use-send-finding-to-agent.ts b/apps/web/hooks/domains/review/use-send-finding-to-agent.ts index 064e595cd5..6765855624 100644 --- a/apps/web/hooks/domains/review/use-send-finding-to-agent.ts +++ b/apps/web/hooks/domains/review/use-send-finding-to-agent.ts @@ -7,7 +7,7 @@ import { useToast } from "@/components/toast-provider"; import { appendToQueue } from "@/lib/api/domains/queue-api"; import { formatFindingAsMarkdown } from "@/lib/review/format"; import type { TaskReviewFinding } from "@/lib/types/review"; -import type { Message } from "@/lib/types/http"; +import type { Message, TaskSession } from "@/lib/types/http"; import { getWebSocketClient } from "@/lib/ws/connection"; import { generateUUID } from "@/lib/utils"; @@ -20,6 +20,42 @@ function isAgentBusy(state: string | undefined): boolean { return state === "STARTING" || state === "RUNNING"; } +async function queueFinding( + taskId: string, + session: TaskSession | null, + sessionId: string, + content: string, + planMode: boolean, +) { + if (!session?.queue_incarnation_id) { + throw new Error("Session is not available for input"); + } + await appendToQueue({ + session_id: sessionId, + session_incarnation_id: session.queue_incarnation_id, + task_id: taskId, + content, + ...(planMode ? { plan_mode: true } : {}), + }); +} + +async function sendFinding(taskId: string, sessionId: string, content: string, planMode: boolean) { + const client = getWebSocketClient(); + if (!client) throw new Error("WebSocket client unavailable"); + return client.request( + "message.add", + { + task_id: taskId, + session_id: sessionId, + client_message_id: generateUUID(), + content, + has_review_comments: true, + ...(planMode ? { plan_mode: true } : {}), + }, + 10000, + ); +} + /** * Sends a finding to the active session as follow-up context. * @@ -42,31 +78,11 @@ export function useSendFindingToAgent({ taskId, sessionId }: Params) { try { if (isAgentBusy(session?.state)) { - await appendToQueue({ - session_id: sessionId, - task_id: taskId, - content, - ...(planMode ? { plan_mode: true } : {}), - }); + await queueFinding(taskId, session, sessionId, content, planMode); toast({ title: t("review:findingQueuedForAgent"), variant: "success" }); return; } - const client = getWebSocketClient(); - if (!client) throw new Error("WebSocket client unavailable"); - const created = await client.request( - "message.add", - { - task_id: taskId, - session_id: sessionId, - client_message_id: generateUUID(), - content, - has_review_comments: true, - ...(planMode ? { plan_mode: true } : {}), - }, - 10000, - ); - // Add the returned message directly so the chat updates even if the - // broadcast is missed; addMessage is idempotent on id. + const created = await sendFinding(taskId, sessionId, content, planMode); if (created?.id && created.session_id) storeApi.getState().addMessage(created); toast({ title: t("review:findingSentToAgent"), variant: "success" }); } catch (error) { diff --git a/apps/web/hooks/domains/session/use-queue-admission.ts b/apps/web/hooks/domains/session/use-queue-admission.ts new file mode 100644 index 0000000000..65645394a7 --- /dev/null +++ b/apps/web/hooks/domains/session/use-queue-admission.ts @@ -0,0 +1,73 @@ +import { useCallback } from "react"; +import { queueMessage } from "@/lib/api/domains/queue-api"; +import type { QueueMessageParams, QueueSessionIdentity } from "@/lib/api/domains/queue-api"; +import type { QueueOperationToken, SessionSlice } from "@/lib/state/slices/session/types"; +import type { QueueRefetch } from "./use-queue-refetch"; +import type { QueueMessageInput } from "./use-queue-types"; + +type BeginQueueOperation = SessionSlice["beginQueueOperation"]; +type FinishQueueOperation = SessionSlice["finishQueueOperation"]; + +async function queueWithTerminalRefetch( + params: QueueMessageParams, + refetch: QueueRefetch, + token: QueueOperationToken, +): Promise { + let mutationError: unknown; + try { + await queueMessage(params); + } catch (error) { + mutationError = error; + } + try { + await refetch(params.session_id, token); + } catch (reconcileError) { + if (mutationError) throw mutationError; + throw reconcileError; + } + if (mutationError) throw mutationError; +} + +export function useQueueAdmissionAction( + identity: QueueSessionIdentity | null, + beginQueueOperation: BeginQueueOperation, + finishQueueOperation: FinishQueueOperation, + refetch: QueueRefetch, +) { + return useCallback( + async ({ + taskId, + content, + model, + planMode, + attachments, + entityReferences, + contextFilesMeta, + }: QueueMessageInput) => { + if (!identity || identity.task_id !== taskId) return; + const { session_id: sessionId, session_incarnation_id: incarnationId } = identity; + const token = beginQueueOperation(sessionId, incarnationId); + if (!token) return; + try { + await queueWithTerminalRefetch( + { + session_id: sessionId, + session_incarnation_id: incarnationId, + task_id: taskId, + content, + model, + plan_mode: planMode, + attachments, + entity_references: entityReferences, + ...(contextFilesMeta ? { context_files: contextFilesMeta } : {}), + }, + refetch, + token, + ); + } finally { + finishQueueOperation(sessionId, token); + } + }, + [beginQueueOperation, finishQueueOperation, identity, refetch], + ); +} diff --git a/apps/web/hooks/domains/session/use-queue-refetch.ts b/apps/web/hooks/domains/session/use-queue-refetch.ts new file mode 100644 index 0000000000..29f6c06fbb --- /dev/null +++ b/apps/web/hooks/domains/session/use-queue-refetch.ts @@ -0,0 +1,85 @@ +import { useCallback, useRef } from "react"; +import { getQueueStatus } from "@/lib/api/domains/queue-api"; +import type { QueueSessionIdentity } from "@/lib/api/domains/queue-api"; +import type { + QueueOperationToken, + QueueStatus, + SessionSlice, +} from "@/lib/state/slices/session/types"; + +type BeginQueueOperation = SessionSlice["beginQueueOperation"]; +type FinishQueueOperation = SessionSlice["finishQueueOperation"]; +type SetQueueEntries = SessionSlice["setQueueEntries"]; + +export type QueueRefetch = (sessionId: string, token?: QueueOperationToken) => Promise; + +function queueStatusMatchesIdentity(status: QueueStatus, identity: QueueSessionIdentity): boolean { + return ( + status.task_id === identity.task_id && + status.session_id === identity.session_id && + status.session_incarnation_id === identity.session_incarnation_id + ); +} + +function queueStatusMeta(status: QueueStatus) { + return { + count: status.count, + max: status.max, + mergeEnabled: status.merge_enabled ?? true, + autoRun: status.auto_run ?? true, + taskId: status.task_id, + sessionIncarnationId: status.session_incarnation_id, + ...(status.status_epoch !== undefined ? { statusEpoch: status.status_epoch } : {}), + ...(status.status_generation !== undefined + ? { statusGeneration: status.status_generation } + : {}), + ...(status.auto_merge_available !== undefined + ? { autoMergeAvailable: status.auto_merge_available } + : {}), + ...(status.auto_merge_enabled !== undefined + ? { autoMergeEnabled: status.auto_merge_enabled } + : {}), + ...(status.auto_merge_source !== undefined + ? { autoMergeSource: status.auto_merge_source } + : {}), + ...(status.auto_merge_revision !== undefined + ? { autoMergeRevision: status.auto_merge_revision } + : {}), + }; +} + +/** Fetch and reconcile one immutable session's authoritative queue snapshot. */ +export function useQueueRefetch( + identity: QueueSessionIdentity | null, + setQueueEntries: SetQueueEntries, + beginQueueOperation: BeginQueueOperation, + finishQueueOperation: FinishQueueOperation, +) { + const refetchVersion = useRef>({}); + const invalidate = useCallback((sessionId: string) => { + refetchVersion.current[sessionId] = (refetchVersion.current[sessionId] ?? 0) + 1; + }, []); + const refetch = useCallback( + async (sessionId, callerToken) => { + if (!identity || identity.session_id !== sessionId) return; + const token = + callerToken ?? beginQueueOperation(identity.session_id, identity.session_incarnation_id); + if (!token) return; + const ownsToken = callerToken === undefined; + const version = (refetchVersion.current[sessionId] ?? 0) + 1; + refetchVersion.current[sessionId] = version; + try { + const status = await getQueueStatus(identity); + if (refetchVersion.current[sessionId] !== version) return; + if (!queueStatusMatchesIdentity(status, identity)) return; + setQueueEntries(sessionId, status.entries ?? [], queueStatusMeta(status), { + establishStatusEpoch: true, + }); + } finally { + if (ownsToken) finishQueueOperation(sessionId, token); + } + }, + [beginQueueOperation, finishQueueOperation, identity, setQueueEntries], + ); + return { refetch, invalidate }; +} diff --git a/apps/web/hooks/domains/session/use-queue-reorder.test.ts b/apps/web/hooks/domains/session/use-queue-reorder.test.ts new file mode 100644 index 0000000000..f756a8b7c7 --- /dev/null +++ b/apps/web/hooks/domains/session/use-queue-reorder.test.ts @@ -0,0 +1,200 @@ +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Mock } from "vitest"; +import type { QueuedMessage, QueueOperationToken } from "@/lib/state/slices/session/types"; + +const queueApiMock = vi.hoisted(() => { + class QueueEntryNotFoundError extends Error {} + class QueueSendNowError extends Error {} + class QueueReorderError extends Error {} + return { + QueueEntryNotFoundError, + QueueSendNowError, + QueueReorderError, + queueMessage: vi.fn(), + clearQueue: vi.fn(), + getQueueStatus: vi.fn(), + updateQueuedMessage: vi.fn(), + removeQueuedEntry: vi.fn(), + mergeQueuedEntry: vi.fn(), + reorderQueuedEntries: vi.fn(), + sendQueuedNow: vi.fn(), + setQueueAutoRun: vi.fn(), + setQueueAutoMerge: vi.fn(), + }; +}); + +type MockQueueState = { + queue: { + bySessionId: Record; + metaBySessionId: Record< + string, + { count: number; max: number; mergeEnabled?: boolean; autoRun?: boolean } + >; + activeOperationBySessionId: Record; + }; + connection: { status: string }; + taskSessions: { + items: Record; + }; + setQueueEntries: Mock; + removeQueueEntry: Mock; + beginQueueOperation: Mock; + finishQueueOperation: Mock; +}; + +let mockState: MockQueueState; + +vi.mock("@/components/state-provider", () => ({ + useAppStore: (selector: (state: MockQueueState) => unknown) => selector(mockState), +})); + +vi.mock("@/lib/api/domains/queue-api", () => queueApiMock); + +import { useQueue } from "./use-queue"; + +const SESSION_ID = "sess-1"; +const TASK_ID = "task-1"; +const IDENTITY = { + task_id: TASK_ID, + session_id: SESSION_ID, + session_incarnation_id: "incarnation-1", +}; + +function entry(overrides: Partial = {}): QueuedMessage { + return { + id: "q-1", + session_id: SESSION_ID, + task_id: TASK_ID, + content: "queued prompt", + plan_mode: false, + queued_at: "2026-06-27T00:00:00Z", + queued_by: "user", + ...overrides, + }; +} + +beforeEach(() => { + mockState = { + queue: { + bySessionId: {}, + metaBySessionId: {}, + activeOperationBySessionId: {}, + }, + connection: { status: "connected" }, + taskSessions: { + items: { + [SESSION_ID]: { + task_id: TASK_ID, + queue_incarnation_id: IDENTITY.session_incarnation_id, + }, + }, + }, + setQueueEntries: vi.fn(), + removeQueueEntry: vi.fn(), + beginQueueOperation: vi + .fn() + .mockReturnValue({ sessionIncarnationId: IDENTITY.session_incarnation_id, generation: 1 }), + finishQueueOperation: vi.fn(), + }; + queueApiMock.getQueueStatus.mockResolvedValue({ ...IDENTITY, entries: [], count: 0, max: 10 }); + queueApiMock.reorderQueuedEntries.mockReset(); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("useQueue reorderEntries", () => { + it("optimistically reorders the store and refetches the authoritative queue", async () => { + const first = entry({ id: "q-1", content: "first" }); + const second = entry({ id: "q-2", content: "second" }); + mockState.queue.bySessionId[SESSION_ID] = [first, second]; + mockState.queue.metaBySessionId[SESSION_ID] = { count: 2, max: 10 }; + queueApiMock.getQueueStatus.mockResolvedValue({ + ...IDENTITY, + entries: [first, second], + count: 2, + max: 10, + }); + queueApiMock.reorderQueuedEntries.mockResolvedValue({ + session_id: SESSION_ID, + reordered: 2, + }); + + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); + queueApiMock.getQueueStatus.mockClear(); + + await act(async () => { + await result.current.reorderEntries([second.id, first.id]); + }); + + expect(queueApiMock.reorderQueuedEntries).toHaveBeenCalledWith({ + session_id: SESSION_ID, + task_id: TASK_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, + ordered_ids: [second.id, first.id], + }); + expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [second, first], { + count: 2, + max: 10, + mergeEnabled: true, + autoRun: true, + }); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + }); + + it("refetches and rethrows when the reorder fails", async () => { + const first = entry({ id: "q-1", content: "first" }); + const second = entry({ id: "q-2", content: "second" }); + queueApiMock.getQueueStatus.mockResolvedValue({ + ...IDENTITY, + entries: [first, second], + count: 2, + max: 10, + }); + queueApiMock.reorderQueuedEntries.mockRejectedValueOnce(new Error("reorder failed")); + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); + queueApiMock.getQueueStatus.mockClear(); + queueApiMock.getQueueStatus.mockResolvedValueOnce({ + ...IDENTITY, + entries: [first, second], + count: 2, + max: 10, + }); + + await act(async () => { + await expect(result.current.reorderEntries([second.id, first.id])).rejects.toThrow( + "reorder failed", + ); + }); + + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + }); + + it("refetches and rethrows a QueueReorderError so the panel can swallow it", async () => { + const first = entry({ id: "q-1", content: "first" }); + queueApiMock.getQueueStatus.mockResolvedValue({ + ...IDENTITY, + entries: [first], + count: 1, + max: 10, + }); + queueApiMock.reorderQueuedEntries.mockRejectedValueOnce(new queueApiMock.QueueReorderError()); + + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); + queueApiMock.getQueueStatus.mockClear(); + + await act(async () => { + await expect(result.current.reorderEntries([first.id])).rejects.toThrow( + queueApiMock.QueueReorderError, + ); + }); + + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + }); +}); diff --git a/apps/web/hooks/domains/session/use-queue-types.ts b/apps/web/hooks/domains/session/use-queue-types.ts new file mode 100644 index 0000000000..278c356e1b --- /dev/null +++ b/apps/web/hooks/domains/session/use-queue-types.ts @@ -0,0 +1,22 @@ +import type { QueueMessageParams } from "@/lib/api/domains/queue-api"; +import type { EntityReference } from "@/lib/types/entity-reference"; + +export type MessageAttachment = { + type: string; + data?: string; + attachment_id?: string; + mime_type: string; + name?: string; + size_bytes?: number; + delivery_mode?: "prompt" | "path"; +}; + +export type QueueMessageInput = { + taskId: string; + content: string; + model?: string; + planMode?: boolean; + attachments?: MessageAttachment[]; + entityReferences?: EntityReference[]; + contextFilesMeta?: QueueMessageParams["context_files"]; +}; diff --git a/apps/web/hooks/domains/session/use-queue.test.ts b/apps/web/hooks/domains/session/use-queue.test.ts index 9a568c5ce5..d3c14ca415 100644 --- a/apps/web/hooks/domains/session/use-queue.test.ts +++ b/apps/web/hooks/domains/session/use-queue.test.ts @@ -1,6 +1,6 @@ import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { QueuedMessage } from "@/lib/state/slices/session/types"; +import type { QueuedMessage, QueueOperationToken } from "@/lib/state/slices/session/types"; import type { EntityReference } from "@/lib/types/entity-reference"; const queueApiMock = vi.hoisted(() => { @@ -20,6 +20,7 @@ const queueApiMock = vi.hoisted(() => { reorderQueuedEntries: vi.fn(), sendQueuedNow: vi.fn(), setQueueAutoRun: vi.fn(), + setQueueAutoMerge: vi.fn(), }; }); @@ -30,13 +31,23 @@ type MockQueueState = { string, { count: number; max: number; mergeEnabled?: boolean; autoRun?: boolean } >; - isLoading: Record; + activeOperationBySessionId: Record; }; connection: { status: string }; - taskSessions: { items: Record }; + taskSessions: { + items: Record< + string, + { + task_id?: string; + queue_incarnation_id?: string; + cancellation_pending?: boolean; + } + >; + }; setQueueEntries: ReturnType; removeQueueEntry: ReturnType; - setQueueLoading: ReturnType; + beginQueueOperation: ReturnType; + finishQueueOperation: ReturnType; }; let mockState: MockQueueState; @@ -51,6 +62,11 @@ import { useQueue } from "./use-queue"; const SESSION_ID = "sess-1"; const TASK_ID = "task-1"; +const IDENTITY = { + task_id: TASK_ID, + session_id: SESSION_ID, + session_incarnation_id: "incarnation-1", +}; const reference: EntityReference = { version: 1, ref: "mention:v1:github:issue:acme%2Frepo:42", @@ -88,28 +104,38 @@ function resetMockState() { queue: { bySessionId: {}, metaBySessionId: {}, - isLoading: {}, + activeOperationBySessionId: {}, }, connection: { status: "connected" }, - taskSessions: { items: {} }, + taskSessions: { + items: { + [SESSION_ID]: { + task_id: TASK_ID, + queue_incarnation_id: IDENTITY.session_incarnation_id, + }, + }, + }, setQueueEntries: vi.fn(), removeQueueEntry: vi.fn(), - setQueueLoading: vi.fn(), + beginQueueOperation: vi + .fn() + .mockReturnValue({ sessionIncarnationId: IDENTITY.session_incarnation_id, generation: 1 }), + finishQueueOperation: vi.fn(), }; } +beforeEach(() => { + resetMockState(); + setDocumentVisibility("visible"); + queueApiMock.getQueueStatus.mockResolvedValue({ ...IDENTITY, entries: [], count: 0, max: 10 }); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + describe("useQueue", () => { - beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); - }); - - afterEach(() => { - cleanup(); - vi.clearAllMocks(); - }); - it("refetches the queue snapshot when the WebSocket reconnects", async () => { mockState.connection.status = "disconnected"; const { rerender } = renderHook(() => useQueue(SESSION_ID)); @@ -120,19 +146,40 @@ describe("useQueue", () => { mockState.connection.status = "connected"; rerender(); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID)); - expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [], { - count: 0, - max: 10, - mergeEnabled: true, - autoRun: true, + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY)); + expect(mockState.setQueueEntries).toHaveBeenCalledWith( + SESSION_ID, + [], + { + count: 0, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: IDENTITY.session_incarnation_id, + }, + { establishStatusEpoch: true }, + ); + }); + + it("rejects an identity-less queue snapshot after session identity is known", async () => { + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); + mockState.setQueueEntries.mockClear(); + queueApiMock.getQueueStatus.mockResolvedValueOnce({ entries: [entry()], count: 1, max: 10 }); + + await act(async () => { + await result.current.refetch(); }); + + expect(mockState.setQueueEntries).not.toHaveBeenCalled(); }); it("refetches a stale queue snapshot when a suspended tab becomes visible again", async () => { mockState.queue.bySessionId[SESSION_ID] = [entry()]; mockState.queue.metaBySessionId[SESSION_ID] = { count: 1, max: 10 }; queueApiMock.getQueueStatus.mockResolvedValueOnce({ + ...IDENTITY, entries: [entry()], count: 1, max: 10, @@ -143,17 +190,29 @@ describe("useQueue", () => { queueApiMock.getQueueStatus.mockClear(); mockState.setQueueEntries.mockClear(); - queueApiMock.getQueueStatus.mockResolvedValueOnce({ entries: [], count: 0, max: 10 }); + queueApiMock.getQueueStatus.mockResolvedValueOnce({ + ...IDENTITY, + entries: [], + count: 0, + max: 10, + }); document.dispatchEvent(new Event("visibilitychange")); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID)); - expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [], { - count: 0, - max: 10, - mergeEnabled: true, - autoRun: true, - }); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY)); + expect(mockState.setQueueEntries).toHaveBeenCalledWith( + SESSION_ID, + [], + { + count: 0, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: IDENTITY.session_incarnation_id, + }, + { establishStatusEpoch: true }, + ); }); it("refetches a stale queue snapshot when the Kandev window regains focus", async () => { @@ -163,7 +222,7 @@ describe("useQueue", () => { window.dispatchEvent(new Event("focus")); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY)); }); it("does not refetch on foreground visibility while disconnected", async () => { @@ -175,7 +234,9 @@ describe("useQueue", () => { expect(queueApiMock.getQueueStatus).not.toHaveBeenCalled(); }); +}); +describe("useQueue message metadata", () => { it("queues structured references with busy-agent messages", async () => { queueApiMock.queueMessage.mockResolvedValue(entry()); const { result } = renderHook(() => useQueue(SESSION_ID)); @@ -192,6 +253,7 @@ describe("useQueue", () => { expect(queueApiMock.queueMessage).toHaveBeenCalledWith({ session_id: SESSION_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, task_id: TASK_ID, content: "queued reference", model: undefined, @@ -201,6 +263,22 @@ describe("useQueue", () => { }); }); + it("refetches authoritative status after a queue admission error", async () => { + const mutationError = new Error("queue failed"); + queueApiMock.queueMessage.mockRejectedValueOnce(mutationError); + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); + queueApiMock.getQueueStatus.mockClear(); + + await act(async () => { + await expect(result.current.queue({ taskId: TASK_ID, content: "rejected" })).rejects.toBe( + mutationError, + ); + }); + + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + }); + it("replaces queued reference metadata with an explicit empty array", async () => { queueApiMock.updateQueuedMessage.mockResolvedValue({ entry_id: "q-1" }); const { result } = renderHook(() => useQueue(SESSION_ID)); @@ -212,6 +290,8 @@ describe("useQueue", () => { expect(queueApiMock.updateQueuedMessage).toHaveBeenCalledWith({ session_id: SESSION_ID, + task_id: TASK_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, entry_id: "q-1", content: "reference removed", attachments: undefined, @@ -221,17 +301,6 @@ describe("useQueue", () => { }); describe("useQueue context file metadata and Send Now", () => { - beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); - }); - - afterEach(() => { - cleanup(); - vi.clearAllMocks(); - }); - it("forwards context file metadata with queued messages", async () => { queueApiMock.queueMessage.mockResolvedValue(entry()); const { result } = renderHook(() => useQueue(SESSION_ID)); @@ -248,6 +317,7 @@ describe("useQueue context file metadata and Send Now", () => { expect(queueApiMock.queueMessage).toHaveBeenCalledWith({ session_id: SESSION_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, task_id: TASK_ID, content: "queued context", model: undefined, @@ -274,12 +344,16 @@ describe("useQueue context file metadata and Send Now", () => { expect(queueApiMock.sendQueuedNow).toHaveBeenCalledWith({ session_id: SESSION_ID, + task_id: TASK_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, scope: "entry", entry_id: "q-2", }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); +}); +describe("useQueue automation controls", () => { it("refetches after an Auto-run mutation failure and preserves its error", async () => { const mutationError = new Error("policy update failed"); queueApiMock.setQueueAutoRun.mockRejectedValueOnce(mutationError); @@ -291,12 +365,16 @@ describe("useQueue context file metadata and Send Now", () => { await expect(result.current.setAutoRun(false)).rejects.toBe(mutationError); }); - expect(queueApiMock.setQueueAutoRun).toHaveBeenCalledWith(SESSION_ID, false); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.setQueueAutoRun).toHaveBeenCalledWith(IDENTITY, false); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); it("exposes authoritative cancellation progress for disabling controls", async () => { - mockState.taskSessions.items[SESSION_ID] = { cancellation_pending: true }; + mockState.taskSessions.items[SESSION_ID] = { + task_id: TASK_ID, + queue_incarnation_id: IDENTITY.session_incarnation_id, + cancellation_pending: true, + }; const { result } = renderHook(() => useQueue(SESSION_ID)); await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); @@ -323,18 +401,53 @@ describe("useQueue context file metadata and Send Now", () => { await setAutoRun!(false); }); - expect(queueApiMock.setQueueAutoRun).toHaveBeenCalledWith(SESSION_ID, false); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.setQueueAutoRun).toHaveBeenCalledWith(IDENTITY, false); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + }); + it("sets Auto-merge for the immutable session and refetches policy", async () => { + queueApiMock.getQueueStatus.mockResolvedValue({ + ...IDENTITY, + entries: [], + count: 0, + max: 10, + ...IDENTITY, + merge_enabled: true, + auto_run: true, + auto_merge_available: true, + auto_merge_enabled: true, + auto_merge_source: "global", + auto_merge_revision: 4, + }); + queueApiMock.setQueueAutoMerge.mockResolvedValue({ + ...IDENTITY, + auto_merge_enabled: false, + auto_merge_source: "session", + auto_merge_revision: 1, + }); + const { result } = renderHook(() => useQueue(SESSION_ID)); + await waitFor(() => expect(mockState.setQueueEntries).toHaveBeenCalled()); + expect(mockState.setQueueEntries).toHaveBeenLastCalledWith( + SESSION_ID, + [], + expect.objectContaining({ + autoMergeAvailable: true, + autoMergeEnabled: true, + autoMergeSource: "global", + autoMergeRevision: 4, + }), + { establishStatusEpoch: true }, + ); + + await act(async () => { + await result.current.setAutoMerge(false); + }); + + expect(queueApiMock.setQueueAutoMerge).toHaveBeenCalledWith(IDENTITY, false); + expect(queueApiMock.getQueueStatus).toHaveBeenLastCalledWith(IDENTITY); }); }); describe("useQueue mergeEntry", () => { - beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); - }); - it("merges an entry and refetches the queue", async () => { queueApiMock.mergeQueuedEntry.mockResolvedValue({ entry_id: "q-1" }); const { result } = renderHook(() => useQueue(SESSION_ID)); @@ -347,9 +460,12 @@ describe("useQueue mergeEntry", () => { expect(queueApiMock.mergeQueuedEntry).toHaveBeenCalledWith({ session_id: SESSION_ID, + task_id: TASK_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, entry_id: "q-2", + user_id: undefined, }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); it("forwards an explicit user_id on merge", async () => { @@ -363,6 +479,8 @@ describe("useQueue mergeEntry", () => { expect(queueApiMock.mergeQueuedEntry).toHaveBeenCalledWith({ session_id: SESSION_ID, + task_id: TASK_ID, + session_incarnation_id: IDENTITY.session_incarnation_id, entry_id: "q-2", user_id: "alice", }); @@ -380,16 +498,13 @@ describe("useQueue mergeEntry", () => { ); }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); }); describe("useQueue clearAll", () => { beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); queueApiMock.clearQueue.mockReset(); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); queueApiMock.clearQueue.mockResolvedValue(undefined); }); @@ -402,8 +517,20 @@ describe("useQueue clearAll", () => { await result.current.clearAll(); }); - expect(queueApiMock.clearQueue).toHaveBeenCalledWith(SESSION_ID); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.clearQueue).toHaveBeenCalledWith(IDENTITY); + expect(mockState.setQueueEntries.mock.calls).toContainEqual([ + SESSION_ID, + [], + { + count: 0, + max: 0, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: IDENTITY.session_incarnation_id, + }, + ]); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); it("refetches authoritative status and rethrows when clear fails", async () => { @@ -413,6 +540,7 @@ describe("useQueue clearAll", () => { await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); queueApiMock.getQueueStatus.mockClear(); queueApiMock.getQueueStatus.mockResolvedValueOnce({ + ...IDENTITY, entries: [authoritative], count: 1, max: 10, @@ -422,13 +550,20 @@ describe("useQueue clearAll", () => { await expect(result.current.clearAll()).rejects.toThrow("clear failed"); }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); - expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [authoritative], { - count: 1, - max: 10, - mergeEnabled: true, - autoRun: true, - }); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + expect(mockState.setQueueEntries).toHaveBeenCalledWith( + SESSION_ID, + [authoritative], + { + count: 1, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: IDENTITY.session_incarnation_id, + }, + { establishStatusEpoch: true }, + ); }); it("discards an in-flight refetch that resolves after the clear", async () => { @@ -471,10 +606,7 @@ describe("useQueue clearAll", () => { describe("useQueue removeEntry", () => { beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); queueApiMock.removeQueuedEntry.mockReset(); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); }); it("optimistically removes then refetches authoritative status after success", async () => { @@ -488,7 +620,7 @@ describe("useQueue removeEntry", () => { }); expect(mockState.removeQueueEntry).toHaveBeenCalledWith(SESSION_ID, "q-1"); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); it("refetches after a drain race without surfacing a benign error", async () => { @@ -503,7 +635,7 @@ describe("useQueue removeEntry", () => { await expect(result.current.removeEntry("q-1")).resolves.toBeUndefined(); }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); }); it("refetches and rethrows a failed removal", async () => { @@ -513,6 +645,7 @@ describe("useQueue removeEntry", () => { await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); queueApiMock.getQueueStatus.mockClear(); queueApiMock.getQueueStatus.mockResolvedValueOnce({ + ...IDENTITY, entries: [authoritative], count: 1, max: 10, @@ -522,104 +655,19 @@ describe("useQueue removeEntry", () => { await expect(result.current.removeEntry("q-1")).rejects.toThrow("remove failed"); }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); - expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [authoritative], { - count: 1, - max: 10, - mergeEnabled: true, - autoRun: true, - }); - }); -}); - -describe("useQueue reorderEntries", () => { - beforeEach(() => { - resetMockState(); - setDocumentVisibility("visible"); - queueApiMock.reorderQueuedEntries.mockReset(); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [], count: 0, max: 10 }); - }); - - it("optimistically reorders the store and refetches the authoritative queue", async () => { - const first = entry({ id: "q-1", content: "first" }); - const second = entry({ id: "q-2", content: "second" }); - // The test harness's setQueueEntries is a no-op spy, so seed the store - // directly: the hook's optimistic reorder reads the current entries. - mockState.queue.bySessionId[SESSION_ID] = [first, second]; - mockState.queue.metaBySessionId[SESSION_ID] = { count: 2, max: 10 }; - queueApiMock.getQueueStatus.mockResolvedValue({ - entries: [first, second], - count: 2, - max: 10, - }); - queueApiMock.reorderQueuedEntries.mockResolvedValue({ - session_id: SESSION_ID, - reordered: 2, - }); - - const { result } = renderHook(() => useQueue(SESSION_ID)); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); - queueApiMock.getQueueStatus.mockClear(); - - await act(async () => { - await result.current.reorderEntries([second.id, first.id]); - }); - - expect(queueApiMock.reorderQueuedEntries).toHaveBeenCalledWith({ - session_id: SESSION_ID, - ordered_ids: [second.id, first.id], - }); - expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [second, first], { - count: 2, - max: 10, - mergeEnabled: true, - autoRun: true, - }); - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); - }); - - it("refetches and rethrows when the reorder fails", async () => { - const first = entry({ id: "q-1", content: "first" }); - const second = entry({ id: "q-2", content: "second" }); - queueApiMock.getQueueStatus.mockResolvedValue({ - entries: [first, second], - count: 2, - max: 10, - }); - queueApiMock.reorderQueuedEntries.mockRejectedValueOnce(new Error("reorder failed")); - const { result } = renderHook(() => useQueue(SESSION_ID)); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); - queueApiMock.getQueueStatus.mockClear(); - queueApiMock.getQueueStatus.mockResolvedValueOnce({ - entries: [first, second], - count: 2, - max: 10, - }); - - await act(async () => { - await expect(result.current.reorderEntries([second.id, first.id])).rejects.toThrow( - "reorder failed", - ); - }); - - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); - }); - - it("refetches and rethrows a QueueReorderError so the panel can swallow it", async () => { - const first = entry({ id: "q-1", content: "first" }); - queueApiMock.getQueueStatus.mockResolvedValue({ entries: [first], count: 1, max: 10 }); - queueApiMock.reorderQueuedEntries.mockRejectedValueOnce(new queueApiMock.QueueReorderError()); - - const { result } = renderHook(() => useQueue(SESSION_ID)); - await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled()); - queueApiMock.getQueueStatus.mockClear(); - - await act(async () => { - await expect(result.current.reorderEntries([first.id])).rejects.toThrow( - queueApiMock.QueueReorderError, - ); - }); - - expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(SESSION_ID); + expect(queueApiMock.getQueueStatus).toHaveBeenCalledWith(IDENTITY); + expect(mockState.setQueueEntries).toHaveBeenCalledWith( + SESSION_ID, + [authoritative], + { + count: 1, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: IDENTITY.session_incarnation_id, + }, + { establishStatusEpoch: true }, + ); }); }); diff --git a/apps/web/hooks/domains/session/use-queue.ts b/apps/web/hooks/domains/session/use-queue.ts index 7f3619c128..a1b53ba7da 100644 --- a/apps/web/hooks/domains/session/use-queue.ts +++ b/apps/web/hooks/domains/session/use-queue.ts @@ -1,10 +1,8 @@ -import { useEffect, useCallback, useRef } from "react"; +import { useEffect, useCallback, useMemo } from "react"; import { useAppStore } from "@/components/state-provider"; import { useForegroundRefresh } from "@/hooks/use-foreground-refresh"; import { - queueMessage, clearQueue, - getQueueStatus, updateQueuedMessage, removeQueuedEntry, mergeQueuedEntry, @@ -12,33 +10,22 @@ import { QueueEntryNotFoundError, sendQueuedNow, setQueueAutoRun, + setQueueAutoMerge, } from "@/lib/api/domains/queue-api"; -import type { QueueMessageParams } from "@/lib/api/domains/queue-api"; -import type { QueuedMessage } from "@/lib/state/slices/session/types"; +import type { QueueSessionIdentity } from "@/lib/api/domains/queue-api"; +import type { + QueueMeta, + QueueOperationToken, + QueuedMessage, +} from "@/lib/state/slices/session/types"; +import type { MessageAttachment } from "./use-queue-types"; +export type { MessageAttachment, QueueMessageInput } from "./use-queue-types"; +import { useQueueAdmissionAction } from "./use-queue-admission"; +import { useQueueRefetch, type QueueRefetch } from "./use-queue-refetch"; import type { EntityReference } from "@/lib/types/entity-reference"; const EMPTY_ENTRIES: QueuedMessage[] = []; -export type MessageAttachment = { - type: string; - data?: string; - attachment_id?: string; - mime_type: string; - name?: string; - size_bytes?: number; - delivery_mode?: "prompt" | "path"; -}; - -export type QueueMessageInput = { - taskId: string; - content: string; - model?: string; - planMode?: boolean; - attachments?: MessageAttachment[]; - entityReferences?: EntityReference[]; - contextFilesMeta?: QueueMessageParams["context_files"]; -}; - /** Selectors over the queue slice for one session. */ function useQueueState(sessionId: string | null) { const entries = useAppStore((state) => @@ -47,195 +34,142 @@ function useQueueState(sessionId: string | null) { const meta = useAppStore((state) => sessionId ? state.queue.metaBySessionId[sessionId] : undefined, ); - const isLoading = useAppStore((state) => - sessionId ? (state.queue.isLoading[sessionId] ?? false) : false, - ); const setQueueEntries = useAppStore((state) => state.setQueueEntries); const removeQueueEntry = useAppStore((state) => state.removeQueueEntry); - const setQueueLoading = useAppStore((state) => state.setQueueLoading); + const activeOperation = useAppStore((state) => + sessionId ? state.queue.activeOperationBySessionId[sessionId] : undefined, + ); + const beginQueueOperation = useAppStore((state) => state.beginQueueOperation); + const finishQueueOperation = useAppStore((state) => state.finishQueueOperation); const cancellationPending = useAppStore( (state) => (sessionId ? state.taskSessions.items[sessionId]?.cancellation_pending : false) === true, ); + const taskSession = useAppStore((state) => + sessionId ? state.taskSessions.items[sessionId] : undefined, + ); return { entries, meta, - isLoading, + isLoading: activeOperation !== undefined, cancellationPending, + taskSession, setQueueEntries, removeQueueEntry, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, }; } type QueueActionsArgs = { - sessionId: string | null; + identity: QueueSessionIdentity | null; entries: QueuedMessage[]; setQueueEntries: ReturnType["setQueueEntries"]; removeQueueEntry: ReturnType["removeQueueEntry"]; - setQueueLoading: ReturnType["setQueueLoading"]; + beginQueueOperation: ReturnType["beginQueueOperation"]; + finishQueueOperation: ReturnType["finishQueueOperation"]; metaMax: number | undefined; metaMergeEnabled: boolean | undefined; metaAutoRun: boolean | undefined; }; +type BeginQueueOperation = ReturnType["beginQueueOperation"]; +type FinishQueueOperation = ReturnType["finishQueueOperation"]; + +async function runQueueMutation( + identity: QueueSessionIdentity | null, + beginQueueOperation: BeginQueueOperation, + finishQueueOperation: FinishQueueOperation, + refetch: QueueRefetch, + mutate: (identity: QueueSessionIdentity) => Promise, +): Promise { + if (!identity) return; + const { session_id: sessionId, session_incarnation_id: incarnationId } = identity; + const token = beginQueueOperation(sessionId, incarnationId); + if (!token) return; + let mutationError: unknown; + try { + try { + await mutate(identity); + } catch (error) { + mutationError = error; + } + try { + await refetch(sessionId, token); + } catch (reconcileError) { + if (mutationError) throw mutationError; + throw reconcileError; + } + if (mutationError) throw mutationError; + } finally { + finishQueueOperation(sessionId, token); + } +} + function useSendNowAction( - sessionId: string | null, - setQueueLoading: ReturnType["setQueueLoading"], - refetch: (sid: string) => Promise, + identity: QueueSessionIdentity | null, + beginQueueOperation: BeginQueueOperation, + finishQueueOperation: FinishQueueOperation, + refetch: QueueRefetch, ) { return useCallback( - async (entryId: string) => { - if (!sessionId) return; - setQueueLoading(sessionId, true); - let mutationFailed = false; - let mutationError: unknown; - try { - try { - await sendQueuedNow({ - session_id: sessionId, - scope: "entry", - entry_id: entryId, - }); - } catch (err) { - mutationFailed = true; - mutationError = err; - } - - try { - await refetch(sessionId); - } catch (reconcileError) { - if (mutationFailed) throw mutationError; - throw reconcileError; - } - if (mutationFailed) throw mutationError; - } finally { - setQueueLoading(sessionId, false); - } - }, - [sessionId, refetch, setQueueLoading], + (entryId: string) => + runQueueMutation(identity, beginQueueOperation, finishQueueOperation, refetch, (current) => + sendQueuedNow({ ...current, scope: "entry", entry_id: entryId }), + ), + [beginQueueOperation, finishQueueOperation, identity, refetch], ); } -function useSetAutoRunAction( - sessionId: string | null, - setQueueLoading: ReturnType["setQueueLoading"], - refetch: (sid: string) => Promise, +type BooleanQueueMutation = (identity: QueueSessionIdentity, enabled: boolean) => Promise; + +function useBooleanQueueAction( + identity: QueueSessionIdentity | null, + beginQueueOperation: BeginQueueOperation, + finishQueueOperation: FinishQueueOperation, + refetch: QueueRefetch, + mutate: BooleanQueueMutation, ) { return useCallback( - async (enabled: boolean) => { - if (!sessionId) return; - setQueueLoading(sessionId, true); - let mutationFailed = false; - let mutationError: unknown; - try { - try { - await setQueueAutoRun(sessionId, enabled); - } catch (err) { - mutationFailed = true; - mutationError = err; - } - - try { - await refetch(sessionId); - } catch (reconcileError) { - if (mutationFailed) throw mutationError; - throw reconcileError; - } - if (mutationFailed) throw mutationError; - } finally { - setQueueLoading(sessionId, false); - } - }, - [sessionId, refetch, setQueueLoading], + (enabled: boolean) => + runQueueMutation(identity, beginQueueOperation, finishQueueOperation, refetch, (current) => + mutate(current, enabled), + ), + [beginQueueOperation, finishQueueOperation, identity, mutate, refetch], ); } -/** Fetches the authoritative queue snapshot and writes it into the slice, - * discarding a response that arrives after a newer request for the same - * session was already issued (out-of-order network resolution). */ -function useQueueRefetch( - setQueueEntries: ReturnType["setQueueEntries"], - setQueueLoading: ReturnType["setQueueLoading"], -) { - const refetchVersion = useRef>({}); - const invalidate = useCallback((sid: string) => { - refetchVersion.current[sid] = (refetchVersion.current[sid] ?? 0) + 1; - }, []); - const refetch = useCallback( - async (sid: string) => { - const version = (refetchVersion.current[sid] ?? 0) + 1; - refetchVersion.current[sid] = version; - try { - setQueueLoading(sid, true); - const status = await getQueueStatus(sid); - if (refetchVersion.current[sid] !== version) return; - setQueueEntries(sid, status.entries ?? [], { - count: status.count, - max: status.max, - mergeEnabled: status.merge_enabled ?? true, - autoRun: status.auto_run ?? true, - }); - } finally { - if (refetchVersion.current[sid] === version) setQueueLoading(sid, false); - } - }, - [setQueueEntries, setQueueLoading], - ); - return { refetch, invalidate }; -} - /** Build an action set bound to the supplied session + slice setters. */ function useQueueActions({ - sessionId, + identity, entries, setQueueEntries, removeQueueEntry, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, metaMax, metaMergeEnabled, metaAutoRun, }: QueueActionsArgs) { const { refetch, invalidate: invalidateRefetch } = useQueueRefetch( + identity, setQueueEntries, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, ); - const queue = useCallback( - async ({ - taskId, - content, - model, - planMode, - attachments, - entityReferences, - contextFilesMeta, - }: QueueMessageInput) => { - if (!sessionId) return; - setQueueLoading(sessionId, true); - try { - await queueMessage({ - session_id: sessionId, - task_id: taskId, - content, - model, - plan_mode: planMode, - attachments, - entity_references: entityReferences, - ...(contextFilesMeta ? { context_files: contextFilesMeta } : {}), - }); - await refetch(sessionId); - } finally { - setQueueLoading(sessionId, false); - } - }, - [sessionId, refetch, setQueueLoading], + const queue = useQueueAdmissionAction( + identity, + beginQueueOperation, + finishQueueOperation, + refetch, ); const clearAll = useClearAllAction({ - sessionId, + identity, setQueueEntries, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, metaMax, metaMergeEnabled, metaAutoRun, @@ -243,20 +177,41 @@ function useQueueActions({ invalidateRefetch, }); - const sendEntryNow = useSendNowAction(sessionId, setQueueLoading, refetch); - const setAutoRun = useSetAutoRunAction(sessionId, setQueueLoading, refetch); + const sendEntryNow = useSendNowAction( + identity, + beginQueueOperation, + finishQueueOperation, + refetch, + ); + const setAutoRun = useBooleanQueueAction( + identity, + beginQueueOperation, + finishQueueOperation, + refetch, + setQueueAutoRun, + ); + const setAutoMerge = useBooleanQueueAction( + identity, + beginQueueOperation, + finishQueueOperation, + refetch, + setQueueAutoMerge, + ); const { editEntry, removeEntry, mergeEntry } = useEntryMutations({ - sessionId, + identity, removeQueueEntry, refetch, + beginQueueOperation, + finishQueueOperation, }); const reorderEntries = useReorderEntriesAction({ - sessionId, + identity, entries, setQueueEntries, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, metaMax, metaMergeEnabled, metaAutoRun, @@ -269,6 +224,7 @@ function useQueueActions({ clearAll, sendEntryNow, setAutoRun, + setAutoMerge, editEntry, removeEntry, mergeEntry, @@ -277,33 +233,36 @@ function useQueueActions({ } type ReorderEntriesArgs = { - sessionId: string | null; + identity: QueueSessionIdentity | null; entries: QueuedMessage[]; setQueueEntries: ReturnType["setQueueEntries"]; - setQueueLoading: ReturnType["setQueueLoading"]; + beginQueueOperation: BeginQueueOperation; + finishQueueOperation: FinishQueueOperation; metaMax: number | undefined; metaMergeEnabled: boolean | undefined; metaAutoRun: boolean | undefined; - refetch: (sid: string) => Promise; + refetch: QueueRefetch; }; type ClearAllArgs = { - sessionId: string | null; + identity: QueueSessionIdentity | null; setQueueEntries: ReturnType["setQueueEntries"]; - setQueueLoading: ReturnType["setQueueLoading"]; + beginQueueOperation: BeginQueueOperation; + finishQueueOperation: FinishQueueOperation; metaMax: number | undefined; metaMergeEnabled: boolean | undefined; metaAutoRun: boolean | undefined; - refetch: (sid: string) => Promise; + refetch: QueueRefetch; invalidateRefetch: (sid: string) => void; }; /** Clears every pending entry: optimistic empty state, then the backend's * authoritative result. Reconcile errors preserve the mutation error. */ function useClearAllAction({ - sessionId, + identity, setQueueEntries, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, metaMax, metaMergeEnabled, metaAutoRun, @@ -311,44 +270,43 @@ function useClearAllAction({ invalidateRefetch, }: ClearAllArgs) { return useCallback(async () => { - if (!sessionId) return; - setQueueLoading(sessionId, true); + if (!identity) return; + const { session_id: sessionId, session_incarnation_id: incarnationId } = identity; + const token = beginQueueOperation(sessionId, incarnationId); + if (!token) return; let mutationFailed = false; let mutationError: unknown; try { try { - await clearQueue(sessionId); - // Invalidate any pre-clear status request before publishing the - // optimistic empty state. The explicit refetch below then owns the - // newest version and replaces it with the backend's exact result. + await clearQueue(identity); invalidateRefetch(sessionId); setQueueEntries(sessionId, [], { count: 0, max: metaMax ?? 0, mergeEnabled: metaMergeEnabled ?? true, autoRun: metaAutoRun ?? true, + taskId: identity.task_id, + sessionIncarnationId: incarnationId, }); } catch (err) { mutationFailed = true; mutationError = err; } - try { - await refetch(sessionId); + await refetch(sessionId, token); } catch (reconcileError) { - // Preserve the mutation error when both operations fail; it best - // explains why the requested action did not complete. if (mutationFailed) throw mutationError; throw reconcileError; } if (mutationFailed) throw mutationError; } finally { - setQueueLoading(sessionId, false); + finishQueueOperation(sessionId, token); } }, [ - sessionId, + beginQueueOperation, + finishQueueOperation, + identity, setQueueEntries, - setQueueLoading, metaMax, metaMergeEnabled, metaAutoRun, @@ -362,10 +320,11 @@ function useClearAllAction({ * (drain/remove/merge raced the drag) is refetched and rethrown as * QueueReorderError so the panel can swallow it without a toast. */ function useReorderEntriesAction({ - sessionId, + identity, entries, setQueueEntries, - setQueueLoading, + beginQueueOperation, + finishQueueOperation, metaMax, metaMergeEnabled, metaAutoRun, @@ -373,9 +332,10 @@ function useReorderEntriesAction({ }: ReorderEntriesArgs) { return useCallback( async (orderedIds: string[]) => { - if (!sessionId) return; - // Optimistic: reflect the drop immediately; the refetch below reconciles - // to the backend's authoritative order and positions. + if (!identity) return; + const { session_id: sessionId, session_incarnation_id: incarnationId } = identity; + const token = beginQueueOperation(sessionId, incarnationId); + if (!token) return; const byId = new Map(entries.map((entry) => [entry.id, entry])); const reordered = orderedIds .map((id) => byId.get(id)) @@ -388,128 +348,194 @@ function useReorderEntriesAction({ autoRun: metaAutoRun ?? true, }); } - setQueueLoading(sessionId, true); let mutationFailed = false; let mutationError: unknown; try { try { - await reorderQueuedEntries({ session_id: sessionId, ordered_ids: orderedIds }); + await reorderQueuedEntries({ ...identity, ordered_ids: orderedIds }); } catch (err) { mutationFailed = true; mutationError = err; } - try { - await refetch(sessionId); + await refetch(sessionId, token); } catch (reconcileError) { - // Preserve the mutation error when both operations fail; it best - // explains why the requested order did not land. if (mutationFailed) throw mutationError; throw reconcileError; } if (mutationFailed) throw mutationError; } finally { - setQueueLoading(sessionId, false); + finishQueueOperation(sessionId, token); } }, [ - sessionId, + beginQueueOperation, entries, - setQueueEntries, - setQueueLoading, + finishQueueOperation, + identity, + metaAutoRun, metaMax, metaMergeEnabled, - metaAutoRun, refetch, + setQueueEntries, ], ); } type EntryMutationsArgs = { - sessionId: string | null; + identity: QueueSessionIdentity | null; removeQueueEntry: ReturnType["removeQueueEntry"]; - refetch: (sid: string) => Promise; + beginQueueOperation: BeginQueueOperation; + finishQueueOperation: FinishQueueOperation; + refetch: QueueRefetch; }; /** Entry-level mutations (edit / remove / merge) that refetch on success and * resync on a drain race (QueueEntryNotFoundError). */ -function useEntryMutations({ sessionId, removeQueueEntry, refetch }: EntryMutationsArgs) { +function useEntryMutations({ + identity, + removeQueueEntry, + beginQueueOperation, + finishQueueOperation, + refetch, +}: EntryMutationsArgs) { + const run = useCallback( + async (operation: (token: QueueOperationToken) => Promise) => { + if (!identity) return; + const token = beginQueueOperation(identity.session_id, identity.session_incarnation_id); + if (!token) return; + try { + await operation(token); + } finally { + finishQueueOperation(identity.session_id, token); + } + }, + [beginQueueOperation, finishQueueOperation, identity], + ); const editEntry = useCallback( async ( entryId: string, content: string, attachments?: MessageAttachment[], entityReferences: EntityReference[] = [], - ) => { - if (!sessionId) return; - try { - await updateQueuedMessage({ - session_id: sessionId, - entry_id: entryId, - content, - attachments, - entity_references: entityReferences, - }); - await refetch(sessionId); - } catch (err) { - if (err instanceof QueueEntryNotFoundError) { - await refetch(sessionId); + ) => + run(async (token) => { + if (!identity) return; + try { + await updateQueuedMessage({ + ...identity, + entry_id: entryId, + content, + attachments, + entity_references: entityReferences, + }); + await refetch(identity.session_id, token); + } catch (err) { + if (err instanceof QueueEntryNotFoundError) { + await refetch(identity.session_id, token); + } + throw err; } - throw err; - } - }, - [sessionId, refetch], + }), + [identity, refetch, run], ); - const removeEntry = useCallback( - async (entryId: string) => { - if (!sessionId) return; - removeQueueEntry(sessionId, entryId); - let mutationFailed = false; - let mutationError: unknown; - try { - await removeQueuedEntry({ session_id: sessionId, entry_id: entryId }); - } catch (err) { - mutationFailed = true; - mutationError = err; - } - - try { - await refetch(sessionId); - } catch (reconcileError) { + async (entryId: string) => + run(async (token) => { + if (!identity) return; + removeQueueEntry(identity.session_id, entryId); + let mutationFailed = false; + let mutationError: unknown; + try { + await removeQueuedEntry({ ...identity, entry_id: entryId }); + } catch (err) { + mutationFailed = true; + mutationError = err; + } + try { + await refetch(identity.session_id, token); + } catch (reconcileError) { + if (mutationFailed && !(mutationError instanceof QueueEntryNotFoundError)) { + throw mutationError; + } + throw reconcileError; + } if (mutationFailed && !(mutationError instanceof QueueEntryNotFoundError)) { throw mutationError; } - throw reconcileError; - } - - // A drain/remove race is already reflected by the authoritative status, - // so it is a successful UI outcome. Other failures restore state above - // and remain actionable to the caller. - if (mutationFailed && !(mutationError instanceof QueueEntryNotFoundError)) { - throw mutationError; - } - }, - [sessionId, refetch, removeQueueEntry], + }), + [identity, refetch, removeQueueEntry, run], ); - const mergeEntry = useCallback( - async (entryId: string, userId?: string) => { - if (!sessionId) return; - try { - await mergeQueuedEntry({ session_id: sessionId, entry_id: entryId, user_id: userId }); - await refetch(sessionId); - } catch (err) { - if (err instanceof QueueEntryNotFoundError) { - await refetch(sessionId); + async (entryId: string, userId?: string) => + run(async (token) => { + if (!identity) return; + try { + await mergeQueuedEntry({ ...identity, entry_id: entryId, user_id: userId }); + await refetch(identity.session_id, token); + } catch (err) { + if (err instanceof QueueEntryNotFoundError) { + await refetch(identity.session_id, token); + } + throw err; } - throw err; - } + }), + [identity, refetch, run], + ); + return { editEntry, removeEntry, mergeEntry }; +} + +function useCurrentQueueIdentity( + sessionId: string | null, + taskSession: ReturnType["taskSession"], +) { + return useMemo(() => { + if (!sessionId || !taskSession?.task_id || !taskSession.queue_incarnation_id) return null; + return { + task_id: taskSession.task_id, + session_id: sessionId, + session_incarnation_id: taskSession.queue_incarnation_id, + }; + }, [sessionId, taskSession?.task_id, taskSession?.queue_incarnation_id]); +} + +function useQueueRefresh( + sessionId: string | null, + connectionStatus: string, + refetch: QueueRefetch, +) { + useEffect(() => { + if (!sessionId || connectionStatus !== "connected") return; + void refetch(sessionId).catch((error) => { + console.error("Failed to fetch queue status:", error); + }); + }, [sessionId, connectionStatus, refetch]); + useForegroundRefresh( + () => { + if (!sessionId || connectionStatus !== "connected") return; + return refetch(sessionId).catch((error) => { + console.error("Failed to fetch queue status after foreground refresh:", error); + }); }, + Boolean(sessionId), + sessionId, + ); + return useCallback( + () => (sessionId ? refetch(sessionId) : Promise.resolve()), [sessionId, refetch], ); +} - return { editEntry, removeEntry, mergeEntry }; +function queueSummary(meta: QueueMeta | undefined, entries: QueuedMessage[]) { + return { + count: meta?.count ?? entries.length, + max: meta?.max ?? 0, + isFull: meta ? meta.count >= meta.max && meta.max > 0 : false, + mergeEnabled: meta?.mergeEnabled ?? true, + autoRun: meta?.autoRun ?? true, + autoMerge: meta?.autoMergeEnabled ?? false, + autoMergeAvailable: meta?.autoMergeAvailable ?? false, + }; } /** @@ -522,67 +548,44 @@ function useEntryMutations({ sessionId, removeQueueEntry, refetch }: EntryMutati */ export function useQueue(sessionId: string | null) { const state = useQueueState(sessionId); - const { entries, meta, isLoading, cancellationPending } = state; + const { entries, meta, isLoading, cancellationPending, taskSession } = state; + const identity = useCurrentQueueIdentity(sessionId, taskSession); const connectionStatus = useAppStore((appState) => appState.connection.status); const { refetch, queue, clearAll, sendEntryNow, + setAutoMerge, setAutoRun, editEntry, removeEntry, mergeEntry, reorderEntries, } = useQueueActions({ - sessionId, + identity, entries, setQueueEntries: state.setQueueEntries, removeQueueEntry: state.removeQueueEntry, - setQueueLoading: state.setQueueLoading, + beginQueueOperation: state.beginQueueOperation, + finishQueueOperation: state.finishQueueOperation, metaMax: meta?.max, metaMergeEnabled: meta?.mergeEnabled, metaAutoRun: meta?.autoRun, }); - useEffect(() => { - if (!sessionId) return; - if (connectionStatus !== "connected") return; - void refetch(sessionId).catch((err) => { - console.error("Failed to fetch queue status:", err); - }); - }, [sessionId, connectionStatus, refetch]); - - useForegroundRefresh( - () => { - if (!sessionId) return; - if (connectionStatus !== "connected") return; - return refetch(sessionId).catch((err) => { - console.error("Failed to fetch queue status after foreground refresh:", err); - }); - }, - Boolean(sessionId), - sessionId, - ); - - const refetchBound = useCallback( - () => (sessionId ? refetch(sessionId) : Promise.resolve()), - [sessionId, refetch], - ); + const refetchBound = useQueueRefresh(sessionId, connectionStatus, refetch); return { entries, - count: meta?.count ?? entries.length, - max: meta?.max ?? 0, - isFull: meta ? meta.count >= meta.max && meta.max > 0 : false, - mergeEnabled: meta?.mergeEnabled ?? true, - autoRun: meta?.autoRun ?? true, + ...queueSummary(meta, entries), isLoading, cancellationPending, queue, clearAll, sendEntryNow, setAutoRun, + setAutoMerge, editEntry, removeEntry, mergeEntry, diff --git a/apps/web/hooks/domains/session/use-request-changes-walkthrough.test.ts b/apps/web/hooks/domains/session/use-request-changes-walkthrough.test.ts index cb176f28cd..6f177badf6 100644 --- a/apps/web/hooks/domains/session/use-request-changes-walkthrough.test.ts +++ b/apps/web/hooks/domains/session/use-request-changes-walkthrough.test.ts @@ -36,8 +36,11 @@ function storeState(sessionState: string, planMode = false, foregroundActivity?: return { taskSessions: { items: { - "session-1": { state: sessionState, foreground_activity: foregroundActivity }, - "other-session": { state: "RUNNING", foreground_activity: "generating" }, + "session-1": { + state: sessionState, + foreground_activity: foregroundActivity, + queue_incarnation_id: "inc-1", + }, }, }, chatInput: { planModeBySessionId: { "session-1": planMode } }, diff --git a/apps/web/hooks/domains/session/use-request-changes-walkthrough.ts b/apps/web/hooks/domains/session/use-request-changes-walkthrough.ts index 16309b9fa0..42a3794153 100644 --- a/apps/web/hooks/domains/session/use-request-changes-walkthrough.ts +++ b/apps/web/hooks/domains/session/use-request-changes-walkthrough.ts @@ -39,11 +39,13 @@ async function loadChangesWalkthroughPromptTemplate(): Promise { async function queueWalkthroughRequest(params: { taskId: string; sessionId: string; + sessionIncarnationId: string; content: string; planModeEnabled: boolean; }) { await queueMessage({ session_id: params.sessionId, + session_incarnation_id: params.sessionIncarnationId, task_id: params.taskId, content: params.content, ...planModePayload(params.planModeEnabled), @@ -108,9 +110,13 @@ export function useRequestChangesWalkthrough({ const template = await loadChangesWalkthroughPromptTemplate(); const content = buildChangesWalkthroughPrompt(template); if (inputMode === "queue") { + if (!activeSession?.queue_incarnation_id) { + throw new Error("Session is not available for input"); + } await queueWalkthroughRequest({ taskId, sessionId, + sessionIncarnationId: activeSession.queue_incarnation_id, content, planModeEnabled, }); diff --git a/apps/web/lib/api/domains/queue-api.test.ts b/apps/web/lib/api/domains/queue-api.test.ts index b4543b0963..e92a845d11 100644 --- a/apps/web/lib/api/domains/queue-api.test.ts +++ b/apps/web/lib/api/domains/queue-api.test.ts @@ -2,6 +2,8 @@ import { beforeEach, describe, it, expect, vi } from "vitest"; import type { EntityReference } from "@/lib/types/entity-reference"; const getWebSocketClientMock = vi.hoisted(() => vi.fn()); +const SESSION_ID = `session-1`; +const INCARNATION_ID = `incarnation-1`; vi.mock("@/lib/ws/connection", () => ({ getWebSocketClient: getWebSocketClientMock, @@ -133,14 +135,16 @@ describe("queue reference payloads", () => { getWebSocketClientMock.mockReturnValue({ request }); await queueMessage({ - session_id: "session-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, task_id: "task-1", content: "queued reference", entity_references: [reference], }); expect(request).toHaveBeenCalledWith("message.queue.add", { - session_id: "session-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, task_id: "task-1", content: "queued reference", entity_references: [reference], @@ -152,14 +156,16 @@ describe("queue reference payloads", () => { getWebSocketClientMock.mockReturnValue({ request }); await queueMessage({ - session_id: "session-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, task_id: "task-1", content: "queued context", context_files: [{ path: "src/components", name: "components", is_directory: true }], }); expect(request).toHaveBeenCalledWith("message.queue.add", { - session_id: "session-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, task_id: "task-1", content: "queued context", context_files: [{ path: "src/components", name: "components", is_directory: true }], @@ -171,13 +177,17 @@ describe("queue reference payloads", () => { getWebSocketClientMock.mockReturnValue({ request }); await updateQueuedMessage({ - session_id: "session-1", + session_id: SESSION_ID, + task_id: "task-1", + session_incarnation_id: INCARNATION_ID, entry_id: "q-1", content: "reference removed", - } as never); + }); expect(request).toHaveBeenCalledWith("message.queue.update", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-1", content: "reference removed", entity_references: [], @@ -189,14 +199,18 @@ describe("queue reference payloads", () => { getWebSocketClientMock.mockReturnValue({ request }); await updateQueuedMessage({ - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-1", content: "reference kept", entity_references: [reference], }); expect(request).toHaveBeenCalledWith("message.queue.update", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-1", content: "reference kept", entity_references: [reference], @@ -210,13 +224,17 @@ describe("mergeQueuedEntry", () => { getWebSocketClientMock.mockReturnValue({ request }); await mergeQueuedEntry({ - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-b", user_id: "user-1", }); expect(request).toHaveBeenCalledWith("message.queue.merge", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-b", user_id: "user-1", }); @@ -226,10 +244,17 @@ describe("mergeQueuedEntry", () => { const request = vi.fn().mockResolvedValue({ entry_id: "q-a" }); getWebSocketClientMock.mockReturnValue({ request }); - await mergeQueuedEntry({ session_id: "session-1", entry_id: "q-b" }); + await mergeQueuedEntry({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + entry_id: "q-b", + }); expect(request).toHaveBeenCalledWith("message.queue.merge", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, entry_id: "q-b", }); }); @@ -241,7 +266,12 @@ describe("mergeQueuedEntry", () => { getWebSocketClientMock.mockReturnValue({ request }); await expect( - mergeQueuedEntry({ session_id: "session-1", entry_id: "q-b" }), + mergeQueuedEntry({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + entry_id: "q-b", + }), ).rejects.toBeInstanceOf(QueueEntryNotFoundError); }); @@ -253,7 +283,12 @@ describe("mergeQueuedEntry", () => { getWebSocketClientMock.mockReturnValue({ request }); await expect( - mergeQueuedEntry({ session_id: "session-1", entry_id: "q-b" }), + mergeQueuedEntry({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + entry_id: "q-b", + }), ).rejects.toBeInstanceOf(MergeReferenceOverflowError); }); }); @@ -261,20 +296,24 @@ describe("mergeQueuedEntry", () => { describe("sendQueuedNow", () => { it("sends an exact entry scope", async () => { const request = vi.fn().mockResolvedValue({ - session_id: "session-1", + session_id: SESSION_ID, dispatched: true, sent_count: 1, }); getWebSocketClientMock.mockReturnValue({ request }); await sendQueuedNow({ - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, scope: "entry", entry_id: "q-2", }); expect(request).toHaveBeenCalledWith("message.queue.send_now", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, scope: "entry", entry_id: "q-2", }); @@ -282,16 +321,23 @@ describe("sendQueuedNow", () => { it("omits entry_id for an all scope snapshot", async () => { const request = vi.fn().mockResolvedValue({ - session_id: "session-1", + session_id: SESSION_ID, dispatched: true, sent_count: 3, }); getWebSocketClientMock.mockReturnValue({ request }); - await sendQueuedNow({ session_id: "session-1", scope: "all" }); + await sendQueuedNow({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + scope: "all", + }); expect(request).toHaveBeenCalledWith("message.queue.send_now", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, scope: "all", }); }); @@ -303,19 +349,31 @@ describe("sendQueuedNow", () => { }); getWebSocketClientMock.mockReturnValue({ request }); - await expect(sendQueuedNow({ session_id: "session-1", scope: "all" })).rejects.toMatchObject({ + await expect( + sendQueuedNow({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + scope: "all", + }), + ).rejects.toMatchObject({ code: "send_now_conflict", }); - await expect(sendQueuedNow({ session_id: "session-1", scope: "all" })).rejects.toBeInstanceOf( - QueueSendNowError, - ); + await expect( + sendQueuedNow({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + scope: "all", + }), + ).rejects.toBeInstanceOf(QueueSendNowError); }); }); describe("setQueueAutoRun", () => { it("sets the per-session queue policy through its exact WebSocket action", async () => { const request = vi.fn().mockResolvedValue({ - session_id: "session-1", + session_id: SESSION_ID, auto_run: false, dispatched: false, }); @@ -323,17 +381,68 @@ describe("setQueueAutoRun", () => { const setQueueAutoRun = ( queueApi as typeof queueApi & { setQueueAutoRun?: ( - sessionId: string, + identity: { + task_id: string; + session_id: string; + session_incarnation_id: string; + }, enabled: boolean, ) => Promise<{ session_id: string; auto_run: boolean; dispatched: boolean }>; } ).setQueueAutoRun; expect(setQueueAutoRun).toBeTypeOf("function"); - await setQueueAutoRun!("session-1", false); + await setQueueAutoRun!( + { task_id: "task-1", session_id: SESSION_ID, session_incarnation_id: INCARNATION_ID }, + false, + ); expect(request).toHaveBeenCalledWith("message.queue.auto_run.set", { - session_id: "session-1", + session_id: SESSION_ID, + task_id: "task-1", + session_incarnation_id: INCARNATION_ID, + enabled: false, + }); + }); +}); +describe("setQueueAutoMerge", () => { + it("sets the session override with the complete immutable identity", async () => { + const request = vi.fn().mockResolvedValue({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + auto_merge_enabled: false, + auto_merge_source: "session", + auto_merge_revision: 1, + }); + getWebSocketClientMock.mockReturnValue({ request }); + const setQueueAutoMerge = ( + queueApi as typeof queueApi & { + setQueueAutoMerge?: ( + identity: { + task_id: string; + session_id: string; + session_incarnation_id: string; + }, + enabled: boolean, + ) => Promise; + } + ).setQueueAutoMerge; + + expect(setQueueAutoMerge).toBeTypeOf("function"); + await setQueueAutoMerge!( + { + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + }, + false, + ); + + expect(request).toHaveBeenCalledWith("message.queue.auto_merge.set", { + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, enabled: false, }); }); @@ -341,16 +450,20 @@ describe("setQueueAutoRun", () => { describe("reorderQueuedEntries", () => { it("sends the full ordered id list through message.queue.reorder", async () => { - const request = vi.fn().mockResolvedValue({ session_id: "session-1", reordered: 3 }); + const request = vi.fn().mockResolvedValue({ session_id: SESSION_ID, reordered: 3 }); getWebSocketClientMock.mockReturnValue({ request }); await reorderQueuedEntries({ - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, ordered_ids: ["q-c", "q-a", "q-b"], }); expect(request).toHaveBeenCalledWith("message.queue.reorder", { - session_id: "session-1", + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, ordered_ids: ["q-c", "q-a", "q-b"], }); }); @@ -363,7 +476,12 @@ describe("reorderQueuedEntries", () => { getWebSocketClientMock.mockReturnValue({ request }); await expect( - reorderQueuedEntries({ session_id: "session-1", ordered_ids: ["q-a"] }), + reorderQueuedEntries({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + ordered_ids: ["q-a"], + }), ).rejects.toBeInstanceOf(QueueReorderError); }); @@ -372,7 +490,12 @@ describe("reorderQueuedEntries", () => { getWebSocketClientMock.mockReturnValue({ request }); await expect( - reorderQueuedEntries({ session_id: "session-1", ordered_ids: ["q-a"] }), + reorderQueuedEntries({ + task_id: "task-1", + session_id: SESSION_ID, + session_incarnation_id: INCARNATION_ID, + ordered_ids: ["q-a"], + }), ).rejects.toBeInstanceOf(QueueEntryNotFoundError); }); }); diff --git a/apps/web/lib/api/domains/queue-api.ts b/apps/web/lib/api/domains/queue-api.ts index 48547e7a2f..f68de9f23f 100644 --- a/apps/web/lib/api/domains/queue-api.ts +++ b/apps/web/lib/api/domains/queue-api.ts @@ -136,8 +136,15 @@ export function rethrowQueueError(err: unknown): never { throw err instanceof Error ? err : new Error(String(err)); } +export type QueueSessionIdentity = { + task_id: string; + session_id: string; + session_incarnation_id: string; +}; + export type QueueMessageParams = { session_id: string; + session_incarnation_id: string; task_id: string; content: string; model?: string; @@ -170,25 +177,26 @@ export async function queueMessage(params: QueueMessageParams): Promise { +export async function clearQueue(identity: QueueSessionIdentity): Promise<{ removed: number }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); } - return client.request<{ removed: number }>("message.queue.cancel", { session_id: sessionId }); + return client.request<{ removed: number }>("message.queue.cancel", identity); } /** Dispatch one queued entry now when the session is ready for input. */ -export async function drainQueuedMessage(sessionId: string): Promise<{ drained: boolean }> { +export async function drainQueuedMessage( + identity: QueueSessionIdentity, +): Promise<{ drained: boolean }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); } - return client.request<{ drained: boolean }>("message.queue.drain", { session_id: sessionId }); + return client.request<{ drained: boolean }>("message.queue.drain", identity); } -export type SendQueuedNowParams = { - session_id: string; +export type SendQueuedNowParams = QueueSessionIdentity & { scope: "entry" | "all"; entry_id?: string; }; @@ -214,7 +222,7 @@ export async function sendQueuedNow( /** Persist the per-session queue Auto-run policy and start the queue when enabling it. */ export async function setQueueAutoRun( - sessionId: string, + identity: QueueSessionIdentity, enabled: boolean, ): Promise<{ session_id: string; auto_run: boolean; dispatched: boolean }> { const client = getWebSocketClient(); @@ -223,28 +231,48 @@ export async function setQueueAutoRun( } return client.request<{ session_id: string; auto_run: boolean; dispatched: boolean }>( "message.queue.auto_run.set", - { session_id: sessionId, enabled }, + { ...identity, enabled }, ); } +export type QueueAutoMergePolicyResponse = QueueSessionIdentity & { + auto_merge_enabled: boolean; + auto_merge_source: "global" | "session"; + auto_merge_revision: number; +}; -/** Fetch the full queue snapshot (entries + capacity). */ -export async function getQueueStatus(sessionId: string): Promise { +/** Persist an explicit automatic-merge policy for one immutable session. */ +export async function setQueueAutoMerge( + identity: QueueSessionIdentity, + enabled: boolean, +): Promise { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); } - return client.request("message.queue.get", { session_id: sessionId }); + return client.request("message.queue.auto_merge.set", { + ...identity, + enabled, + }); +} + +/** Fetch the full queue snapshot for one immutable session identity. */ +export async function getQueueStatus(identity: QueueSessionIdentity): Promise { + const client = getWebSocketClient(); + if (!client) { + throw new Error(WS_CLIENT_UNAVAILABLE); + } + return client.request("message.queue.get", identity); } /** Append content onto the tail entry when the same caller authored it; otherwise insert a new entry. */ -export async function appendToQueue(params: { - session_id: string; - task_id: string; - content: string; - model?: string; - plan_mode?: boolean; - user_id?: string; -}): Promise<{ entry_id: string; was_append: boolean }> { +export async function appendToQueue( + params: QueueSessionIdentity & { + content: string; + model?: string; + plan_mode?: boolean; + user_id?: string; + }, +): Promise<{ entry_id: string; was_append: boolean }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); @@ -260,22 +288,23 @@ export async function appendToQueue(params: { } /** Replace the content/attachments of a queued entry. Throws QueueEntryNotFoundError if drained. */ -export async function updateQueuedMessage(params: { - session_id: string; - entry_id: string; - content: string; - attachments?: Array<{ - type: string; - data?: string; - attachment_id?: string; - mime_type: string; - name?: string; - size_bytes?: number; - delivery_mode?: "prompt" | "path"; - }>; - entity_references: EntityReference[]; - user_id?: string; -}): Promise<{ entry_id: string }> { +export async function updateQueuedMessage( + params: QueueSessionIdentity & { + entry_id: string; + content: string; + attachments?: Array<{ + type: string; + data?: string; + attachment_id?: string; + mime_type: string; + name?: string; + size_bytes?: number; + delivery_mode?: "prompt" | "path"; + }>; + entity_references?: EntityReference[]; + user_id?: string; + }, +): Promise<{ entry_id: string }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); @@ -291,10 +320,9 @@ export async function updateQueuedMessage(params: { } /** Remove a single queued entry by id. Throws QueueEntryNotFoundError if drained. */ -export async function removeQueuedEntry(params: { - session_id: string; - entry_id: string; -}): Promise<{ entry_id: string }> { +export async function removeQueuedEntry( + params: QueueSessionIdentity & { entry_id: string }, +): Promise<{ entry_id: string }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); @@ -307,11 +335,9 @@ export async function removeQueuedEntry(params: { } /** Fold a queued entry into the entry directly above it. Throws QueueEntryNotFoundError if drained. */ -export async function mergeQueuedEntry(params: { - session_id: string; - entry_id: string; - user_id?: string; -}): Promise<{ entry_id: string }> { +export async function mergeQueuedEntry( + params: QueueSessionIdentity & { entry_id: string; user_id?: string }, +): Promise<{ entry_id: string }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); @@ -326,10 +352,9 @@ export async function mergeQueuedEntry(params: { /** Rewrite the visible pending order of a session's queue. Throws * QueueReorderError when the queue changed since the client's snapshot; the * reorder was rejected atomically and the caller should refetch. */ -export async function reorderQueuedEntries(params: { - session_id: string; - ordered_ids: string[]; -}): Promise<{ session_id: string; reordered: number }> { +export async function reorderQueuedEntries( + params: QueueSessionIdentity & { ordered_ids: string[] }, +): Promise<{ session_id: string; reordered: number }> { const client = getWebSocketClient(); if (!client) { throw new Error(WS_CLIENT_UNAVAILABLE); diff --git a/apps/web/lib/state/app-state-types.ts b/apps/web/lib/state/app-state-types.ts index 552a1cdb08..eceb33fd8d 100644 --- a/apps/web/lib/state/app-state-types.ts +++ b/apps/web/lib/state/app-state-types.ts @@ -88,6 +88,12 @@ import type { TodoEntry, UserShellInfo, } from "./slices/session-runtime/types"; +import type { + QueueMeta, + QueueMetaUpdateOptions, + QueueOperationToken, + QueuedMessage, +} from "./slices/session/types"; // Combined AppState type export type AppState = KanbanSlice & { // Workspace slice @@ -526,11 +532,16 @@ export type AppState = KanbanSlice & { // Queue actions setQueueEntries: ( sessionId: string, - entries: import("./slices/session/types").QueuedMessage[], - meta: import("./slices/session/types").QueueMeta, + entries: QueuedMessage[], + meta: QueueMeta, + options?: QueueMetaUpdateOptions, ) => void; removeQueueEntry: (sessionId: string, entryId: string) => void; - setQueueLoading: (sessionId: string, loading: boolean) => void; + beginQueueOperation: ( + sessionId: string, + sessionIncarnationId: string, + ) => QueueOperationToken | null; + finishQueueOperation: (sessionId: string, token: QueueOperationToken) => void; clearQueueStatus: (sessionId: string) => void; // Available commands actions setAvailableCommands: (sessionId: string, commands: AvailableCommand[]) => void; diff --git a/apps/web/lib/state/slices/session/remove-task-session.test.ts b/apps/web/lib/state/slices/session/remove-task-session.test.ts index f3d801c466..6f63cb092c 100644 --- a/apps/web/lib/state/slices/session/remove-task-session.test.ts +++ b/apps/web/lib/state/slices/session/remove-task-session.test.ts @@ -23,6 +23,7 @@ function makeStore() { const TASK_ID = "task-1"; const SESSION_ID = "session-1"; +const INCARNATION_ID = `incarnation-1`; describe("removeTaskSession cleanup cascade", () => { let store: ReturnType; @@ -86,4 +87,41 @@ describe("removeTaskSession cleanup cascade", () => { expect(after.shell.outputs["env-1"]).toBeUndefined(); expect(after.environmentIdBySessionId[SESSION_ID]).toBeUndefined(); }); + + it("clears queue metadata and protects a replacement operation token", () => { + store.setState((draft) => { + draft.taskSessions.items[SESSION_ID] = { + id: SESSION_ID, + task_id: TASK_ID, + queue_incarnation_id: INCARNATION_ID, + } as never; + }); + expect(store.getState().beginQueueOperation(SESSION_ID, "stale-incarnation")).toBeNull(); + const first = store.getState().beginQueueOperation(SESSION_ID, INCARNATION_ID); + expect(first).not.toBeNull(); + expect(store.getState().beginQueueOperation(SESSION_ID, INCARNATION_ID)).toBeNull(); + store.getState().setQueueEntries(SESSION_ID, [], { + count: 0, + max: 10, + autoRun: true, + mergeEnabled: true, + sessionIncarnationId: INCARNATION_ID, + }); + + store.getState().removeTaskSession(TASK_ID, SESSION_ID); + expect(store.getState().queue.metaBySessionId[SESSION_ID]).toBeUndefined(); + expect(store.getState().queue.activeOperationBySessionId[SESSION_ID]).toBeUndefined(); + + store.setState((draft) => { + draft.taskSessions.items[SESSION_ID] = { + id: SESSION_ID, + task_id: TASK_ID, + queue_incarnation_id: "incarnation-2", + } as never; + }); + const replacement = store.getState().beginQueueOperation(SESSION_ID, "incarnation-2"); + expect(replacement).not.toBeNull(); + store.getState().finishQueueOperation(SESSION_ID, first!); + expect(store.getState().queue.activeOperationBySessionId[SESSION_ID]).toEqual(replacement); + }); }); diff --git a/apps/web/lib/state/slices/session/session-slice.ts b/apps/web/lib/state/slices/session/session-slice.ts index b01a2ca8b0..d48ec821da 100644 --- a/apps/web/lib/state/slices/session/session-slice.ts +++ b/apps/web/lib/state/slices/session/session-slice.ts @@ -2,7 +2,7 @@ import type { StateCreator } from "zustand"; import { original } from "immer"; import type { Message, TaskSession } from "@/lib/types/http"; -import type { SessionSlice, SessionSliceState } from "./types"; +import type { QueueMeta, QueueOperationToken, SessionSlice, SessionSliceState } from "./types"; import { buildTurnActions, isSettledSessionState, parseTurnTimestamp } from "./turn-actions"; import { buildTaskSessionProjectionActions, @@ -214,7 +214,12 @@ export const defaultSessionState: SessionSliceState = { activeStepByTaskId: {}, lastSeenUpdatedAtByTaskId: {}, }, - queue: { bySessionId: {}, metaBySessionId: {}, isLoading: {} }, + queue: { + bySessionId: {}, + metaBySessionId: {}, + activeOperationBySessionId: {}, + nextOperationGeneration: 0, + }, }; type ImmerSet = Parameters[0]; @@ -534,6 +539,9 @@ function buildRemoveTaskSessionAction(set: ImmerSet) { ); } delete draft.pendingActionProjectionsBySessionId[sessionId]; + delete draft.queue.bySessionId[sessionId]; + delete draft.queue.metaBySessionId[sessionId]; + delete draft.queue.activeOperationBySessionId[sessionId]; // Drop the conversation history owned by this session. delete draft.messages.bySession[sessionId]; delete draft.messages.metaBySession[sessionId]; @@ -552,6 +560,23 @@ function buildRemoveTaskSessionAction(set: ImmerSet) { }); } +function resetQueueStateForReincarnation( + draft: Pick, + existing: TaskSession | undefined, + incoming: Pick, +): void { + if ( + !existing || + incoming.queue_incarnation_id === undefined || + existing.queue_incarnation_id === incoming.queue_incarnation_id + ) { + return; + } + delete draft.queue.bySessionId[incoming.id]; + delete draft.queue.metaBySessionId[incoming.id]; + delete draft.queue.activeOperationBySessionId[incoming.id]; +} + /** Build actions that reconcile complete session snapshots with partial live events. */ function buildTaskSessionReconciliationActions(set: ImmerSet) { return { @@ -563,6 +588,7 @@ function buildTaskSessionReconciliationActions(set: ImmerSet) { set((draft) => { const merged = sessions.map((session) => { const existing = draft.taskSessions.items[session.id]; + resetQueueStateForReincarnation(draft, existing, session); const snapshot = mergeTaskSessionSnapshot( existing, session, @@ -595,6 +621,7 @@ function buildTaskSessionReconciliationActions(set: ImmerSet) { epochs[session.id] = (epochs[session.id] ?? 0) + 1; } const existing = draft.taskSessions.items[session.id]; + resetQueueStateForReincarnation(draft, existing, session); if (!existing && draft.taskSessionsByTask.loadedByTaskId[taskId]) { // State events intentionally carry partial session rows. When one // introduces a new session, let useTaskSessions hydrate fields such @@ -626,6 +653,7 @@ function buildTaskSessionActions(set: ImmerSet) { setTaskSession: (session: Parameters[0]) => set((draft) => { const existingSession = draft.taskSessions.items[session.id]; + resetQueueStateForReincarnation(draft, existingSession, session); const mergedSession = mergeOrphanPendingActionProjection( draft.pendingActionProjectionsBySessionId, existingSession ? mergeTaskSession(existingSession, session) : session, @@ -664,6 +692,157 @@ function buildTaskSessionActions(set: ImmerSet) { }; } +type QueueMetaInput = Parameters[2]; + +function queueMetaIdentityMatches(currentIncarnationId: string | undefined, meta: QueueMetaInput) { + return !meta.sessionIncarnationId || currentIncarnationId === meta.sessionIncarnationId; +} + +function canCarryPreviousQueueMeta( + previous: QueueMeta | undefined, + currentIncarnationId: string | undefined, + meta: QueueMetaInput, +) { + return ( + previous !== undefined && + previous.sessionIncarnationId === currentIncarnationId && + (!meta.sessionIncarnationId || meta.sessionIncarnationId === previous.sessionIncarnationId) + ); +} + +function isStaleQueueSnapshot( + previous: QueueMeta | undefined, + meta: QueueMetaInput, + establishStatusEpoch: boolean, +) { + if ( + meta.statusEpoch !== undefined && + previous?.statusEpoch !== undefined && + meta.statusEpoch !== previous.statusEpoch + ) { + return !establishStatusEpoch; + } + return ( + meta.statusEpoch !== undefined && + meta.statusEpoch === previous?.statusEpoch && + meta.statusGeneration !== undefined && + previous.statusGeneration !== undefined && + meta.statusGeneration <= previous.statusGeneration + ); +} + +function shouldPreserveSessionPolicy(previous: QueueMeta | undefined, meta: QueueMetaInput) { + return ( + previous?.sessionIncarnationId === meta.sessionIncarnationId && + previous?.autoMergeSource === "session" && + meta.autoMergeSource === "global" + ); +} + +function shouldPreservePolicyRevision(previous: QueueMeta | undefined, meta: QueueMetaInput) { + return ( + previous?.sessionIncarnationId === meta.sessionIncarnationId && + previous?.autoMergeSource === meta.autoMergeSource && + previous?.autoMergeRevision !== undefined && + meta.autoMergeRevision !== undefined && + meta.autoMergeRevision < previous.autoMergeRevision + ); +} + +function copyAutoMergePolicy(target: QueueMetaInput, source: QueueMeta) { + target.autoMergeAvailable = source.autoMergeAvailable; + target.autoMergeEnabled = source.autoMergeEnabled; + target.autoMergeSource = source.autoMergeSource; + target.autoMergeRevision = source.autoMergeRevision; +} + +function resolveQueueMeta( + currentIncarnationId: string | undefined, + previous: QueueMeta | undefined, + meta: QueueMetaInput, + establishStatusEpoch: boolean, +): QueueMetaInput | null { + if (!queueMetaIdentityMatches(currentIncarnationId, meta)) return null; + const nextMeta = { ...meta }; + if (canCarryPreviousQueueMeta(previous, currentIncarnationId, meta)) { + Object.assign(nextMeta, previous, meta); + } + if (isStaleQueueSnapshot(previous, meta, establishStatusEpoch)) return null; + if ( + previous && + (shouldPreserveSessionPolicy(previous, meta) || shouldPreservePolicyRevision(previous, meta)) + ) { + copyAutoMergePolicy(nextMeta, previous); + } + return nextMeta; +} + +function buildQueueActions(set: ImmerSet) { + return { + setQueueEntries: ( + sessionId: Parameters[0], + entries: Parameters[1], + meta: QueueMetaInput, + options?: Parameters[3], + ) => + set((draft) => { + const nextMeta = resolveQueueMeta( + draft.taskSessions.items[sessionId]?.queue_incarnation_id, + draft.queue.metaBySessionId[sessionId], + meta, + options?.establishStatusEpoch === true, + ); + if (!nextMeta) return; + draft.queue.bySessionId[sessionId] = entries; + draft.queue.metaBySessionId[sessionId] = nextMeta; + }), + removeQueueEntry: (sessionId: string, entryId: string) => + set((draft) => { + const list = draft.queue.bySessionId[sessionId]; + if (!list) return; + draft.queue.bySessionId[sessionId] = list.filter((entry) => entry.id !== entryId); + const meta = draft.queue.metaBySessionId[sessionId]; + if (meta) meta.count = draft.queue.bySessionId[sessionId].length; + }), + beginQueueOperation: (sessionId: string, sessionIncarnationId: string) => { + let token: QueueOperationToken | null = null; + set((draft) => { + const session = draft.taskSessions.items[sessionId]; + if ( + session?.queue_incarnation_id !== sessionIncarnationId || + draft.queue.activeOperationBySessionId[sessionId] + ) { + return; + } + token = { + sessionIncarnationId, + generation: ++draft.queue.nextOperationGeneration, + }; + draft.queue.activeOperationBySessionId[sessionId] = token; + }); + return token; + }, + finishQueueOperation: (sessionId: string, token: QueueOperationToken) => + set((draft) => { + const current = draft.queue.activeOperationBySessionId[sessionId]; + const session = draft.taskSessions.items[sessionId]; + if ( + current?.generation === token.generation && + current.sessionIncarnationId === token.sessionIncarnationId && + session?.queue_incarnation_id === token.sessionIncarnationId + ) { + delete draft.queue.activeOperationBySessionId[sessionId]; + } + }), + clearQueueStatus: (sessionId: string) => + set((draft) => { + delete draft.queue.bySessionId[sessionId]; + delete draft.queue.metaBySessionId[sessionId]; + delete draft.queue.activeOperationBySessionId[sessionId]; + }), + }; +} + /** Create the session slice, combining the default state with all session action builders wired to the store's Immer set/get. */ export const createSessionSlice: StateCreator< SessionSlice, @@ -704,29 +883,5 @@ export const createSessionSlice: StateCreator< }), ...buildTaskPlanActions(set, get), ...buildWalkthroughActions(set, get), - setQueueEntries: (sessionId, entries, meta) => - set((draft) => { - draft.queue.bySessionId[sessionId] = entries; - draft.queue.metaBySessionId[sessionId] = meta; - }), - removeQueueEntry: (sessionId, entryId) => - set((draft) => { - const list = draft.queue.bySessionId[sessionId]; - if (!list) return; - draft.queue.bySessionId[sessionId] = list.filter((entry) => entry.id !== entryId); - const meta = draft.queue.metaBySessionId[sessionId]; - if (meta) { - meta.count = draft.queue.bySessionId[sessionId].length; - } - }), - setQueueLoading: (sessionId, loading) => - set((draft) => { - draft.queue.isLoading[sessionId] = loading; - }), - clearQueueStatus: (sessionId) => - set((draft) => { - delete draft.queue.bySessionId[sessionId]; - delete draft.queue.metaBySessionId[sessionId]; - delete draft.queue.isLoading[sessionId]; - }), + ...buildQueueActions(set), }); diff --git a/apps/web/lib/state/slices/session/session-slice.upsert.test.ts b/apps/web/lib/state/slices/session/session-slice.upsert.test.ts index e364cdcfdc..32712d4a3b 100644 --- a/apps/web/lib/state/slices/session/session-slice.upsert.test.ts +++ b/apps/web/lib/state/slices/session/session-slice.upsert.test.ts @@ -557,8 +557,10 @@ describe("queue actions", () => { mergeEnabled: false, autoRun: false, }); - expect(store.getState().queue.metaBySessionId[SESSION_ID]?.mergeEnabled).toBe(false); - expect(store.getState().queue.metaBySessionId[SESSION_ID]?.autoRun).toBe(false); + expect(store.getState().queue.metaBySessionId[SESSION_ID]).toMatchObject({ + mergeEnabled: false, + autoRun: false, + }); }); it("removeQueueEntry drops a single entry by id and refreshes meta.count", () => { @@ -574,8 +576,7 @@ describe("queue actions", () => { store.getState().removeQueueEntry(SESSION_ID, "e2"); expect(store.getState().queue.bySessionId[SESSION_ID].map((e) => e.id)).toEqual(["e1", "e3"]); - expect(store.getState().queue.metaBySessionId[SESSION_ID].count).toBe(2); - expect(store.getState().queue.metaBySessionId[SESSION_ID].max).toBe(10); + expect(store.getState().queue.metaBySessionId[SESSION_ID]).toMatchObject({ count: 2, max: 10 }); }); it("removeQueueEntry is a no-op when the session has no entries", () => { @@ -583,6 +584,42 @@ describe("queue actions", () => { store.getState().removeQueueEntry(SESSION_ID, "missing"); expect(store.getState().queue.bySessionId[SESSION_ID]).toBeUndefined(); }); +}); + +describe("queue snapshot actions", () => { + it("requires an authoritative snapshot to replace the accepted status epoch", () => { + const store = makeStore(); + store.getState().setTaskSession(makeSession({ queue_incarnation_id: "incarnation-1" })); + const currentEntries = [makeEntry({ id: "current" })]; + store.getState().setQueueEntries(SESSION_ID, currentEntries, { + count: 1, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: "incarnation-1", + statusEpoch: "epoch-1", + }); + const replacementEntries = [makeEntry({ id: "replacement" })]; + const replacementMeta = { + count: 1, + max: 10, + mergeEnabled: false, + autoRun: false, + sessionIncarnationId: "incarnation-1", + statusEpoch: "epoch-2", + }; + + store.getState().setQueueEntries(SESSION_ID, replacementEntries, replacementMeta); + expect(store.getState().queue.bySessionId[SESSION_ID]).toEqual(currentEntries); + expect(store.getState().queue.metaBySessionId[SESSION_ID]?.statusEpoch).toBe("epoch-1"); + + store.getState().setQueueEntries(SESSION_ID, replacementEntries, replacementMeta, { + establishStatusEpoch: true, + }); + expect(store.getState().queue.bySessionId[SESSION_ID]).toEqual(replacementEntries); + expect(store.getState().queue.metaBySessionId[SESSION_ID]?.statusEpoch).toBe("epoch-2"); + }); it("clearQueueStatus removes both entries and meta", () => { const store = makeStore(); @@ -599,10 +636,6 @@ describe("queue actions", () => { expect(store.getState().queue.metaBySessionId[SESSION_ID]).toBeUndefined(); }); - // AC-75: a queued-but-unadmitted prompt is a backend-owned reading (task-05's - // D8 table decides whether parked_on_background_work stays true); the queue - // actions here are purely local UI state and must never locally clear the - // rendered parked/foreground affordance as a side effect. it("does not clear parked_on_background_work or foreground_activity as a side effect of queue actions", () => { const store = makeStore(); store.getState().upsertTaskSessionFromEvent( @@ -622,7 +655,6 @@ describe("queue actions", () => { autoRun: true, }); store.getState().removeQueueEntry(SESSION_ID, "missing-entry"); - store.getState().setQueueLoading(SESSION_ID, true); store.getState().clearQueueStatus(SESSION_ID); const session = store.getState().taskSessions.items[SESSION_ID]; @@ -631,3 +663,61 @@ describe("queue actions", () => { expect(session.parked_epoch).toBe(100); }); }); + +describe("queue state reincarnation", () => { + function seedOldIncarnation() { + const store = makeStore(); + store.getState().setTaskSession(makeSession({ queue_incarnation_id: "old-incarnation" })); + store.getState().setQueueEntries(SESSION_ID, [makeEntry()], { + count: 1, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: TASK_ID, + sessionIncarnationId: "old-incarnation", + }); + expect(store.getState().beginQueueOperation(SESSION_ID, "old-incarnation")).not.toBeNull(); + return store; + } + + function expectQueueStateCleared(store: ReturnType) { + expect(store.getState().queue.bySessionId[SESSION_ID]).toBeUndefined(); + expect(store.getState().queue.metaBySessionId[SESSION_ID]).toBeUndefined(); + expect(store.getState().queue.activeOperationBySessionId[SESSION_ID]).toBeUndefined(); + } + + it("clears stale queue state on an event upsert", () => { + const store = seedOldIncarnation(); + + store + .getState() + .upsertTaskSessionFromEvent( + TASK_ID, + makeSession({ queue_incarnation_id: "new-incarnation" }), + ); + + expectQueueStateCleared(store); + }); + + it("clears stale queue state on a session snapshot", () => { + const store = seedOldIncarnation(); + + store + .getState() + .setTaskSessionsForTask( + TASK_ID, + [makeSession({ queue_incarnation_id: "new-incarnation" })], + {}, + ); + + expectQueueStateCleared(store); + }); + + it("clears stale queue state on a direct session set", () => { + const store = seedOldIncarnation(); + + store.getState().setTaskSession(makeSession({ queue_incarnation_id: "new-incarnation" })); + + expectQueueStateCleared(store); + }); +}); diff --git a/apps/web/lib/state/slices/session/types.ts b/apps/web/lib/state/slices/session/types.ts index 35452fcd6a..71c453fcef 100644 --- a/apps/web/lib/state/slices/session/types.ts +++ b/apps/web/lib/state/slices/session/types.ts @@ -192,17 +192,42 @@ export type QueueMeta = { max: number; /** Backend-owned queue motion policy. Missing server state defaults to on. */ autoRun: boolean; - /** Mirrors the server's message queue merge_enabled setting; hides the - * "Merge with above" affordance without a separate settings fetch. */ + /** Mirrors the server's manual merge setting. */ mergeEnabled: boolean; + taskId?: string; + sessionIncarnationId?: string; + statusEpoch?: string; + statusGeneration?: number; + retiredStatusEpochs?: string[]; + autoMergeAvailable?: boolean; + autoMergeEnabled?: boolean; + autoMergeSource?: "global" | "session"; + autoMergeRevision?: number; +}; + +export type QueueMetaUpdateOptions = { + establishStatusEpoch?: boolean; }; export type QueueStatus = { entries: QueuedMessage[]; count: number; max: number; + task_id?: string; + session_id?: string; + session_incarnation_id?: string; + status_epoch?: string; + status_generation?: number; merge_enabled: boolean; auto_run?: boolean; + auto_merge_available?: boolean; + auto_merge_enabled?: boolean; + auto_merge_source?: "global" | "session"; + auto_merge_revision?: number; +}; +export type QueueOperationToken = { + sessionIncarnationId: string; + generation: number; }; export type QueueState = { @@ -210,7 +235,8 @@ export type QueueState = { bySessionId: Record; /** Per-session capacity snapshot from the latest server response. */ metaBySessionId: Record; - isLoading: Record; + activeOperationBySessionId: Record; + nextOperationGeneration: number; }; export type SessionSliceState = { @@ -376,9 +402,18 @@ export type SessionSliceActions = { setWalkthroughActiveStep: (taskId: string, stepIndex: number) => void; markWalkthroughSeen: (taskId: string) => void; // Queue actions - setQueueEntries: (sessionId: string, entries: QueuedMessage[], meta: QueueMeta) => void; + setQueueEntries: ( + sessionId: string, + entries: QueuedMessage[], + meta: QueueMeta, + options?: QueueMetaUpdateOptions, + ) => void; removeQueueEntry: (sessionId: string, entryId: string) => void; - setQueueLoading: (sessionId: string, loading: boolean) => void; + beginQueueOperation: ( + sessionId: string, + sessionIncarnationId: string, + ) => QueueOperationToken | null; + finishQueueOperation: (sessionId: string, token: QueueOperationToken) => void; clearQueueStatus: (sessionId: string) => void; }; diff --git a/apps/web/lib/types/http.ts b/apps/web/lib/types/http.ts index 384c149284..a3bab8e96b 100644 --- a/apps/web/lib/types/http.ts +++ b/apps/web/lib/types/http.ts @@ -554,6 +554,8 @@ export type TaskSessionWorktree = { export type TaskSession = ActiveSubagentCountFields & { id: SessionId; task_id: TaskId; + /** Immutable queue ownership identity; changes when a textual session ID is recreated. */ + queue_incarnation_id?: string; /** Optional user-supplied label shown on the session tab. */ name?: string; agent_profile_id?: AgentProfileId; diff --git a/apps/web/lib/types/session-events.ts b/apps/web/lib/types/session-events.ts index ea75f4172c..ab807ac864 100644 --- a/apps/web/lib/types/session-events.ts +++ b/apps/web/lib/types/session-events.ts @@ -204,12 +204,20 @@ export type ProcessStatusPayload = { }; export type QueueStatusChangedPayload = { + task_id?: string; session_id: string; + session_incarnation_id?: string; + status_epoch?: string; + status_generation?: number; entries?: QueuedMessage[] | null; count?: number; max?: number; merge_enabled?: boolean; auto_run?: boolean; + auto_merge_available?: boolean; + auto_merge_enabled?: boolean; + auto_merge_source?: "global" | "session"; + auto_merge_revision?: number; }; export type AvailableCommandPayload = { diff --git a/apps/web/lib/ws/client.test.ts b/apps/web/lib/ws/client.test.ts index 43cf742cf5..e4c30b981a 100644 --- a/apps/web/lib/ws/client.test.ts +++ b/apps/web/lib/ws/client.test.ts @@ -169,6 +169,54 @@ describe("session subscription readiness", () => { }); }); +describe("connection generations", () => { + it("ignores notifications delivered by a replaced socket", () => { + vi.useFakeTimers(); + const { client, socket } = connectClient({ enabled: true, initialDelay: 0, maxAttempts: 1 }); + const handler = vi.fn(); + client.on("message.queue.status_changed", handler); + + socket.close(); + vi.advanceTimersByTime(0); + const reconnectedSocket = FakeWebSocket.latest(); + reconnectedSocket.open(); + + socket.receive({ + id: "stale-status", + type: "notification", + action: "message.queue.status_changed", + payload: { + task_id: "task-1", + session_id: "session-1", + session_incarnation_id: "incarnation-1", + status_epoch: "old-backend", + status_generation: 99, + count: 1, + max: 10, + merge_enabled: true, + }, + }); + expect(handler).not.toHaveBeenCalled(); + + reconnectedSocket.receive({ + id: "current-status", + type: "notification", + action: "message.queue.status_changed", + payload: { + task_id: "task-1", + session_id: "session-1", + session_incarnation_id: "incarnation-1", + status_epoch: "current-backend", + status_generation: 1, + count: 0, + max: 10, + merge_enabled: true, + }, + }); + expect(handler).toHaveBeenCalledOnce(); + }); +}); + describe("request errors", () => { it("retains the backend code and details when a request fails", async () => { const { client, socket } = connectClient(); diff --git a/apps/web/lib/ws/client.ts b/apps/web/lib/ws/client.ts index ed255f72af..56eb9b7589 100644 --- a/apps/web/lib/ws/client.ts +++ b/apps/web/lib/ws/client.ts @@ -103,16 +103,19 @@ export class WebSocketClient { this.intentionalClose = false; this.clearReconnectTimer(); this.setStatus("connecting"); - this.socket = new WebSocket(this.url); + const socket = new WebSocket(this.url); + this.socket = socket; - this.socket.onopen = () => { + socket.onopen = () => { + if (this.socket !== socket) return; this.reconnectAttempts = 0; this.setStatus("connected"); this.resubscribe(); this.flushQueue(); }; - this.socket.onmessage = (event) => { + socket.onmessage = (event) => { + if (this.socket !== socket) return; const parts = (event.data as string).split("\n"); for (const part of parts) { const trimmed = part.trim(); @@ -126,11 +129,10 @@ export class WebSocketClient { } }; - this.socket.onerror = () => { - this.setStatus("error"); - }; + socket.onerror = () => (this.socket === socket ? this.setStatus("error") : undefined); - this.socket.onclose = (event) => { + socket.onclose = (event) => { + if (this.socket !== socket) return; this.socket = null; this.handleDisconnect(event); }; diff --git a/apps/web/lib/ws/handlers/agent-session.test.ts b/apps/web/lib/ws/handlers/agent-session.test.ts index 5a2354ccc4..2bb5b59f2f 100644 --- a/apps/web/lib/ws/handlers/agent-session.test.ts +++ b/apps/web/lib/ws/handlers/agent-session.test.ts @@ -11,6 +11,9 @@ import type { TaskSessionCancellationChangedPayload, TaskSessionStateChangedPayload, } from "@/lib/types/backend"; +const QUEUE_STATUS_ACTION = `message.queue.status_changed`; +const CURRENT_INCARNATION = `incarnation-2`; +const CURRENT_STATUS_EPOCH = `status-epoch-1`; function makeStore(overrides: Record = {}) { const state: Record = { @@ -55,12 +58,12 @@ describe("message.queue.status_changed handler", () => { it("stores the backend-owned Auto-run policy", () => { const setQueueEntries = vi.fn(); const store = makeStore({ setQueueEntries }); - const handler = registerTaskSessionHandlers(store)["message.queue.status_changed"]!; + const handler = registerTaskSessionHandlers(store)[QUEUE_STATUS_ACTION]!; handler({ id: "queue-status-1", type: "notification", - action: "message.queue.status_changed", + action: QUEUE_STATUS_ACTION, payload: { session_id: "s-1", entries: [], @@ -90,12 +93,12 @@ describe("message.queue.status_changed handler", () => { }, }, }); - const handler = registerTaskSessionHandlers(store)["message.queue.status_changed"]!; + const handler = registerTaskSessionHandlers(store)[QUEUE_STATUS_ACTION]!; handler({ id: "queue-status-compat", type: "notification", - action: "message.queue.status_changed", + action: QUEUE_STATUS_ACTION, payload: { session_id: "s-1", entries: [], count: 0, max: 10 }, } as never); @@ -107,6 +110,201 @@ describe("message.queue.status_changed handler", () => { }); }); }); +function makeIncarnationPolicyStore(setQueueEntries: ReturnType) { + return makeStore({ + setQueueEntries, + taskSessions: { + items: { + "s-1": { + id: "s-1", + task_id: "task-1", + queue_incarnation_id: CURRENT_INCARNATION, + }, + }, + }, + queue: { + bySessionId: { "s-1": [] }, + metaBySessionId: { + "s-1": { + count: 0, + max: 10, + mergeEnabled: true, + autoRun: true, + taskId: "task-1", + sessionIncarnationId: CURRENT_INCARNATION, + statusEpoch: CURRENT_STATUS_EPOCH, + statusGeneration: 4, + autoMergeAvailable: true, + autoMergeEnabled: false, + autoMergeSource: "session", + autoMergeRevision: 1, + }, + }, + }, + }); +} + +it("rejects delayed legacy queue status after an incarnated session refetch", () => { + const setQueueEntries = vi.fn(); + const store = makeStore({ + setQueueEntries, + taskSessions: { + items: { + "s-1": { + id: "s-1", + task_id: "task-1", + queue_incarnation_id: CURRENT_INCARNATION, + }, + }, + }, + queue: { bySessionId: {}, metaBySessionId: {} }, + }); + const handler = registerTaskSessionHandlers(store)[QUEUE_STATUS_ACTION]!; + + handler({ + id: "delayed-legacy-status", + type: "notification", + action: QUEUE_STATUS_ACTION, + payload: { + session_id: "s-1", + entries: [{ id: "stale-entry", content: "stale" }], + count: 1, + max: 10, + auto_run: false, + }, + } as never); + + expect(setQueueEntries).not.toHaveBeenCalled(); +}); + +it("applies only the current incarnation's newest Auto-merge policy", () => { + const setQueueEntries = vi.fn(); + const store = makeIncarnationPolicyStore(setQueueEntries); + const handler = registerTaskSessionHandlers(store)[QUEUE_STATUS_ACTION]!; + + handler({ + id: "stale-incarnation", + type: "notification", + action: QUEUE_STATUS_ACTION, + payload: { + task_id: "task-1", + session_id: "s-1", + session_incarnation_id: "incarnation-1", + status_epoch: "incarnation-1", + status_generation: 99, + entries: [], + count: 0, + max: 10, + auto_merge_available: true, + auto_merge_enabled: true, + auto_merge_source: "global", + auto_merge_revision: 8, + }, + } as never); + expect(setQueueEntries).not.toHaveBeenCalled(); + + handler({ + id: "new-status", + type: "notification", + action: QUEUE_STATUS_ACTION, + payload: { + task_id: "task-1", + session_id: "s-1", + session_incarnation_id: CURRENT_INCARNATION, + status_epoch: CURRENT_STATUS_EPOCH, + status_generation: 5, + entries: [], + count: 0, + max: 10, + merge_enabled: true, + auto_run: true, + auto_merge_available: true, + auto_merge_enabled: true, + auto_merge_source: "session", + auto_merge_revision: 2, + }, + } as never); + expect(setQueueEntries).toHaveBeenCalledWith( + "s-1", + [], + expect.objectContaining({ + sessionIncarnationId: CURRENT_INCARNATION, + statusGeneration: 5, + autoMergeAvailable: true, + autoMergeEnabled: true, + autoMergeSource: "session", + autoMergeRevision: 2, + }), + ); +}); + +it("establishes a new backend status epoch and rejects delayed retired events", () => { + const setQueueEntries = vi.fn(); + const store = makeIncarnationPolicyStore(setQueueEntries); + const handler = registerTaskSessionHandlers(store)[QUEUE_STATUS_ACTION]!; + + handler({ + id: "new-backend-epoch", + type: "notification", + action: QUEUE_STATUS_ACTION, + payload: { + task_id: "task-1", + session_id: "s-1", + session_incarnation_id: CURRENT_INCARNATION, + status_epoch: "status-epoch-2", + status_generation: 1, + entries: [{ id: "current-entry", content: "current" }], + count: 1, + max: 10, + merge_enabled: true, + auto_run: false, + auto_merge_available: true, + auto_merge_enabled: true, + auto_merge_source: "session", + auto_merge_revision: 2, + }, + } as never); + expect(setQueueEntries).toHaveBeenCalledWith( + "s-1", + [{ id: "current-entry", content: "current" }], + expect.objectContaining({ + statusEpoch: "status-epoch-2", + statusGeneration: 1, + retiredStatusEpochs: [CURRENT_STATUS_EPOCH], + }), + { establishStatusEpoch: true }, + ); + + store.getState().queue.metaBySessionId["s-1"] = { + ...store.getState().queue.metaBySessionId["s-1"], + statusEpoch: "status-epoch-2", + statusGeneration: 1, + retiredStatusEpochs: [CURRENT_STATUS_EPOCH], + }; + handler({ + id: "delayed-retired-backend-epoch", + type: "notification", + action: QUEUE_STATUS_ACTION, + payload: { + task_id: "task-1", + session_id: "s-1", + session_incarnation_id: CURRENT_INCARNATION, + status_epoch: CURRENT_STATUS_EPOCH, + status_generation: 99, + entries: [{ id: "stale-entry", content: "stale" }], + count: 1, + max: 10, + merge_enabled: true, + auto_run: false, + auto_merge_available: true, + auto_merge_enabled: true, + auto_merge_source: "session", + auto_merge_revision: 2, + }, + } as never); + + expect(setQueueEntries).toHaveBeenCalledTimes(1); +}); const STATE_CHANGED_EVENT = "session.state_changed"; const ACTIVITY_EVENT = "session.activity_changed"; diff --git a/apps/web/lib/ws/handlers/agent-session.ts b/apps/web/lib/ws/handlers/agent-session.ts index a837097186..39ba2fbb65 100644 --- a/apps/web/lib/ws/handlers/agent-session.ts +++ b/apps/web/lib/ws/handlers/agent-session.ts @@ -2,6 +2,7 @@ import type { StoreApi } from "zustand"; import { createDebugLogger } from "@/lib/debug/log"; import type { AppState } from "@/lib/state/store"; +import type { QueueMeta } from "@/lib/state/slices/session/types"; import type { WsHandlers } from "@/lib/ws/handlers/types"; import { sessionId as toSessionId, @@ -635,8 +636,130 @@ function handleCancellationPendingMessage( applyCancellationPending(store, payload); } +function isCompleteQueueStatus(payload: QueueStatusChangedPayload): boolean { + return ( + payload.task_id !== undefined && + payload.session_incarnation_id !== undefined && + payload.status_epoch !== undefined && + payload.status_generation !== undefined && + payload.entries !== undefined && + typeof payload.count === "number" && + typeof payload.max === "number" && + typeof payload.merge_enabled === "boolean" && + typeof payload.auto_run === "boolean" + ); +} + +// eslint-disable-next-line complexity -- incarnation, epoch, and generation form one ordering boundary. +function rejectsQueueStatus( + state: AppState, + payload: QueueStatusChangedPayload, + previousMeta: QueueMeta | undefined, +): boolean { + const currentSession = state.taskSessions.items[payload.session_id]; + if (currentSession?.queue_incarnation_id && payload.session_incarnation_id === undefined) { + return true; + } + if ( + payload.session_incarnation_id !== undefined && + (!currentSession || + currentSession.task_id !== payload.task_id || + currentSession.queue_incarnation_id !== payload.session_incarnation_id) + ) { + return true; + } + const statusEpochChanged = + payload.status_epoch !== undefined && + previousMeta?.statusEpoch !== undefined && + payload.status_epoch !== previousMeta.statusEpoch; + if (statusEpochChanged && payload.status_epoch !== undefined) { + if (previousMeta.retiredStatusEpochs?.includes(payload.status_epoch)) { + return true; + } + return !isCompleteQueueStatus(payload); + } + return ( + payload.status_generation !== undefined && + previousMeta?.statusGeneration !== undefined && + payload.status_generation <= previousMeta.statusGeneration + ); +} + +// eslint-disable-next-line complexity -- status-epoch replacement retains every prior epoch. +function queueIdentityMeta( + payload: QueueStatusChangedPayload, + previousMeta: QueueMeta | undefined, +) { + if ( + payload.session_incarnation_id === undefined && + previousMeta?.sessionIncarnationId === undefined + ) { + return {}; + } + const statusEpoch = payload.status_epoch ?? previousMeta?.statusEpoch; + const statusEpochChanged = + payload.status_epoch !== undefined && + previousMeta?.statusEpoch !== undefined && + payload.status_epoch !== previousMeta.statusEpoch; + const retiredStatusEpochs = statusEpochChanged + ? [...(previousMeta?.retiredStatusEpochs ?? []), previousMeta?.statusEpoch] + .filter((epoch): epoch is string => epoch !== undefined && epoch !== statusEpoch) + .filter((epoch, index, epochs) => epochs.indexOf(epoch) === index) + : previousMeta?.retiredStatusEpochs; + return { + taskId: payload.task_id ?? previousMeta?.taskId, + sessionIncarnationId: payload.session_incarnation_id ?? previousMeta?.sessionIncarnationId, + statusEpoch, + statusGeneration: payload.status_generation ?? previousMeta?.statusGeneration, + retiredStatusEpochs, + }; +} + +function resolveQueuePolicyValue( + preserveSessionPolicy: boolean, + previousValue: T | undefined, + incomingValue: T | undefined, +): T | undefined { + return preserveSessionPolicy ? previousValue : (incomingValue ?? previousValue); +} + +function queueAutoMergeMeta( + payload: QueueStatusChangedPayload, + previousMeta: QueueMeta | undefined, +) { + if ( + payload.auto_merge_available === undefined && + previousMeta?.autoMergeAvailable === undefined + ) { + return {}; + } + const preserveSessionPolicy = + previousMeta?.sessionIncarnationId === payload.session_incarnation_id && + previousMeta?.autoMergeSource === "session" && + payload.auto_merge_source === "global"; + return { + autoMergeAvailable: payload.auto_merge_available ?? previousMeta?.autoMergeAvailable, + autoMergeEnabled: resolveQueuePolicyValue( + preserveSessionPolicy, + previousMeta?.autoMergeEnabled, + payload.auto_merge_enabled, + ), + autoMergeSource: resolveQueuePolicyValue( + preserveSessionPolicy, + previousMeta?.autoMergeSource, + payload.auto_merge_source, + ), + autoMergeRevision: resolveQueuePolicyValue( + preserveSessionPolicy, + previousMeta?.autoMergeRevision, + payload.auto_merge_revision, + ), + }; +} + /** Writes a message.queue.status_changed broadcast into the queue slice, * preserving known policy values when an older publisher omits them. */ +// eslint-disable-next-line complexity -- one handler atomically establishes a queue snapshot. function handleQueueStatusChangedMessage( store: StoreApi, payload: QueueStatusChangedPayload, @@ -645,19 +768,28 @@ function handleQueueStatusChangedMessage( console.warn("[Queue] Missing session_id in queue status change event"); return; } + const state = store.getState(); + const previousMeta = state.queue.metaBySessionId[payload.session_id]; + if (rejectsQueueStatus(state, payload, previousMeta)) return; + const entries = payload.entries ?? []; - const count = typeof payload.count === "number" ? payload.count : entries.length; - const max = typeof payload.max === "number" ? payload.max : 0; - const previousMeta = store.getState().queue.metaBySessionId[payload.session_id]; - const mergeEnabled = - typeof payload.merge_enabled === "boolean" - ? payload.merge_enabled - : (previousMeta?.mergeEnabled ?? true); - const autoRun = - typeof payload.auto_run === "boolean" ? payload.auto_run : (previousMeta?.autoRun ?? true); - store - .getState() - .setQueueEntries(payload.session_id, entries, { count, max, mergeEnabled, autoRun }); + const meta = { + count: typeof payload.count === "number" ? payload.count : entries.length, + max: typeof payload.max === "number" ? payload.max : 0, + mergeEnabled: payload.merge_enabled ?? previousMeta?.mergeEnabled ?? true, + autoRun: payload.auto_run ?? previousMeta?.autoRun ?? true, + ...queueIdentityMeta(payload, previousMeta), + ...queueAutoMergeMeta(payload, previousMeta), + }; + const establishesStatusEpoch = + payload.status_epoch !== undefined && + previousMeta?.statusEpoch !== undefined && + payload.status_epoch !== previousMeta.statusEpoch; + if (establishesStatusEpoch) { + state.setQueueEntries(payload.session_id, entries, meta, { establishStatusEpoch: true }); + return; + } + state.setQueueEntries(payload.session_id, entries, meta); } /** Registers the task-session WebSocket handlers (state, messages, workspace sources, queue). */ diff --git a/apps/web/lib/ws/handlers/tasks-archive.test.ts b/apps/web/lib/ws/handlers/tasks-archive.test.ts index a93eca914c..c72d346cae 100644 --- a/apps/web/lib/ws/handlers/tasks-archive.test.ts +++ b/apps/web/lib/ws/handlers/tasks-archive.test.ts @@ -25,12 +25,14 @@ function makeStore(initial: Partial = {}) { lastSessionByTaskId: {}, }, taskSessionsByTask: { itemsByTaskId: {}, loadedByTaskId: {}, loadingByTaskId: {} }, + taskSessions: { items: {} }, environmentIdBySessionId: {}, setActiveSessionAuto: vi.fn(), removeTaskFromSidebarPrefs: vi.fn(), setTaskDeletedNotification: vi.fn(), upsertQuickChatSessionFromEvent: vi.fn(), removeQuickChatSessionsForTask: vi.fn(), + clearQueueStatus: vi.fn(), setOfficeRefetchTrigger: vi.fn(), ...initial, } as unknown as AppState; diff --git a/apps/web/lib/ws/handlers/tasks-walkthrough-cleanup.test.ts b/apps/web/lib/ws/handlers/tasks-walkthrough-cleanup.test.ts index 9871e7dd2b..0e6c239ea2 100644 --- a/apps/web/lib/ws/handlers/tasks-walkthrough-cleanup.test.ts +++ b/apps/web/lib/ws/handlers/tasks-walkthrough-cleanup.test.ts @@ -20,6 +20,7 @@ function makeStore() { lastSessionByTaskId: {}, }, taskSessionsByTask: { itemsByTaskId: {}, loadedByTaskId: {}, loadingByTaskId: {} }, + taskSessions: { items: {} }, environmentIdBySessionId: {}, walkthroughs: { byTaskId: { @@ -41,6 +42,7 @@ function makeStore() { setTaskDeletedNotification: vi.fn(), upsertQuickChatSessionFromEvent: vi.fn(), removeQuickChatSessionsForTask: vi.fn(), + clearQueueStatus: vi.fn(), } as unknown as AppState; const listeners = new Set(); diff --git a/apps/web/lib/ws/handlers/tasks.deleted.test.ts b/apps/web/lib/ws/handlers/tasks.deleted.test.ts index c90d764681..5cff505e1e 100644 --- a/apps/web/lib/ws/handlers/tasks.deleted.test.ts +++ b/apps/web/lib/ws/handlers/tasks.deleted.test.ts @@ -62,6 +62,27 @@ describe("task.deleted cleanup", () => { expect(state.tasks.pinnedSessionId).toBeNull(); expect(state.tasks.lastSessionByTaskId).not.toHaveProperty("t1"); expect(state.tasks.lastSessionByTaskId).toHaveProperty("t2", SESS_OTHER); + expect(state.clearQueueStatus).toHaveBeenCalledWith(SESS_PINNED); + }); + + it("clears normalized sessions even when no queue metadata was fetched", () => { + const store = makeStore({ + taskSessions: { + items: { + "sess-normalized": { + id: "sess-normalized", + task_id: "t1", + queue_incarnation_id: "inc-normalized", + }, + }, + }, + } as unknown as Partial); + + registerTasksHandlers(store)["task.deleted"]!( + makeDeletedMessage({ task_id: "t1", workflow_id: "wf1" }), + ); + + expect(store.getState().clearQueueStatus).toHaveBeenCalledWith("sess-normalized"); }); it("removes deleted tasks from the archived sidebar projection", () => { diff --git a/apps/web/lib/ws/handlers/tasks.test-helpers.ts b/apps/web/lib/ws/handlers/tasks.test-helpers.ts index 98d29d277d..0f1f380ac7 100644 --- a/apps/web/lib/ws/handlers/tasks.test-helpers.ts +++ b/apps/web/lib/ws/handlers/tasks.test-helpers.ts @@ -22,6 +22,7 @@ export function makeStore(initial: Partial = {}) { resumeSkippedSessionIds: {}, }, taskSessionsByTask: { itemsByTaskId: {}, loadedByTaskId: {}, loadingByTaskId: {} }, + taskSessions: { items: {} }, environmentIdBySessionId: {}, setActiveSession: vi.fn((taskId: string, sessionId: string | null) => { state = { @@ -51,6 +52,7 @@ export function makeStore(initial: Partial = {}) { setTaskDeletedNotification: vi.fn(), upsertQuickChatSessionFromEvent: vi.fn(), removeQuickChatSessionsForTask: vi.fn(), + clearQueueStatus: vi.fn(), ...initial, } as unknown as AppState; diff --git a/apps/web/lib/ws/handlers/tasks.ts b/apps/web/lib/ws/handlers/tasks.ts index 20057e8055..cce9dbe76e 100644 --- a/apps/web/lib/ws/handlers/tasks.ts +++ b/apps/web/lib/ws/handlers/tasks.ts @@ -397,14 +397,21 @@ export function registerTasksHandlers(store: StoreApi): WsHandlers { "task.updated": (message) => handleTaskUpdated(store, message), "task.deleted": (message) => { const deletedId = message.payload.task_id; + const currentState = store.getState(); removeRecentTask(deletedId); // A quick chat closed on another device must not linger here as a tab // pointing at a task the backend already deleted. store.getState().removeQuickChatSessionsForTask(deletedId); - const currentState = store.getState(); - const sessionIds = (currentState.taskSessionsByTask.itemsByTaskId[deletedId] ?? []).map( - (s) => s.id, + const sessionIds = Array.from( + new Set([ + ...(currentState.taskSessionsByTask.itemsByTaskId[deletedId] ?? []).map( + (session) => session.id, + ), + ...Object.values(currentState.taskSessions?.items ?? {}) + .filter((session) => session.task_id === deletedId) + .map((session) => session.id), + ]), ); const task = currentState.kanban.tasks.find((t) => t.id === deletedId); if (task?.primarySessionId) { @@ -425,6 +432,7 @@ export function registerTasksHandlers(store: StoreApi): WsHandlers { currentState.removeTaskFromSidebarPrefs(deletedId); for (const sid of sessionIds) { useContextFilesStore.getState().clearSession(sid); + currentState.clearQueueStatus?.(sid); } const wasActive = currentState.tasks.activeTaskId === deletedId; diff --git a/apps/web/lib/ws/handlers/workspaces.test.ts b/apps/web/lib/ws/handlers/workspaces.test.ts index e6c86e71a2..bf1ecedea6 100644 --- a/apps/web/lib/ws/handlers/workspaces.test.ts +++ b/apps/web/lib/ws/handlers/workspaces.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createStore } from "zustand"; import type { StoreApi } from "zustand"; import type { AppState } from "@/lib/state/store"; @@ -74,3 +74,49 @@ describe("workspace.updated placement", () => { expect(store.getState().workspaces.items[0].unit_id).toBe("unit-new"); }); }); + +describe("workspace.deleted queue cleanup", () => { + const WORKSPACE_ID = "workspace-1"; + + it("clears normalized task sessions without relying on queue metadata", () => { + const clearQueueStatus = vi.fn(); + let state = { + workspaces: { + items: [{ id: WORKSPACE_ID }, { id: "workspace-2" }], + activeId: WORKSPACE_ID, + }, + workflows: { items: [], activeId: null }, + kanban: { + workflowId: "workflow-1", + steps: [], + tasks: [{ id: "task-1", workspaceId: WORKSPACE_ID }], + }, + kanbanMulti: { snapshots: {} }, + taskSessions: { + items: { + "session-1": { + id: "session-1", + task_id: "task-1", + queue_incarnation_id: "incarnation-1", + }, + }, + }, + clearQueueStatus, + } as unknown as AppState; + const store = { + getState: () => state, + setState: (updater: AppState | Partial | ((value: AppState) => AppState)) => { + state = ( + typeof updater === "function" ? updater(state) : { ...state, ...updater } + ) as AppState; + }, + } as StoreApi; + + registerWorkspacesHandlers(store)["workspace.deleted"]!({ + payload: { id: WORKSPACE_ID }, + } as never); + + expect(clearQueueStatus).toHaveBeenCalledOnce(); + expect(clearQueueStatus).toHaveBeenCalledWith("session-1"); + }); +}); diff --git a/apps/web/lib/ws/handlers/workspaces.ts b/apps/web/lib/ws/handlers/workspaces.ts index 0b55a549ee..7b05b85f50 100644 --- a/apps/web/lib/ws/handlers/workspaces.ts +++ b/apps/web/lib/ws/handlers/workspaces.ts @@ -4,6 +4,38 @@ import type { WsHandlers } from "@/lib/ws/handlers/types"; type WorkspaceItem = WorkspaceState["items"][number]; +function handleWorkspaceDeleted(store: StoreApi, workspaceId: string): void { + const currentState = store.getState(); + const workspaceTaskIds = new Set( + [ + ...currentState.kanban.tasks, + ...Object.values(currentState.kanbanMulti.snapshots).flatMap((snapshot) => snapshot.tasks), + ] + .filter((task) => task.workspaceId === workspaceId) + .map((task) => task.id), + ); + const sessionIds = Object.values(currentState.taskSessions.items) + .filter((session) => workspaceTaskIds.has(session.task_id)) + .map((session) => session.id); + for (const sessionId of sessionIds) { + currentState.clearQueueStatus(sessionId); + } + store.setState((state) => { + const items = state.workspaces.items.filter((item) => item.id !== workspaceId); + const activeId = + state.workspaces.activeId === workspaceId + ? (items[0]?.id ?? null) + : state.workspaces.activeId; + const clearBoards = state.workspaces.activeId === workspaceId; + return { + ...state, + workspaces: { items, activeId }, + workflows: clearBoards ? { items: [], activeId: null } : state.workflows, + kanban: clearBoards ? { workflowId: null, steps: [], tasks: [] } : state.kanban, + }; + }); +} + export function registerWorkspacesHandlers(store: StoreApi): WsHandlers { return { "workspace.created": (message) => { @@ -72,24 +104,6 @@ export function registerWorkspacesHandlers(store: StoreApi): WsHandler }, })); }, - "workspace.deleted": (message) => { - store.setState((state) => { - const items = state.workspaces.items.filter((item) => item.id !== message.payload.id); - const activeId = - state.workspaces.activeId === message.payload.id - ? (items[0]?.id ?? null) - : state.workspaces.activeId; - const clearBoards = state.workspaces.activeId === message.payload.id; - return { - ...state, - workspaces: { - items, - activeId, - }, - workflows: clearBoards ? { items: [], activeId: null } : state.workflows, - kanban: clearBoards ? { workflowId: null, steps: [], tasks: [] } : state.kanban, - }; - }); - }, + "workspace.deleted": (message) => handleWorkspaceDeleted(store, message.payload.id), }; } diff --git a/apps/web/src/locales/en/chat.json b/apps/web/src/locales/en/chat.json index 7e9b92cc02..8c8bd59dad 100644 --- a/apps/web/src/locales/en/chat.json +++ b/apps/web/src/locales/en/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "Failed to remove queued message.", "failedToReorderQueuedMessages": "Failed to reorder queued messages.", "failedToSetQueueAutoRun": "Failed to update queue Auto-run.", + "failedToSetQueueAutoMerge": "Failed to update queue Auto-merge.", "failedToStartQuickChat": "Failed to start quick chat", "fileTreeActions": "File actions", "loadingRepositories": "Loading repositories", @@ -89,6 +90,7 @@ "queueCapacityFull_one": "{{count}} of {{max}} · full", "queueCapacityFull_other": "{{count}} of {{max}} · full", "queueAutoRun": "Auto-run", + "queueAutoMerge": "Auto-merge", "queueAutoRunOffHelp": "Finishes the current response, then queued messages wait.", "queueAutoRunOnHelp": "Runs queued messages one at a time.", "queueChipExpand_one": "{{count}} queued message, click to expand", diff --git a/apps/web/src/locales/en/system.json b/apps/web/src/locales/en/system.json index b62679449b..9abdb87c94 100644 --- a/apps/web/src/locales/en/system.json +++ b/apps/web/src/locales/en/system.json @@ -165,7 +165,7 @@ "licensesTitle": "Third-party licenses", "licensesToggleLabel": "Show license text for {{name}}", "messageQueueAdminOnly": "Only administrators can change this setting.", - "messageQueueAutoMergeDescription": "Combines consecutive compatible queued messages from the same source into one turn before delivery.", + "messageQueueAutoMergeDescription": "Sets the default for all sessions. A session follows this value until you change Auto-merge in that session's queue.", "messageQueueAutoMergeNotice": "If a message has a different source or incompatible task, model, mode, metadata, or shared limits, it stays as a separate queued message. This applies only to new admissions and does not change the manual merge control.", "messageQueueAutoMergeToggleLabel": "Automatically merge consecutive messages", "messageQueueConfigured": "Configured:", diff --git a/apps/web/src/locales/pseudo/chat.json b/apps/web/src/locales/pseudo/chat.json index 21ec2d4516..1cf69ba7f9 100644 --- a/apps/web/src/locales/pseudo/chat.json +++ b/apps/web/src/locales/pseudo/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "Ƒàĩĺēď ţō ŕēḿōvē qũēũēď ḿēśśàĝē.", "failedToReorderQueuedMessages": "Ƒàĩĺēď ţō ŕēōŕďēŕ qũēũēď ḿēśśàĝēś.", "failedToSetQueueAutoRun": "Ƒàĩĺēď ţō ũƥďàţē qũēũē Àũţō-ŕũń.", + "failedToSetQueueAutoMerge": "Ƒàĩĺēď ţō ũƥďàţē qũēũē Àũţō-ḿēŕĝē.", "failedToStartQuickChat": "Ƒàĩĺēď ţō śţàŕţ qũĩćķ ćĥàţ", "fileTreeActions": "Ƒĩĺē àćţĩōńś", "loadingRepositories": "Ĺōàďĩńĝ ŕēƥōśĩţōŕĩēś", @@ -89,6 +90,7 @@ "queueCapacityFull_one": "{{count}} ōƒ {{max}} · ƒũĺĺ", "queueCapacityFull_other": "{{count}} ōƒ {{max}} · ƒũĺĺ", "queueAutoRun": "Àũţō-ŕũń", + "queueAutoMerge": "Àũţō-ḿēŕĝē", "queueAutoRunOffHelp": "Ƒĩńĩśĥēś ţĥē ćũŕŕēńţ ŕēśƥōńśē, ţĥēń qũēũēď ḿēśśàĝēś ŵàĩţ.", "queueAutoRunOnHelp": "Ŕũńś qũēũēď ḿēśśàĝēś ōńē àţ à ţĩḿē.", "queueChipExpand_one": "{{count}} qũēũēď ḿēśśàĝē, ćĺĩćķ ţō ēxƥàńď", diff --git a/apps/web/src/locales/pseudo/system.json b/apps/web/src/locales/pseudo/system.json index 68630107b3..c84a22d8c4 100644 --- a/apps/web/src/locales/pseudo/system.json +++ b/apps/web/src/locales/pseudo/system.json @@ -165,7 +165,7 @@ "licensesTitle": "Ţĥĩŕď-ƥàŕţŷ ĺĩćēńśēś", "licensesToggleLabel": "Śĥōŵ ĺĩćēńśē ţēxţ ƒōŕ {{name}}", "messageQueueAdminOnly": "Ōńĺŷ àďḿĩńĩśţŕàţōŕś ćàń ćĥàńĝē ţĥĩś śēţţĩńĝ.", - "messageQueueAutoMergeDescription": "Ćōḿƀĩńēś ćōńśēćũţĩvē ćōḿƥàţĩƀĺē qũēũēď ḿēśśàĝēś ƒŕōḿ ţĥē śàḿē śōũŕćē ĩńţō ōńē ţũŕń ƀēƒōŕē ďēĺĩvēŕŷ.", + "messageQueueAutoMergeDescription": "Śēţś ţĥē ďēƒàũĺţ ƒōŕ àĺĺ śēśśĩōńś. À śēśśĩōń ƒōĺĺōŵś ţĥĩś vàĺũē ũńţĩĺ ŷōũ ćĥàńĝē Àũţō-ḿēŕĝē ĩń ţĥàţ śēśśĩōń'ś qũēũē.", "messageQueueAutoMergeNotice": "Ĩƒ à ḿēśśàĝē ĥàś à ďĩƒƒēŕēńţ śōũŕćē ōŕ ĩńćōḿƥàţĩƀĺē ţàśķ, ḿōďēĺ, ḿōďē, ḿēţàďàţà, ōŕ śĥàŕēď ĺĩḿĩţś, ĩţ śţàŷś àś à śēƥàŕàţē qũēũēď ḿēśśàĝē. Ţĥĩś àƥƥĺĩēś ōńĺŷ ţō ńēŵ àďḿĩśśĩōńś àńď ďōēś ńōţ ćĥàńĝē ţĥē ḿàńũàĺ ḿēŕĝē ćōńţŕōĺ.", "messageQueueAutoMergeToggleLabel": "Àũţōḿàţĩćàĺĺŷ ḿēŕĝē ćōńśēćũţĩvē ḿēśśàĝēś", "messageQueueConfigured": "Ćōńƒĩĝũŕēď:", diff --git a/apps/web/src/locales/pt-pt/chat.json b/apps/web/src/locales/pt-pt/chat.json index 1a80135a31..82f03ddcba 100644 --- a/apps/web/src/locales/pt-pt/chat.json +++ b/apps/web/src/locales/pt-pt/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "Não foi possível remover a mensagem em fila.", "failedToReorderQueuedMessages": "Não foi possível reordenar as mensagens em fila.", "failedToSetQueueAutoRun": "Não foi possível atualizar a execução automática da fila.", + "failedToSetQueueAutoMerge": "Não foi possível atualizar a fusão automática da fila.", "failedToStartQuickChat": "Não foi possível iniciar o chat rápido", "fileTreeActions": "Ações de ficheiro", "loadingRepositories": "A carregar os repositórios", @@ -73,6 +74,7 @@ "queueCapacityFull_one": "{{count}} de {{max}} · cheia", "queueCapacityFull_other": "{{count}} de {{max}} · cheia", "queueAutoRun": "Execução automática", + "queueAutoMerge": "Fusão automática", "queueAutoRunOffHelp": "Termina a resposta atual e depois as mensagens em fila aguardam.", "queueAutoRunOnHelp": "Executa as mensagens em fila, uma de cada vez.", "queueChipExpand_one": "{{count}} mensagem em fila, clique para expandir", diff --git a/apps/web/src/locales/pt-pt/system.json b/apps/web/src/locales/pt-pt/system.json index eb56b54fbb..6eac1b82a4 100644 --- a/apps/web/src/locales/pt-pt/system.json +++ b/apps/web/src/locales/pt-pt/system.json @@ -533,7 +533,7 @@ "versionTitle": "Versão", "versionViewUpdates": "Ver as atualizações", "dataStoragePageDescription": "Estatísticas da base de dados, cópias de segurança e registos do servidor.", - "messageQueueAutoMergeDescription": "Junta mensagens consecutivas compatíveis em fila da mesma origem num único turno antes da entrega.", + "messageQueueAutoMergeDescription": "Define a predefinição para todas as sessões. Uma sessão segue este valor até alterar a Junção automática na fila dessa sessão.", "messageQueueAutoMergeNotice": "Se uma mensagem tiver outra origem ou tarefa, modelo, modo, metadados ou limites partilhados incompatíveis, permanece como mensagem em fila separada. Isto aplica-se apenas a novas admissões e não altera o controlo de junção manual.", "messageQueueAutoMergeToggleLabel": "Juntar automaticamente mensagens consecutivas", "navDataStorage": "Dados e registos" diff --git a/apps/web/src/locales/zh-cn/chat.json b/apps/web/src/locales/zh-cn/chat.json index 8a8097f3ca..cdd0479846 100644 --- a/apps/web/src/locales/zh-cn/chat.json +++ b/apps/web/src/locales/zh-cn/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "移除排队中的消息失败。", "failedToReorderQueuedMessages": "无法重新排序队列中的消息。", "failedToSetQueueAutoRun": "更新队列自动运行设置失败。", + "failedToSetQueueAutoMerge": "更新队列自动合并设置失败。", "failedToStartQuickChat": "启动快速聊天失败", "fileTreeActions": "文件操作", "loadingRepositories": "正在加载仓库", @@ -80,6 +81,7 @@ "queueCapacityFull_one": "{{count}} / {{max}} · 已满", "queueCapacityFull_other": "{{count}} / {{max}} · 已满", "queueAutoRun": "自动运行", + "queueAutoMerge": "自动合并", "queueAutoRunOffHelp": "完成当前回复后，排队中的消息将等待。", "queueAutoRunOnHelp": "逐条运行排队中的消息。", "queueChipExpand_one": "{{count}} 条排队消息，点击展开", diff --git a/apps/web/src/locales/zh-cn/system.json b/apps/web/src/locales/zh-cn/system.json index b0b84a72fa..c70e59d7e9 100644 --- a/apps/web/src/locales/zh-cn/system.json +++ b/apps/web/src/locales/zh-cn/system.json @@ -534,7 +534,7 @@ "versionLatest": "最新", "versionTitle": "版本", "versionViewUpdates": "查看更新", - "messageQueueAutoMergeDescription": "在投递前，把来自同一来源的连续且兼容的排队消息合并为一轮。", + "messageQueueAutoMergeDescription": "设置所有会话的默认值。会话会沿用此值，直到您在该会话的队列中更改自动合并。", "messageQueueAutoMergeNotice": "如果某条消息的来源不同，或任务、模型、模式、元数据或共享限制不兼容，它会保持为单独的排队消息。这只适用于新接收的消息，不会改变手动合并控制。", "messageQueueAutoMergeToggleLabel": "自动合并连续消息" } diff --git a/apps/web/src/locales/zh-hk/chat.json b/apps/web/src/locales/zh-hk/chat.json index 366b5d66d8..540bf01a9a 100644 --- a/apps/web/src/locales/zh-hk/chat.json +++ b/apps/web/src/locales/zh-hk/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "移除排隊中的訊息失敗。", "failedToReorderQueuedMessages": "無法重新排序佇列中的訊息。", "failedToSetQueueAutoRun": "更新佇列自動執行設定失敗。", + "failedToSetQueueAutoMerge": "更新佇列自動合併設定失敗。", "failedToStartQuickChat": "啓動快速聊天失敗", "fileTreeActions": "檔案操作", "loadingRepositories": "正在載入儲存庫", @@ -80,6 +81,7 @@ "queueCapacityFull_one": "{{count}} / {{max}} · 已滿", "queueCapacityFull_other": "{{count}} / {{max}} · 已滿", "queueAutoRun": "自動執行", + "queueAutoMerge": "自動合併", "queueAutoRunOffHelp": "完成目前回覆後，排隊中的訊息將等待。", "queueAutoRunOnHelp": "逐條執行排隊中的訊息。", "queueChipExpand_one": "{{count}} 條排隊訊息，點擊展開", diff --git a/apps/web/src/locales/zh-hk/system.json b/apps/web/src/locales/zh-hk/system.json index 54af93bcaf..5762ad3822 100644 --- a/apps/web/src/locales/zh-hk/system.json +++ b/apps/web/src/locales/zh-hk/system.json @@ -534,7 +534,7 @@ "versionLatest": "最新", "versionTitle": "版本", "versionViewUpdates": "查看更新", - "messageQueueAutoMergeDescription": "在投遞前，把來自同一來源的連續且兼容的排隊訊息合併為一輪。", + "messageQueueAutoMergeDescription": "設定所有工作階段的預設值。工作階段會沿用此值，直到您在該工作階段的佇列中變更自動合併。", "messageQueueAutoMergeNotice": "如果某條訊息的來源不同，或任務、模型、模式、元數據或共享限制不兼容，它會保持為單獨的排隊訊息。這隻適用於新接收的訊息，不會改變手動合併控制。", "messageQueueAutoMergeToggleLabel": "自動合併連續訊息" } diff --git a/apps/web/src/locales/zh-tw/chat.json b/apps/web/src/locales/zh-tw/chat.json index 981919aaac..bcb2b04b6e 100644 --- a/apps/web/src/locales/zh-tw/chat.json +++ b/apps/web/src/locales/zh-tw/chat.json @@ -45,6 +45,7 @@ "failedToRemoveQueuedMessage": "移除排隊中的訊息失敗。", "failedToReorderQueuedMessages": "無法重新排序佇列中的訊息。", "failedToSetQueueAutoRun": "更新佇列自動執行設定失敗。", + "failedToSetQueueAutoMerge": "更新佇列自動合併設定失敗。", "failedToStartQuickChat": "啟動快速聊天失敗", "fileTreeActions": "檔案操作", "loadingRepositories": "正在載入儲存庫", @@ -80,6 +81,7 @@ "queueCapacityFull_one": "{{count}} / {{max}} · 已滿", "queueCapacityFull_other": "{{count}} / {{max}} · 已滿", "queueAutoRun": "自動執行", + "queueAutoMerge": "自動合併", "queueAutoRunOffHelp": "完成目前回覆後，排隊中的訊息將等待。", "queueAutoRunOnHelp": "逐條執行排隊中的訊息。", "queueChipExpand_one": "{{count}} 條排隊訊息，點選展開", diff --git a/apps/web/src/locales/zh-tw/system.json b/apps/web/src/locales/zh-tw/system.json index e1c6dd643f..a5d3029d9a 100644 --- a/apps/web/src/locales/zh-tw/system.json +++ b/apps/web/src/locales/zh-tw/system.json @@ -534,7 +534,7 @@ "versionLatest": "最新", "versionTitle": "版本", "versionViewUpdates": "檢視更新", - "messageQueueAutoMergeDescription": "在投遞前，把來自同一來源的連續且相容的排隊訊息合併為一輪。", + "messageQueueAutoMergeDescription": "設定所有工作階段的預設值。工作階段會沿用此值，直到您在該工作階段的佇列中變更自動合併。", "messageQueueAutoMergeNotice": "如果某條訊息的來源不同，或任務、模型、模式、後設資料或共享限制不相容，它會保持為單獨的排隊訊息。這隻適用於新接收的訊息，不會改變手動合併控制。", "messageQueueAutoMergeToggleLabel": "自動合併連續訊息" } diff --git a/docs/decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md b/docs/decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md new file mode 100644 index 0000000000..e9eb6c26da --- /dev/null +++ b/docs/decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md @@ -0,0 +1,159 @@ +# ADR-2026-09-04-inherit-queue-auto-merge-until-overridden: Inherit Queue Auto-merge Until Overridden + +**Status:** accepted +**Date:** 2026-09-04 +**Area:** backend, frontend, protocol + +## Context + +Automatic queued-message merging has one install-wide setting. The queue panel +needs a per-session switch without making every session a frozen copy of the +global value when it is created. + +A binary switch cannot also expose a third "follow global" state. The product +contract therefore needs a precedence rule that lets untouched sessions follow +administrator changes while preserving an explicit session choice. + +## Decision + +Resolve automatic-merge policy for each message admission and queue-status read +with this precedence: + +1. an explicit per-session override, when present; +2. otherwise the current install-wide automatic-merge value. + +New sessions have no override. Existing sessions that have never changed the +queue-panel switch continue to inherit later global changes. The first user +change to the session switch persists an explicit boolean, and that session +remains independent of later global changes for its lifetime. + +The message-queue domain stores the nullable override beside its existing +per-session Auto-run state. The global setting remains owned by the system +message-queue settings service. Successful queue status projects availability, +the effective boolean, and transport-only source and revision metadata. The +panel still renders a binary value and does not expose inherited state or a +reset action. + +An override is attached to the exact session identity. Workflow queue transfer +does not copy it to the destination session; the destination keeps its own +override or inherits the current global value, and the source state is removed. +Deleting a session, task, or workspace removes state for the deleted session +identities atomically with that lifecycle operation. Archiving preserves state +because its sessions survive and may be unarchived. Reusing a deleted identity +therefore starts without an override. + +Session deletion relies on queue repository transactional cleanup instead of a +post-delete queue mutation. A separate deletion-only post-commit callback +carries authoritative session/incarnation pairs to ephemeral memory storage; +the archive task purger never invokes it. The later recount is task-scoped with +an empty session ID, remains available to the internal status-summary +projector, and is consumed by gateway routing without browser broadcast. + +Each task-session row also owns an opaque server-generated queue incarnation +UUID. It is immutable for that row and changes when a deleted textual session +ID is recreated. Its migration runs after every existing task-session table +rebuild, backfills distinct UUIDs replay-safely, and future rebuilds must +preserve it. Every session-scoped queue status carries this identity so the +client can reject delayed status from a removed incarnation before applying +entries or policy. + +Ordinary admissions and Auto-merge mutations carry the expected incarnation +into persistence. A mutation request cannot be silently retargeted: a missing or +changed incarnation returns the existing non-enumerating session-not-found +response and writes nothing. + +An override read failure fails closed for that admission: Kandev does not merge +messages when it cannot prove that an explicit OFF override is absent. A failed +write reports failure and leaves the previously persisted value authoritative. + +An override read failure during queue-status construction does not synthesize an +effective OFF value. Session status reconciles independent queue data, marks +Auto-merge unavailable, omits its value/source/revision, and retains the status +epoch/generation allocated before the failed read. Clients apply availability +only from the newest status generation, retain any last authoritative tuple, +and disable the control until a newer successful resolution. + +Global and session values each carry a persisted monotonic revision. The system +settings service commits the global value and revision together, then publishes +one immutable `{Enabled, Revision}` runtime snapshot through an atomic pointer. +Backend startup hydrates that pointer from the persisted pair before admissions; +it never reconstructs revision `0` from the boolean alone. Admissions and status +reads load the pair once, so they cannot observe a value from one generation +with another generation's revision. A global revision advances only when the +global value changes; a session revision advances with each successful override +write. + +The repository owns the durable policy and lifecycle linearization point. SQL +admission, override mutation, and session deletion lock the owning task row +before the per-session queue lock, then validate the exact +task/session/incarnation. Admission reads the override and inserts or directly +folds in that transaction. This lock order matches task/workspace purge and +avoids both orphan insertion and lock inversion. The memory repository validates +session authority under an incarnation-tagged bucket mutex and deletion CAS +purges only matching buckets. The service admission mutex remains useful for +local ordering but is not the cross-connection correctness boundary. The +immutable global snapshot is process-local under Kandev's +single-orchestrator-owner deployment invariant. + +Session queue status carries the immutable session incarnation plus a process +epoch and generation allocated before policy resolution. Clients first require +the status incarnation to match the current task-session row. A new matching +incarnation resets the prior queue-policy ordering state; a mismatch is +discarded. Within an incarnation, clients order available and unavailable +results by epoch/generation, then use policy source and revision to order +effective values. They accept the irreversible global-to-session transition, +reject a later global-source event for that incarnation, and compare revisions +within one source. A process restart changes the epoch and startup preserves +the persisted global policy revision. This prevents delayed failure, +pre-mutation status, or a deleted incarnation from replacing newer +authoritative value without scanning sessions when the global setting changes. + +## Consequences + +- Administrator changes immediately govern later admissions for every + unmodified session without writing or scanning session rows. +- A user can make one session permanently differ from the install default with + a compact binary switch. +- The UI cannot tell whether the displayed effective value is inherited or + explicit, and it cannot restore inheritance after the first change. +- Successful session queue status, WebSocket events, admission logic, and + full-capacity candidate folding resolve the same effective value, source, and + revision. Failed session status resolution carries ordering plus an + unavailable marker. Task-only status-summary signals are explicitly scoped + and carry no session policy. +- Session-state storage requires a nullable override, monotonic revision, and + replay-safe migration in both supported SQL dialects. +- The persisted global settings envelope gains an internal monotonic revision; + public settings responses do not expose it as editable configuration. +- The queue service replaces its independent atomic boolean with one atomic + immutable value/revision snapshot and hydrates it from persistence at startup. +- Session status gains process epoch/generation ordering so delayed unavailable + results cannot disable newer successful state. +- Override writes and policy-aware admissions share the durable session lock, + so a write through another database connection is ordered before or after the + admission policy decision. +- Destructive lifecycle cleanup must distinguish session identity removal from + task archive; only the former deletes queue policy state. +- Every task-session creation and legacy backfill needs a stable non-empty queue + incarnation UUID. The migration runs after existing task-session rebuilds, + and future rebuilds preserve it. Frontend deletion clears queue metadata, + workspace deletion derives affected sessions before removing ownership + indexes, and status apply compares against the current session incarnation. +- Task-scoped queue recounts remain internal; the gateway never broadcasts them + through a session or empty-workspace route. +- Queue mutation requests and ordinary admissions require the expected + incarnation; stale work cannot target a recreated textual session ID. +- Memory cleanup needs a deletion-only incarnation-aware callback distinct from + archive queue-row purging. + +## Alternatives Considered + +1. **Snapshot the global value when each session is created.** Rejected because + untouched active sessions would not inherit later administrator changes. +2. **Add a tri-state control with Follow global.** Rejected because the requested + queue header control is a compact toggle, and no reset workflow is required. +3. **Store the override on the task or workspace.** Rejected because the user + asked to isolate behavior per session and tasks can have several sessions. +4. **Copy the override during workflow session transfer.** Rejected because an + override belongs to the exact session; the destination already has an + independent policy identity. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index f617cc580e..a8082f8c95 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -253,6 +253,7 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-09-06-opt-in-external-e2e-runners | [Opt in selected Linux CI jobs to external runners](2026-09-06-opt-in-external-e2e-runners.md) | accepted | infra, workflow, security | 2026-09-06 | | 2026-09-06-explicit-pr-auto-fix-outcomes | [Require Explicit Outcomes for PR Auto-Fix Attempts](2026-09-06-explicit-pr-auto-fix-outcomes.md) | proposed | backend, frontend, protocol, workflow, GitHub | 2026-09-06 | | 2026-09-05-bounded-progressive-storage-analysis | [Use bounded progressive storage analysis](2026-09-05-bounded-progressive-storage-analysis.md) | accepted | backend, frontend, protocol | 2026-09-05 | +| 2026-09-04-inherit-queue-auto-merge-until-overridden | [Inherit Queue Auto-merge Until Overridden](2026-09-04-inherit-queue-auto-merge-until-overridden.md) | accepted | backend, frontend, protocol | 2026-09-04 | | 2026-09-07-separate-runtime-permissions-and-advisory-actions | [Keep MCP availability separate from runtime permissions](2026-09-07-separate-runtime-permissions-and-advisory-actions.md) | accepted | backend, protocol | 2026-09-07 | | 2026-09-07-isolate-fork-preview-builds | [Isolate contributor preview builds from deployment credentials](2026-09-07-isolate-fork-preview-builds.md) | accepted | infra, workflow, security | 2026-09-07 | | 2026-09-05-script-capable-html-preview-isolation | [Capability-Free Runtime for Script-Capable HTML Preview](2026-09-05-script-capable-html-preview-isolation.md) | proposed | frontend, desktop, security | 2026-09-05 | diff --git a/docs/plans/message-queue-session-automation/plan.md b/docs/plans/message-queue-session-automation/plan.md new file mode 100644 index 0000000000..7005e5d8fb --- /dev/null +++ b/docs/plans/message-queue-session-automation/plan.md @@ -0,0 +1,318 @@ +--- +created: 2026-09-04 +status: completed +requirements: + - REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001 + - REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001 +system_design: + - ../../specs/ui/system-design/message-queue-automation-controls.md +legacy_specs: + - ../../specs/ui/requirements/message-queue-run.md + - ../../specs/ui/requirements/message-queue-auto-merge.md +--- + +# Implementation Plan: Message Queue Session Automation + +## Overview + +Add the per-session Auto-merge policy before exposing it in the queue panel, so +every rendered switch is backed by authoritative persistence and event +projection. Then recompose Auto-run and Auto-merge as compact header pills, +prove the same workflow on desktop and mobile, and reconcile public queue and +WebSocket documentation. + +## Scope + +### In scope + +- Preserve current install-wide automatic-merge configuration as the fallback. +- Persist an explicit automatic-merge override only after a session switch is + changed. +- Resolve untouched sessions against later global changes and keep overridden + sessions independent. +- Project status epoch/generation and policy availability plus the effective + value, source, and revision through queue reads and every session-scoped live + status producer. +- Move Auto-run into a compact pill beside the queue summary and add the + matching Auto-merge pill in the same responsive header region. +- Preserve queue actions, touch sizing, one scroll owner, and mobile capability. +- Define future eligible direct folding at or above the limit and + staged-attachment rejection in the draft replacement contract. Keep active + capacity requirements unchanged until verified implementation and lifecycle + cutover. + +### Out of scope + +- A third Follow global state or reset-override action. +- Per-task or per-workspace automatic-merge policy. +- Retroactive compaction of pending rows. +- Changes to compatibility, content/entity combination, manual merge, Send Now, + Auto-run delivery, capacity configuration, or workflow-coalescing semantics. + Coalescing only gains the common session-identity guard. Automatic fold + queued time remains the shipped later target/source value. +- Copying an override to a different session during workflow session transfer. + +## Technical approach + +### Backend policy and protocol + +- Extend `queue_session_state` in + `apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go` with + nullable `auto_merge_override` and monotonic `auto_merge_revision` columns in + fresh schema and replay-safe migration paths. +- Extend `Repository` plus memory and SQL implementations with nullable + override reads and atomic explicit writes that increment and return the + session revision. +- Require `QueueSessionIdentity` on every service and repository operation that + can add, append, replace, restore, reserve, acknowledge, consume, update, + remove, merge, reorder, clear, transfer, or change session policy. Browser, + plugin, MCP, clarification, workflow, retry, pending-move, drain, and Send Now + paths migrate without textual-ID mutating overloads. +- Define required SQL and memory session-authority seams in `messagequeue`. + Besides task locks and session validation, their guarded boundary claims + typed staged attachments. Production injects the task repository; SQL + fixtures seed task/session/attachment rows and memory fixtures seed a locked + authority. Remove task-table probing, nil authority, and fail-open identity + validation. +- Use one repository transaction helper: lock sorted unique task rows through + the authority, then sorted repository-local and durable session locks, then + validate exact identities. Admission resolves policy, inserts, claims staged + attachments, and folds inside that boundary; claim failure rolls everything + back. Override, pending-move, and destructive lifecycle mutations use the same + order. Remove the post-commit after-insert callback and compensation delete. +- Transfer requires exact source and destination identities in one transaction, + using sorted task rows then sorted session locks before validation. It removes + source state without replacing destination policy. Deterministic + two-connection tests cover both race orders with session/task/workspace + delete/recreate and opposite-direction transfer. +- Add queue-owned session-state cleanup to destructive transactions. Preserve + state on archive. A separate deletion-only post-commit callback carries + authoritative identities; memory buckets CAS-purge only matching deleted + incarnations, while the existing archive/delete purger and recount notifier + stay separate. +- Add `pending_moves.session_incarnation_id`. Deferred apply CASes identity, + move/source/target, increments task transition generation, and returns an + immutable effect token. Post-commit task effects CAS generation/target; + session/queue effects validate token identity/generation. Recreation or a + successor move makes every stale effect fail closed. +- Add an immutable server-generated `queue_incarnation_id` to each task-session + row and expose it with `TaskSession`. Register its replay-safe distinct UUID + backfill after every existing task-session table rebuild, require future + rebuilds to preserve it, and carry it as `session_incarnation_id` on every + session-scoped queue status. +- Persist an internal global automatic-merge revision with the existing system + message-queue settings value. Increment it only when the global boolean + changes, commit both together, and keep the revision out of editable public + settings responses. +- Replace the queue service's independent automatic boolean with one immutable + value/revision snapshot published through an atomic pointer after settings + commit. Hydrate the same pair from persisted queue settings during backend + startup before admissions become available. +- Update `messagequeue.Service` admission and status paths to resolve the + override against one loaded global snapshot. Use the repository-linearized + effective snapshot for ordinary and eligible direct folds; fail closed on an + override read error, preserve the shipped newest queued time, and reject + staged-attachment admissions at or above the row limit without claiming an + attachment. +- Add status ordering, availability, and effective policy to `QueueStatus`, plus + `message.queue.auto_merge.set`. Every browser queue request, including get + and all mutations, requires the task/session/incarnation triplet. Gateway and + handler authorize the supplied pair and pass the triplet unchanged. +- Replace separate queue/session reads with + `Snapshot(expected QueueSessionIdentity)`. Browser requests/mutations supply + expected identity; internal work retains it. Persistence revalidates and + snapshots under task-then-session locks. Responses/events use that identity + and publish nothing on mismatch, including mutation-commit/recreation races. +- Tag reservations, `SendNowClaim`, queued dispatch/cancellation, deferred + transition dispatch, and in-flight ownership with identity plus operation + generation. Revalidate before cancel, prompt, restore, ack, and CAS cleanup + without locks over agent I/O. Post-claim races cover manual drain, Auto-run, + and both Send Now scopes. +- Include the identity, ordering, and available tuple or unavailable marker in + get/mutation responses and every session-scoped producer. +- After session deletion, rely on repository-transaction cleanup and its + deletion-only memory callback rather than post-delete `CancelAll`, then + publish only an empty-session explicit task-scoped recount. Gateway routing + retains that signal for internal status-summary projection but drops it + before browser broadcast; invalid empty-session queue events are also + dropped. +- Preserve current snapshot/restore semantics and Auto-run transfer behavior. + +### Frontend state and controls + +- Preserve backend JSON naming in frontend wire-derived types: + `TaskSession.task_id`, `TaskSession.queue_incarnation_id`, and queue payload + `session_incarnation_id`. Reject session-scoped queue payloads whose + incarnation differs from the current `TaskSession`, and reset policy ordering + when a matching new incarnation replaces stored metadata. +- Replace textual-session loading booleans with active operation tokens carrying + session incarnation and a monotonically allocated client operation + generation. All terminal paths compare-and-clear only the exact token while + its incarnation is still current, so delayed completion cannot clear a + replacement session's mutation. +- Clear queue metadata/loading on every session and task removal. For + `workspace.deleted`, snapshot affected session IDs by joining normalized + current `TaskSession.task_id` values to locally cached workspace tasks before + removing indexes, then invoke the same session cleanup. +- Add status epoch/generation, policy availability, and the known Auto-merge + value/source/revision to `QueueMeta` and queue read/event contracts, with + preservation logic in every queue metadata replacement. +- Apply newest-status availability, process-epoch retirement, source precedence, + and same-source policy revision comparison within one incarnation so failed + resolution or a delayed pre-mutation event cannot disable, invent, or + overwrite newer authoritative state. +- Require current task/session/incarnation in every queue request in + `queue-api.ts`; `useQueue` supplies `TaskSession` identity, refetches after + stale outcomes, and never retries. Raw E2E WebSocket helpers accept and emit + the same complete triplet for get, add, and cancel. +- Update `QueueAffordance`, `QueuePanel`, and `QueuePanelHeader` wiring for the + effective value and mutation. +- Replace the dedicated Auto-run row in + `queued-ghost-panel-header.tsx` with one responsive automation region. Wide + layouts keep the pills beside the summary on the first visual line; narrow + layouts may wrap logical groups. Each compact bordered pill contains a + localized visible label and shared `Switch`; Auto-run retains state-specific + accessible help. +- Update the global Message Queue settings description and component test to + explain inheritance until a per-session change. +- Add complete English, Portuguese, Simplified Chinese, and generated + Traditional Chinese locale entries for the pills, mutation error, and global + inheritance description. + +### Browser coverage + +- Extend `apps/web/e2e/tests/chat/message-queue.spec.ts` with a desktop flow that + verifies both pills share the summary header, an untouched session follows a + changed global value, a user change persists across reload, and a later + global change no longer affects that session. +- Extend + `apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts` with the + same user value on `mobile-chrome`: both controls are reachable by touch, + mutation persists, the wrapping header remains contained, and the document + has no horizontal overflow. +- Use API setup only for global-setting preconditions; all session-override + changes and assertions go through the rendered queue controls. +- Update `apps/web/e2e/helpers/api-client.ts` and every queue helper caller. + Focused sidebar and workflow scenarios cover raw get/add/cancel triplets; + typecheck enforces the clean helper-signature cutover across chat, task, + workflow, and SSH suites. + +### Public documentation + +- Update `docs/public/coordination.md`, `docs/public/operations.md`, + `docs/public/sessions-and-review.md`, `docs/public/configuration.md`, and + `docs/public/websocket-api.md` where they describe queue Auto-run, + install-wide automatic merging, full-capacity behavior, or queue protocol + fields. +- After production and behavioral checks pass, activate the replacement + requirements, deprecate and forward-link the unchanged superseded contracts, + amend the active management capacity text to the delivered exceptions, mark + the system design current, and update the UI specification indexes. + +## Tests + +- `AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2`, `.3`, `.7`, `.8`, `.9`, + `.11`, `.12`, `.13`, `.15`, `.16`, `.17`, `.18`, `.19`, `.21`, `.22`, `.23`, + `.24`, and `.25` map to settings, startup, migrations, authority, claims, + deferred transactions/effects, every request/mutator, async dispatch + ownership, lifecycle races, expected-identity snapshots, and producer tests. +- `AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.2`, `.7`, and `.8` plus + `AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.4`, `.12`, `.13`, `.15`, `.16`, + `.17`, `.18`, `.19`, `.20`, and `.24` map to queue API, hook, + incarnation/generation operation tokens, slice, removal, connection epoch, + WebSocket-handler, settings, and header tests under `apps/web`. +- Existing Auto-run, automatic-merge compatibility, manual merge, + full-capacity, and Send Now tests remain regression coverage for unchanged + behavior. + +## E2E tests + +- `apps/web/e2e/tests/chat/message-queue.spec.ts` covers + `AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.1`, `.2`, `.7` and + `AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2`, `.3`, `.4`, `.11`, `.12`, + `.13` on Chromium. +- `apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts` covers + `AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.9` and + `AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.14` on `mobile-chrome`. +- `apps/web/e2e/tests/task/sidebar-queued-count.spec.ts` covers complete raw + queue get/add/cancel identities for + `AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.19` and `.24`. + +## Work orders + +- [ ] [Task 01: Persist Session Auto-merge Overrides](task-01-persist-session-auto-merge-overrides.md) +- [ ] [Task 02: Deliver Queue Automation Pills](task-02-deliver-queue-automation-pills.md) +- [ ] [Task 03: Update Queue Automation Documentation](task-03-update-queue-automation-documentation.md) + +## Verification results + +Pending. + +After all work-order checks pass, run the user-requested repository gates from +the repository root in this order: + +```bash +make fmt +make typecheck test lint +``` + +## Risks + +- Collapsing nullable inheritance into a boolean during persistence would freeze + untouched sessions and violate later global changes. +- Missing one synthesized `QueueMeta` path could make Auto-merge visually reset + after clear, reorder, reconnect, an unavailable status, or an older event. +- A compact desktop header can become an unusable compressed toolbar on mobile; + the control group must wrap while preserving action visibility and one scroll + owner. +- Queue Auto-merge terminology can be confused with pull-request Auto-merge; + copy and documentation must keep the message-queue context explicit. +- Every browser read/mutator and internal operation carries expected identity. + A textual-ID path lets delayed work target a replacement; compile-time + signatures plus typed and raw-client stale-token tests enforce inventory. +- Queue repositories require injected SQL/memory authority. Typed arguments + alone cannot validate current task-session rows inside the lock boundary. +- Transfer locks both task rows before sorted session locks and revalidates both + endpoints; another order can deadlock deletion or move stale rows. +- Attachment claims execute through authority inside admission; a post-commit + callback leaves a deletion/recreation gap. +- Deferred apply returns an immutable transition effect token. Task effects CAS + generation/target and session effects validate identity/generation; tests + race both recreation and successor moves after commit. +- Reusing archive's task purger for memory policy cleanup either resets archive + state or leaks deleted overrides. A separate deletion-only callback carries + authoritative incarnation pairs and CAS-purges matching memory buckets. +- Queue metadata keyed only by textual session ID can accept delayed policy + from a deleted incarnation. Status must match the current immutable + task-session incarnation, and every removal path must clear queue state. +- A status publisher that re-resolves after authorization or mutation commit + can expose replacement state. Producers use + `Snapshot(expected QueueSessionIdentity)` and publish nothing on mismatch. +- Untagged async claims let delayed drain, Auto-run, deferred dispatch, or Send + Now work target or clear replacement ownership. Identity/generation guards + survive claim through prompt, restore, acknowledgement, and cleanup. +- A raw helper signature change without migrating every caller leaves hidden + WebSocket payloads stale. The work order inventories chat, task, workflow, and + SSH callers; focused add/cancel/get E2E plus typecheck enforces the cutover. +- An incarnation migration placed before an existing `task_sessions` rebuild + is silently dropped; it must run after all current rebuilds and future + projections must retain it. +- Workspace deletion loses empty-queue metadata unless it derives affected + sessions before removing task/workspace indexes. +- A delayed status result can regress or disable the new switch unless every + session-scoped producer emits epoch/generation and + availability/source/revision metadata and both refetch and event paths use + the same comparisons. +- Independent runtime atomics for the global boolean and revision could expose + a policy generation that never existed; all reads use one immutable snapshot + hydrated from the persisted revision at startup. +- The global settings description becomes part of the precedence contract; all + locales and its component test must change together. +- The replacement requirements must remain draft until Tasks 01 and 02 pass, + then cut over atomically with deprecation links from the prior contracts. +- Task-only status-summary signals cannot carry session policy or reach browser + routing; explicit scope, an empty session ID, early gateway consumption, and + producer/gateway tests enforce the boundary. +- PostgreSQL verification must include both changed DSN-gated repository + packages; running only message-queue tests misses task-session migration and + lifecycle behavior. diff --git a/docs/plans/message-queue-session-automation/task-01-persist-session-auto-merge-overrides.md b/docs/plans/message-queue-session-automation/task-01-persist-session-auto-merge-overrides.md new file mode 100644 index 0000000000..214801ebc0 --- /dev/null +++ b/docs/plans/message-queue-session-automation/task-01-persist-session-auto-merge-overrides.md @@ -0,0 +1,364 @@ +--- +id: "01-persist-session-auto-merge-overrides" +title: "Persist Session Auto-merge Overrides" +status: completed +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001 +acceptance_criteria: + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.5 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.7 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.8 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.11 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.12 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.13 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.16 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.17 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.18 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.19 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.21 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.22 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.23 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.24 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.25 +system_design: + - ../../specs/ui/system-design/message-queue-automation-controls.md +--- + +# Task 01: Persist Session Auto-merge Overrides + +## Summary + +Add a nullable, queue-owned per-session automatic-merge override with a +monotonic revision and resolve it against the revisioned live install-wide +fallback. Give every task-session row an immutable queue incarnation UUID. +Project that incarnation, status epoch/generation, policy availability, value, +source, and revision through every session-scoped producer before any frontend +control consumes it. + +## In scope + +- Write failing queue-settings, startup wiring, service, repository, handler, + orchestrator, gateway, and MCP producer tests before production code. +- Persist absent, explicit OFF, and explicit ON as distinct states with a + monotonic session revision in memory, SQLite, and PostgreSQL-compatible + storage. +- Introduce required `QueueSessionIdentity{TaskID, SessionID, +SessionIncarnationID}` arguments across every service and repository + operation that inserts, appends, replaces, restores, reserves, acknowledges, + consumes, updates, removes, merges, reorders, clears, transfers, or changes + session policy. Admission, coalescing, retry, pending-move, drain, Send Now, + Auto-run, and Auto-merge paths retain captured identities; remove every + textual-ID mutating overload and `QueueMessageWithMetadataAfterInsert`. +- Require `task_id`, `session_id`, and `session_incarnation_id` on every browser + queue read or mutation. Gateway and handler authorize the supplied pair, then + pass the unchanged triplet to persistence. Stale identity returns the existing + non-enumerating session-not-found response without mutation, claim, dispatch, + attachment, response-state, or status-publication effects. +- Define `SQLSessionAuthority` and `MemorySessionAuthority` in `messagequeue`. + The SQL seam locks tasks, validates sessions, and claims a typed + `QueueAttachmentClaim` on the queue transaction; the memory seam supplies an + equivalent claim-capable guarded boundary. Backend wiring injects the task + repository into the queue repository. Remove `tasksTablePresent` and every + missing-schema or nil-authority bypass. SQL fixtures create task, session, and + attachment authority rows; memory fixtures seed a lock-protected authority + whose guard encloses bucket mutation. +- Add one repository policy-aware admission operation carrying the identity. + SQL locks the active task row, then the durable `queue_session_locks` row, + revalidates that exact session incarnation, and reads the override plus + inserts or directly folds in one transaction. Route override writes and + lifecycle-coalescing inserts through the same identity check and lock order. + The memory repository validates through task-session authority while holding + its incarnation-tagged bucket mutex. +- Resolve authenticated attachment owner/workspace and staged IDs before queue + admission. Pass the claim value into the policy-aware repository operation; + identity validation, insert, attachment claim, and eligible fold share one + transaction or memory critical section. Any failure rolls back every queue + and attachment change. A two-connection test pauses inside the claim seam and + races session deletion/recreation in both orders. +- Add deterministic two-connection tests for every mutation category and + identity-bound status publication against session/task/workspace recreation, + including a pause after mutation commit before status snapshot. Cover + opposite transfer ordering and deferred-transition pauses before writes and + after commit before effects. Channel-controlled tests recreate after a drain, + Auto-run, or Send Now claim and before prompt/restore. Prove stale work cannot + mutate, dispatch to, publish, restore into, block, or clear replacement state. +- Persist an internal global revision with the automatic-merge setting, advance + it atomically only when that boolean changes, publish one immutable + value/revision runtime snapshot after commit, and hydrate the same pair during + backend startup. +- Preserve inheritance for untouched sessions across later global changes and + isolate explicit overrides by session. +- Preserve the shipped automatic-fold timestamp behavior: the target row + survives while its queued time becomes the later target/source time. +- Apply one effective policy tuple to ordinary and eligible direct folds. +- Keep staged-attachment admissions at or above capacity out of the direct fold + path, with no attachment claim or tail mutation. +- Add queue-owned policy cleanup under a common transaction helper that locks + sorted unique task rows through the injected authority, then sorted + repository-local and durable queue-session locks, then revalidates every + identity. Session, task, and workspace deletion remove state atomically; + archive preserves it. Transfer validates exact source and destination + identities in that order, removes source state, and leaves the destination + override unchanged. +- Add a separate deletion-only post-commit callback carrying complete + task/session/incarnation identities to the memory repository. Keep archive's + purger/recount notifier separate. Memory cleanup CAS-purges only exact + incarnation-tagged buckets. +- Add `task_sessions.queue_incarnation_id`, generate it once during session + creation, and expose it on `TaskSession`. Register its replay-safe ALTER and + distinct empty-row UUID backfill after both existing `task_sessions` rebuild + migrations; require future rebuild projections to preserve it. Test legacy + rebuild order, partial-backfill restart, stable reread, and textual ID reuse. +- Add the authorized `message.queue.auto_merge.set` mutation with required + `session_incarnation_id` plus status epoch/generation, + `auto_merge_available`, and the effective policy tuple. Compare the expected + incarnation again under the persistence locks and return the existing + non-enumerating session-not-found response on mismatch. +- Add `pending_moves.session_incarnation_id`; backfill it by joining the owning + task session and delete unmatched legacy rows. Carry `QueueSessionIdentity` + through set/read/take, retry, reaper CAS, snapshot/restore, and transfer. +- Make deferred apply one transaction that CASes captured identity, pending move, + expected source, and target; applies direct writes; increments task workflow + transition generation; and returns an immutable effect token. Same-target + uses the same operation. +- After commit, task effects CAS token generation/target and session/queue + effects validate token identity/generation. Publication, history, exit/enter, + prompt cleanup, drain, and pull-next fail closed after recreation or a + successor move. +- Replace separate `GetStatus`/`SessionTaskID` assembly with + `Snapshot(QueueSessionIdentity)`. Browser requests supply expected identity; + internal work retains it. Responses/events snapshot that same identity after + commit and publish nothing on mismatch. +- Put identity plus operation generation on queue reservations, `SendNowClaim`, + queued dispatch/cancellation, deferred dispatch, and in-flight ownership. + Restore/ack revalidates through the repository; cancellation, prompt, and + cleanup verify identity/generation against task-session and execution + ownership without holding database locks across agent I/O. +- Update queue handler, session-scoped orchestrator lifecycle, and MCP producers + and tests for ordered available and unavailable shapes. Give task-only + status-summary signals an explicit scope and keep them out of frontend + session reconciliation. +- Replace the post-delete `CancelAll` path in `task_operations.go` with the + repository-owned transactional cleanup and memory callback. Publish only + `publishTaskQueueStatusEvent(taskID, "")` after deletion. +- Make gateway queue routing consume explicit task-scoped status internally + without browser broadcast. Drop invalid empty-session queue events rather + than falling through to an empty-workspace broadcast. +- Preserve user-visible snapshot/restore, manual merge, Auto-run, and Send Now + semantics while migrating their internal ownership to exact identities. + +## Out of scope + +- Frontend state or queue-panel controls. +- A clear-override API. +- Changing automatic-merge compatibility, content/entity combination, or + retroactively folding queued rows. Queued-time behavior stays aligned with + the existing `latestQueuedAt` implementation. +- Changing the global settings endpoint or administrator permissions. + +## Acceptance + +- Tests first fail on missing inheritance, authority/claim injection, complete + request and internal identities, task-row/session-lock ordering, attachment + rollback, atomic deferred writes, guarded post-commit effects, identity-bound + publication, async drain/Auto-run/Send Now ownership, stale mutation/status/ + transfer/pending-move rejection, lifecycle cleanup, migrations, task-only + routing, startup snapshots, unavailable status, capacity, or producer shape; + minimum implementation makes them pass. +- Untouched sessions use every later committed global snapshot; restart retains + its persisted revision, and an explicitly changed session retains its value + and monotonic revision across queue drain and repository reconstruction + without changing another session. +- Every queue read/mutation uses exact typed identity and task-then-session + revalidation. Ordinary admission uses one policy snapshot; staged claim, + insert, and fold share the transaction. Claim failure rolls back all writes; + mismatch publishes no status; task-only signals never reach browsers. +- Deferred writes CAS identity/move/source/target in one transaction and return + an effect token. Task effects CAS transition generation/target; session + effects validate identity/generation. Pre-write, post-commit, recreation, and + successor-move races cannot redirect an effect. +- Drain, Auto-run, deferred dispatch, and Send Now reservations/claims retain + identity/generation through cancel, prompt, restore, acknowledgement, and + cleanup. Delayed old work cannot dispatch to, block, or clear a replacement. +- Session/task/workspace deletion removes policy; archive preserves it; + transfer removes source state and preserves destination policy. Session + delete performs no second `CancelAll` and emits only a task recount. +- Every browser read/mutator and internal producer carries expected identity. + Responses/events snapshot that identity and publish nothing after mismatch. + A recreated textual ID receives a new incarnation; task-only status has none. +- Automatic fold tests retain the target identity and assert that its queued + time is the later target/source time. + +## Verification + +```bash +(cd apps/backend && go test -race ./internal/system/queuesettings ./internal/backendapp ./internal/orchestrator/messagequeue ./internal/orchestrator/handlers ./internal/orchestrator ./internal/mcp/handlers ./internal/gateway/websocket ./internal/task/repository/sqlite) +(cd apps/backend && go test ./pkg/websocket) +if [ -n "${KANDEV_TEST_POSTGRES_DSN:-}" ]; then (cd apps/backend && go test -race ./internal/task/repository/sqlite ./internal/orchestrator/messagequeue); else printf '%s\n' 'SKIP: KANDEV_TEST_POSTGRES_DSN unavailable'; fi +git ls-files --cached --others --exclude-standard -- '*.go' | xargs -r gofmt -l +``` + +The final formatting command must print no files. The conditional PostgreSQL +command reruns every changed package containing DSN-gated coverage: task-session +ALTER/rebuild/backfill and destructive lifecycle tests in +`internal/task/repository/sqlite`, plus queue policy and cross-connection tests +in `internal/orchestrator/messagequeue`. Its explicit output records an +environment skip when the DSN is unavailable. + +## Files likely touched + +- `apps/backend/internal/system/queuesettings/types.go` +- `apps/backend/internal/system/queuesettings/store.go` +- `apps/backend/internal/system/queuesettings/service.go` +- `apps/backend/internal/system/queuesettings/queuesettings_test.go` +- `apps/backend/internal/backendapp/orchestrator.go` +- `apps/backend/internal/backendapp/orchestrator_wiring_test.go` +- `apps/backend/internal/backendapp/adapters_plugin_messenger.go` +- `apps/backend/internal/backendapp/adapters_plugin_messenger_test.go` +- `apps/backend/internal/task/models/models.go` +- `apps/backend/internal/task/dto/dto.go` +- `apps/backend/internal/task/repository/sqlite/base_schema.go` +- `apps/backend/internal/task/repository/sqlite/base_migrations.go` +- `apps/backend/internal/task/repository/sqlite/session_test.go` +- `apps/backend/internal/task/repository/sqlite/session_incarnation_migration_test.go` +- `apps/backend/internal/task/repository/sqlite/base.go` +- `apps/backend/internal/task/repository/sqlite/session.go` +- `apps/backend/internal/task/repository/sqlite/attachment.go` +- `apps/backend/internal/task/repository/sqlite/attachment_postgres_test.go` +- `apps/backend/internal/task/repository/sqlite/task.go` +- `apps/backend/internal/task/repository/sqlite/workspace.go` +- `apps/backend/internal/task/repository/sqlite/task_queue_purge_notify_test.go` +- `apps/backend/internal/task/repository/sqlite/session_queue_state_cleanup_test.go` +- `apps/backend/internal/orchestrator/messagequeue/repository.go` +- `apps/backend/internal/orchestrator/messagequeue/repository_memory.go` +- `apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go` +- `apps/backend/internal/orchestrator/messagequeue/types.go` +- `apps/backend/internal/orchestrator/messagequeue/service.go` +- `apps/backend/internal/orchestrator/messagequeue/repository_auto_merge_override_test.go` +- `apps/backend/internal/orchestrator/messagequeue/service_auto_merge_test.go` +- `apps/backend/internal/orchestrator/messagequeue/pending_move_identity_test.go` +- `apps/backend/internal/orchestrator/messagequeue/repository_postgres_pending_move_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_auto_merge_policy_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_publish_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_send_now_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_merge_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers_reorder_test.go` +- `apps/backend/internal/orchestrator/event_handlers_agent.go` +- `apps/backend/internal/orchestrator/event_handlers_clarification.go` +- `apps/backend/internal/orchestrator/event_handlers_workflow.go` +- `apps/backend/internal/orchestrator/queue_admission_identity_test.go` +- `apps/backend/internal/orchestrator/workflow_store.go` +- `apps/backend/internal/orchestrator/workflow_store_deferred_identity_test.go` +- `apps/backend/internal/orchestrator/queued_dispatch.go` +- `apps/backend/internal/orchestrator/queued_dispatch_identity_test.go` +- `apps/backend/internal/orchestrator/queue_send_now.go` +- `apps/backend/internal/orchestrator/queue_send_now_test.go` +- `apps/backend/internal/orchestrator/queue_auto_run_test.go` +- `apps/backend/internal/orchestrator/queue_auto_run_review_test.go` +- `apps/backend/internal/orchestrator/event_handlers_queue_test.go` +- `apps/backend/internal/orchestrator/event_handlers_queue_general_test.go` +- `apps/backend/internal/orchestrator/event_handlers_queue_lifecycle_test.go` +- `apps/backend/internal/orchestrator/queue_purge_status_test.go` +- `apps/backend/internal/orchestrator/task_operations.go` +- `apps/backend/internal/orchestrator/session_delete_contract_test.go` +- `apps/backend/internal/orchestrator/pending_move_identity_test.go` +- `apps/backend/internal/mcp/handlers/handlers.go` +- `apps/backend/internal/mcp/handlers/queue_status_event_test.go` +- `apps/backend/internal/gateway/websocket/task_notifications.go` +- `apps/backend/internal/gateway/websocket/task_notifications_test.go` +- `apps/backend/internal/gateway/websocket/dispatch_scope.go` +- `apps/backend/internal/gateway/websocket/dispatch_scope_test.go` +- `apps/backend/internal/mcp/handlers/config_task_handlers.go` +- `apps/backend/pkg/websocket/actions.go` + +## Dependencies + +None. + +## Risks + +- Converting a missing override to `false` would stop untouched sessions from + following the global value; converting it to `true` would ignore a global OFF + value. +- An upsert that replaces the whole state row could reset the independent + Auto-run value. +- A service-only admission gate does not serialize another repository + connection. Admission, override writes, and deletion must all acquire task + row before session lock and validate expected incarnation; reversing the + order deadlocks task/workspace purge, while omitting validation lets a stale + waiter target a recreated ID. +- Reusing the archive task purger for policy cleanup either erases surviving + archive policy or leaves deleted memory overrides. A separate deletion-only, + incarnation-aware callback owns memory policy cleanup. +- Reusing a textual session ID without a distinct stable incarnation lets a + delayed status from the deleted row restore its irreversible session-source + tuple. Creation and legacy backfill must always supply a non-empty UUID, and + every session-scoped producer must echo it. +- Adding the incarnation migration before a legacy `task_sessions` rebuild + silently drops the column. It runs after all current rebuilds, and every + future rebuild must project it. +- A status producer that re-resolves identity after mutation commit can publish + replacement state. Responses/events snapshot the operation's expected + identity; browser reads carry the same triplet through authorization. +- A textual-ID mutator or async ownership record lets delayed work target or + clear a replacement. Compile-time signatures, identity/generation-tagged + claims, and post-claim race tests enforce the complete inventory. +- Letting the queue repository probe for task tables or accept a nil authority + makes production identity validation optional and isolated tests + unrepresentative. Constructors require injected authorities, and fixtures + explicitly seed them. +- Transfer that locks session rows before both task rows in deterministic order + can deadlock deletion or move rows across recreated identities. It uses the + common task-then-sorted-session transaction helper and validates both ends. +- The current after-insert callback runs after the queue transaction commits, + so compensation cannot prevent a delete/recreate claim race. Attachment claim + becomes a typed authority operation inside the admission transaction; the + callback and rollback delete are removed. +- Atomic deferred writes do not protect effects after commit. Task effects CAS + committed move/target while session and queue effects retain identity, with + deterministic recreation tests on both sides of commit. +- PostgreSQL and SQLite represent nullable booleans differently at the driver + boundary; tests must exercise the repository contract rather than raw SQL + text. +- Running only message-queue PostgreSQL tests misses task-session migration and + lifecycle code; the conditional DSN pass covers both repository packages plus + orchestrator and gateway integration. +- Missing one session-scoped producer leaves clients vulnerable to value + regression even if the primary queue handler is correct; task-only signals + need an explicit exemption. +- The persisted global value and revision must share one write, and the runtime + service must publish, hydrate, and read them as one immutable snapshot; a + boolean-only startup path or two atomics can create a policy generation that + never existed. +- Treating a status-read failure as authoritative OFF would replace valid client + state; an unsequenced unavailable marker could also disable newer success. + +## Parallelism + +`sequential` + +## Inputs + +- `REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001` +- `docs/specs/ui/system-design/message-queue-automation-controls.md` +- `docs/decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md` +- `apps/backend/internal/orchestrator/messagequeue/service.go` +- `apps/backend/internal/orchestrator/messagequeue/repository_auto_run_test.go` +- `apps/backend/internal/orchestrator/handlers/queue_handlers.go` + +## Results + +Completed. Added durable task-session queue incarnations, revisioned global and +per-session Auto-merge policy, identity-bound browser and internal queue +producers, pending-move identity retention, destructive lifecycle cleanup, and +ordered status snapshots. Admission skips folding when policy reads fail; +status reads preserve entries while reporting policy unavailable. SQLite, +orchestrator, handler, gateway, MCP, migration, and race-focused tests pass. diff --git a/docs/plans/message-queue-session-automation/task-02-deliver-queue-automation-pills.md b/docs/plans/message-queue-session-automation/task-02-deliver-queue-automation-pills.md new file mode 100644 index 0000000000..066f3a166a --- /dev/null +++ b/docs/plans/message-queue-session-automation/task-02-deliver-queue-automation-pills.md @@ -0,0 +1,248 @@ +--- +id: "02-deliver-queue-automation-pills" +title: "Deliver Queue Automation Pills" +status: completed +wave: 2 +depends_on: + - "01-persist-session-auto-merge-overrides" +plan: "plan.md" +requirements: + - REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001 + - REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001 +acceptance_criteria: + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.1 + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.2 + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.7 + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.8 + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.9 + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.10 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.1 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.4 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.11 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.12 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.13 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.14 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.16 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.17 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.18 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.19 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.20 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.24 +system_design: + - ../../specs/ui/system-design/message-queue-automation-controls.md +--- + +# Task 02: Deliver Queue Automation Pills + +## Summary + +Drive session incarnation, status epoch/generation, policy availability, and +the effective tuple through frontend API, store, live-event, and mutation paths +with stale-update protection. Recompose the expanded queue header into compact +Auto-run and Auto-merge pills, with desktop and mobile Playwright scenarios +written and observed failing before UI implementation. + +## In scope + +- Bootstrap workspace dependencies once if this worktree has no + `apps/node_modules`. +- Write failing queue API, store/event ordering, settings, header, desktop/mobile, + and raw WebSocket helper/browser coverage before production frontend code. +- Preserve the backend JSON names `queue_incarnation_id` and `task_id` on + `TaskSession`, and `session_incarnation_id` on every session-scoped queue + payload; do not introduce a partial camel-case mapping layer. Reject the + whole status when it does not match the current session, and reset prior + policy ordering only when a new matching incarnation replaces stored queue + metadata. +- Preserve matching-incarnation status epoch/generation, policy availability, + and the known Auto-merge value/source/revision through fetch, clear, reorder, + WebSocket notification, and older-payload compatibility paths. +- Apply availability only from the newest generation in the current process + epoch, retire prior epochs, reject stale same-source revisions, and reject + global-source status after the irreversible first session override within + that incarnation. +- Replace `QueueState.isLoading[sessionId]` booleans with + `activeOperationBySessionId` tokens containing the current session + incarnation and a monotonically allocated client operation generation. + `beginQueueOperation` verifies the current incarnation, rejects a conflict, + installs a fresh token, and returns it; `finishQueueOperation` + compare-and-clears only that token while the incarnation remains current. + Independent refetches own a token, while a mutation's terminal refetch retains + its caller token. Route every success, error, cancellation, early return, + refetch, and `finally` path through these APIs. +- Make session and task removal clear queue entries, metadata, and the active + operation token for every removed session. In `workspace.deleted`, snapshot + affected session IDs before state removal by joining normalized + `TaskSession.task_id` records to locally cached tasks for that workspace, + then call the same session cleanup for each ID. Incarnation validation + guarantees cached queue metadata has a corresponding current `TaskSession`. +- Add same-process delete/recreate coverage with delayed old-incarnation status + success and unavailable events plus mutation success, error, refetch, and + `finally` cleanup while a new-incarnation mutation remains pending. Also + cover workspace deletion with empty-queue metadata and an active operation + token. +- Require `task_id`, `session_id`, and `session_incarnation_id` in every + frontend queue request, including status get and all mutations. `useQueue` + supplies current identity. Raw E2E helpers require the same triplet, and the + clean signature cutover migrates every caller rather than inferring identity + from textual session ID. Stale responses refetch but never retry. One + matching-token conflict predicate covers both switches, Send Now, and reorder; + Auto-merge also requires policy availability. +- Place both labeled switch pills beside the queue summary in one wrapping + header composition and remove the dedicated Auto-run row. +- Preserve the desktop-only pin, all queue actions, one internal scroll owner, + keyboard semantics, and coarse-pointer hit areas. +- Update the global settings description and test to explain inheritance until + a session override. +- Add complete English, Portuguese, Simplified Chinese, Hong Kong Chinese, and + Traditional Chinese locale entries using the repository's generation flow. + +## Out of scope + +- Backend storage, protocol handlers, and global settings behavior or permissions. +- A Follow global control or inherited/explicit visual badge. +- Changes to row rendering, ordering, manual merge, Send Now, or queue capacity. +- A mobile-only drawer or alternate queue route. + +## Acceptance + +- RED evidence covers incarnation isolation, complete get/mutation wire + payloads from typed and raw clients, status ordering, policy availability, + generation-scoped cleanup, delayed unavailable protection, shared disabling, + settings explanation, and compact composition. +- Both pills display authoritative state and disable during conflicting work. + Every queue request sends current task/session/incarnation, never retargets + after textual ID reuse, and never retries stale work. Focused tests reject + lower-generation terminal updates, recover from unavailable state, survive + reload, and preserve global precedence. +- Session/task/workspace removal leaves no queue metadata or operation token. + Recreating the same textual session ID resets source ordering only for its new + matching incarnation; delayed status from the removed incarnation is ignored + before and after the new status. Delayed terminal paths from an old mutation + cannot clear the active new-incarnation token or re-enable controls. +- Desktop preserves dense pill sizing; `mobile-chrome` proves touch-effective + controls, wrapping containment, no document horizontal overflow, and the same + session override outcome. + +## Verification + +```bash +(cd apps && pnpm install --frozen-lockfile) +(cd apps && pnpm --filter @kandev/web test -- --run lib/api/domains/queue-api.test.ts hooks/domains/session/use-queue.test.ts hooks/use-task-removal.test.ts lib/state/slices/session/session-slice.upsert.test.ts lib/state/slices/session/remove-task-session.test.ts lib/ws/handlers/agent-session.test.ts lib/ws/handlers/tasks.deleted.test.ts lib/ws/handlers/workspaces.test.ts components/settings/system/message-queue-settings.test.tsx components/task/chat/queued-ghost-panel-header.test.tsx components/task/chat/queued-ghost-list.test.tsx) +(cd apps/web && pnpm run typecheck) +(cd apps/web && pnpm run i18n:check) +(cd apps/web && pnpm exec eslint components/settings/system/message-queue-settings.tsx components/task/chat/queued-ghost-panel-header.tsx components/task/chat/queued-ghost-list.tsx hooks/domains/session/use-queue.ts lib/api/domains/queue-api.ts lib/state/app-state-types.ts lib/state/slices/session/session-slice.ts lib/state/slices/session/types.ts lib/ws/handlers/agent-session.ts lib/types/session-events.ts e2e/helpers/api-client.ts e2e/tests/chat/message-queue-scroll-helpers.ts e2e/tests/chat/message-queue.spec.ts e2e/tests/chat/mid-turn-steering.spec.ts e2e/tests/chat/mobile-message-queue-management.spec.ts e2e/tests/ssh/launch-task.spec.ts e2e/tests/task/mobile-sidebar-queued-count.spec.ts e2e/tests/task/sidebar-queued-count.spec.ts e2e/tests/workflow/workflow-queue-helpers.ts e2e/tests/workflow/workflow-manual-move-queue.spec.ts e2e/tests/workflow/mobile-workflow-manual-move-queue.spec.ts) +(cd apps/web && pnpm e2e:run tests/chat/message-queue.spec.ts tests/task/sidebar-queued-count.spec.ts tests/workflow/workflow-manual-move-queue.spec.ts) +(cd apps/web && pnpm e2e:run --project mobile-chrome tests/chat/mobile-message-queue-management.spec.ts) +``` + +The Playwright tests run through the managed production build. Record the +expected RED assertion before production UI changes and the final discovered +and passing test counts after implementation. + +## Files likely touched + +- `apps/web/lib/types/http.ts` +- `apps/web/lib/state/app-state-types.ts` +- `apps/web/lib/state/slices/session/types.ts` +- `apps/web/lib/state/slices/session/session-slice.upsert.test.ts` +- `apps/web/lib/state/slices/session/session-slice.ts` +- `apps/web/lib/state/slices/session/remove-task-session.test.ts` +- `apps/web/lib/types/session-events.ts` +- `apps/web/lib/ws/handlers/agent-session.ts` +- `apps/web/lib/ws/handlers/agent-session.test.ts` +- `apps/web/lib/ws/handlers/tasks.ts` +- `apps/web/lib/ws/handlers/tasks.deleted.test.ts` +- `apps/web/hooks/use-task-removal.ts` +- `apps/web/hooks/use-task-removal.test.ts` +- `apps/web/lib/ws/handlers/workspaces.ts` +- `apps/web/lib/ws/handlers/workspaces.test.ts` +- `apps/web/lib/api/domains/queue-api.ts` +- `apps/web/lib/api/domains/queue-api.test.ts` +- `apps/web/hooks/domains/session/use-queue.ts` +- `apps/web/hooks/domains/session/use-queue.test.ts` +- `apps/web/components/task/chat/queued-ghost-list.tsx` +- `apps/web/components/task/chat/queued-ghost-list.test.tsx` +- `apps/web/components/task/chat/queued-ghost-panel-header.tsx` +- `apps/web/components/task/chat/queued-ghost-panel-header.test.tsx` +- `apps/web/components/settings/system/message-queue-settings.tsx` +- `apps/web/components/settings/system/message-queue-settings.test.tsx` +- `apps/web/src/locales/en/chat.json` +- `apps/web/src/locales/pt-pt/chat.json` +- `apps/web/src/locales/zh-cn/chat.json` +- `apps/web/src/locales/zh-hk/chat.json` +- `apps/web/src/locales/zh-tw/chat.json` +- `apps/web/src/locales/en/system.json` +- `apps/web/src/locales/pt-pt/system.json` +- `apps/web/src/locales/zh-cn/system.json` +- `apps/web/src/locales/zh-hk/system.json` +- `apps/web/src/locales/zh-tw/system.json` +- `apps/web/e2e/tests/chat/message-queue.spec.ts` +- `apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts` +- `apps/web/e2e/helpers/api-client.ts` +- `apps/web/e2e/tests/task/sidebar-queued-count.spec.ts` +- `apps/web/e2e/tests/chat/message-queue-scroll-helpers.ts` +- `apps/web/e2e/tests/chat/mid-turn-steering.spec.ts` +- `apps/web/e2e/tests/ssh/launch-task.spec.ts` +- `apps/web/e2e/tests/task/mobile-sidebar-queued-count.spec.ts` +- `apps/web/e2e/tests/workflow/workflow-queue-helpers.ts` +- `apps/web/e2e/tests/workflow/workflow-manual-move-queue.spec.ts` +- `apps/web/e2e/tests/workflow/mobile-workflow-manual-move-queue.spec.ts` + +## Dependencies + +- Task 01 must provide the queue status field and mutation action before the + final frontend and browser checks can pass. + +## Risks + +- Older queue events may omit the new fields; replacing instead of preserving + metadata would reset the visible switch. Complete tuples and unavailable + results can also arrive late, so every apply path must enforce status + epoch/generation, availability, source, and revision semantics. +- A delayed payload from a deleted session can restore stale source precedence + after textual ID reuse unless queue status is checked against the current + immutable session incarnation and every removal path clears queue metadata. +- Clearing workspace records before deriving affected sessions loses the only + local ownership join for empty-queue metadata; snapshot session IDs first and + reuse the session cleanup action. +- Omitting identity from get, a mutation, or a raw WebSocket helper lets delayed + work read or change a replacement. Typed/raw clients and every raw-helper + caller migrate to the server-issued triplet and never retry stale work. +- An unavailable status must reconcile entries without presenting an invented + OFF value or leaving the control actionable; a lower-generation unavailable + status must not disable newer success. +- A separate Auto-merge disabled expression can drift from Auto-run and leave + policy mutation clickable during queue or cancellation work; both use one + shared conflict predicate. +- Two compact pills plus queue actions may fit desktop but overlap at the mobile + breakpoint unless the logical groups wrap independently. +- Enlarging the shared switch itself could make desktop controls visually too + large; coarse-pointer sizing belongs on the effective hit area. +- Global-setting setup in E2E can leak between worker-scoped tests unless the + original value is restored in teardown. + +## Parallelism + +`sequential` + +## Inputs + +- `REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001` +- `REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001` +- `docs/specs/ui/system-design/message-queue-automation-controls.md` +- `apps/web/components/task/chat/queued-ghost-panel-header.tsx` +- `apps/web/hooks/domains/session/use-queue.ts` +- `apps/web/e2e/tests/chat/message-queue.spec.ts` +- `apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts` + +## Results + +Completed. Queue clients and store state now retain task/session/incarnation, +status ordering, policy availability, and operation generations. The expanded +header uses wrapping Auto-run and Auto-merge pills with localized accessible +state descriptions and coarse-pointer targets. Focused frontend tests, +typecheck, lint, desktop Playwright, and mobile-chrome Playwright pass. diff --git a/docs/plans/message-queue-session-automation/task-03-update-queue-automation-documentation.md b/docs/plans/message-queue-session-automation/task-03-update-queue-automation-documentation.md new file mode 100644 index 0000000000..80b58baeb2 --- /dev/null +++ b/docs/plans/message-queue-session-automation/task-03-update-queue-automation-documentation.md @@ -0,0 +1,147 @@ +--- +id: "03-update-queue-automation-documentation" +title: "Update Queue Automation Documentation" +status: completed +wave: 3 +depends_on: + - "01-persist-session-auto-merge-overrides" + - "02-deliver-queue-automation-pills" +plan: "plan.md" +requirements: + - REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001 + - REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001 +acceptance_criteria: + - AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.1 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.1 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.7 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.8 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.11 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.17 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.18 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.19 + - AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.24 +system_design: + - ../../specs/ui/system-design/message-queue-automation-controls.md +--- + +# Task 03: Update Queue Automation Documentation + +## Summary + +Reconcile public queue guides, configuration reference, and WebSocket reference +with the delivered session override and compact header controls. Remove stale +claims that capacity is always rejected before a compatible automatic fold. + +## In scope + +- Explain global inheritance and permanent per-session override precedence in + user-facing queue guidance. +- Describe the compact Auto-run and Auto-merge pills in the expanded queue + header without confusing message merging with pull-request Auto-merge. +- Correct capacity automatic-fold behavior wherever public docs describe queue + admission, including the staged-attachment exception. +- Add `queue_incarnation_id` to task-session fields. Document required + `task_id`, `session_id`, and `session_incarnation_id` on status get and every + queue mutation; the gateway/handler authorize that pair and pass the exact + triplet unchanged. Document stale non-retargeting errors, operation-bound + response/status snapshots, epoch/generation, availability, and effective + Auto-merge value/source/revision. Task-only recounts remain internal. +- Keep configuration, operations, coordination, and session guidance consistent + without copying detailed protocol text between pages. +- After implementation and behavioral checks pass, perform one lifecycle + cutover: activate both replacement requirements; deprecate the two superseded + requirements without changing their IDs or acceptance text; amend + `message-queue-management.md` with the delivered compatible-fold and + staged-attachment capacity exceptions; mark the system design current; and + update the UI specification indexes. + +## Out of scope + +- New screenshots, diagrams, or a new public page. +- Changes to GitHub or GitLab pull-request automation documentation. +- Production implementation details already owned by Tasks 01 and 02. + +## Acceptance + +- Public docs state that untouched sessions follow later global changes and an + explicitly changed session remains independent for that session's lifetime. +- Queue guidance names both compact pills, eligible direct folding at or above + capacity, and staged-attachment rejection, with no contradictory claim in + affected pages. +- The WebSocket reference documents exact action/field names, pair + authorization, required task/session/incarnation payloads for get and every + mutation, non-retargeting errors, operation-bound status snapshots, + unavailable ordering, task-only scope, incarnation matching, + source/revision, and effective-value semantics. Task-session responses expose + immutable queue incarnation. The final cutover leaves + replacement requirements active, superseded requirements deprecated with + forward links, the management capacity text aligned to delivered behavior, + and the system design current. + +## Verification + +```bash +python3 scripts/lint-spec-files.test.py +python3 scripts/lint-spec-files.py --all +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +git diff --check -- docs +git diff --cached --check -- docs +git ls-files -z --others --exclude-standard -- docs \ + | xargs -0 -r -n1 sh -c 'test -z "$(git diff --no-index --check /dev/null "$1" 2>&1)"' sh +``` + +Run all commands from the repository root. + +## Files likely touched + +- `docs/public/coordination.md` +- `docs/public/operations.md` +- `docs/public/sessions-and-review.md` +- `docs/public/configuration.md` +- `docs/specs/ui/requirements/message-queue-automation-controls.md` +- `docs/specs/ui/requirements/message-queue-auto-merge-session-overrides.md` +- `docs/specs/ui/requirements/message-queue-run.md` +- `docs/specs/ui/requirements/message-queue-management.md` +- `docs/specs/ui/requirements/message-queue-auto-merge.md` +- `docs/specs/ui/system-design/message-queue-automation-controls.md` +- `docs/specs/ui/README.md` +- `docs/public/websocket-api.md` + +## Dependencies + +- Task 01 defines the final protocol and persistence behavior. +- Task 02 defines the final user-facing labels and responsive composition. +- This task performs the lifecycle cutover only after Tasks 01 and 02 have + passed their behavioral verification; drafts must not become active earlier. + +## Risks + +- Existing pages describe both queued-message Auto-merge and pull-request + Auto-merge; edits must remain scoped to message queues. +- Repeating the complete precedence contract across every guide would create + another set of drift-prone sources instead of linking to the owning guidance. +- Activating replacements without deprecating and forward-linking the old + contracts would leave implementers with conflicting authoritative behavior. + +## Parallelism + +`sequential` + +## Inputs + +- `REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001` +- `REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001` +- `docs/specs/ui/system-design/message-queue-automation-controls.md` +- `docs/decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md` +- Current public queue sections in the files listed above. + +## Results + +Completed. Activated the replacement requirements and design, deprecated the +superseded queue contracts with forward links, recorded the inheritance +decision, and updated public configuration, coordination, operations, session, +and WebSocket documentation. Specification and public-doc validation pass. diff --git a/docs/public/coordination.md b/docs/public/coordination.md index 2728c60b4e..fa34163a9c 100644 --- a/docs/public/coordination.md +++ b/docs/public/coordination.md @@ -141,15 +141,16 @@ For predictable top-level creation, pass `repository_url`, `repository_id`, or ` | Created but not started | Starts the session with the message. | | Failed or cancelled | Returns an error. | -The default delivery mode is queued. Each session accepts 10 queued messages by default. An admin can change the install-wide limit under **Settings > Task Behavior > Message Queue**; `0` means unlimited. The saved value applies immediately to new admissions without removing messages already waiting. `KANDEV_QUEUE_MAX_PER_SESSION` has higher precedence, locks only the capacity field, and requires a restart when changed; zero or a negative value means unlimited. With **Auto-run** ON, one queued message runs per agent turn. When the cap is reached, the sender receives a structured `queue_full` error and should retry after space becomes available. +The default delivery mode is queued. Each session accepts 10 queued messages by default. An admin can change the install-wide limit under **Settings > Task Behavior > Message Queue**; `0` means unlimited. The saved value applies immediately to new admissions without removing messages already waiting. `KANDEV_QUEUE_MAX_PER_SESSION` has higher precedence, locks only the capacity field, and requires a restart when changed; zero or a negative value means unlimited. With **Auto-run** ON, one queued message runs per agent turn. At or above the cap, an eligible direct automatic fold into the pending tail may still succeed without creating another row. Other admissions return `queue_full`; staged attachments are rejected before any fold or attachment claim. -**Automatically merge consecutive messages** is enabled by default on the same settings card. After capacity admission succeeds, a new message may fold into the immediately preceding pending entry when both have the same strict source and compatible task, model, mode, metadata, attachments, and references. Otherwise it remains a separate FIFO entry. The earlier entry survives, so a successful admission may return an older queue-entry ID. The switch affects only later admissions and never sweeps rows already waiting. It is independent from **Enable queued message merging**, which controls the manual **Merge with above** action. +**Automatically merge consecutive messages** is enabled by default on the same settings card. Each session follows later changes to this install-wide value until its **Auto-merge** queue pill is changed. That first session change creates an explicit override that remains independent for the lifetime of that session, including across an empty queue and backend restart. For each new admission, Kandev resolves the effective session or global setting once. A compatible message may fold into the immediately preceding pending entry when both have the same strict source and compatible task, model, mode, metadata, attachments, and references. Otherwise it remains a separate FIFO entry. The earlier entry survives, so a successful admission may return an older queue-entry ID. The setting affects only later admissions and never sweeps rows already waiting. It is independent from **Enable queued message merging**, which controls the manual **Merge with above** action. -In the task workbench, expand the queue chip to manage pending messages. Every visible pending row has **Remove**, whether it came from a user, another agent, workflow automation, or a server action. **Clear all** removes all visible pending rows in that session and releases their capacity. After removal, merge, or drain, displayed positions compact to `#1` through `#N` while FIFO order stays unchanged. Provenance still matters for editing and merging: only user-origin content can be edited. A row already reserved for delivery is hidden from the panel and cannot be cancelled there. +In the task workbench, expand the queue chip to manage pending messages. The compact header places the **Auto-run** and **Auto-merge** pills beside the queue count; these control queued-message delivery and folding, not pull-request Auto-merge. Every visible pending row has **Remove**, whether it came from a user, another agent, workflow automation, or a server action. **Clear all** removes all visible pending rows in that session and releases their capacity. After removal, merge, or drain, displayed positions compact to `#1` through `#N` while FIFO order stays unchanged. Provenance still matters for editing and merging: only user-origin content can be edited. A row already reserved for delivery is hidden from the panel and cannot be cancelled there. Use the queue controls according to the outcome you want: - **Auto-run** is ON by default. ON runs eligible messages one at a time in FIFO order. OFF lets the current response finish, then holds every later message. The per-session setting survives an empty queue, reload, and backend restart. Turning it ON starts the head immediately when the session is promptable; clarification, workflow transitions, cancellation, and other lifecycle guards can defer delivery without changing the displayed ON state. +- **Auto-merge** shows the effective automatic-fold policy for this session. Changing it creates a permanent session override; untouched sessions continue to inherit the install-wide setting. - Every row's **Send Now** targets that message. It sends directly when the session is promptable; when an agent turn is active, it waits for backend cancellation acknowledgement and replaces that captured turn. A successful Send Now turns Auto-run ON, runs the selected row first, then continues the preserved remainder as separate FIFO turns. It does not record ordinary Cancel side effects, complete the cancelled workflow step, or move the task to review. - **Clear all** removes every visible pending row without sending a prompt. - **Cancel** in the chat toolbar stops the active turn immediately as a user cancellation. It may record the cancellation, complete an eligible workflow step, and move the task to review. It does not send queued content; when pending rows remain, Auto-run becomes OFF so they stay parked. diff --git a/docs/public/operations.md b/docs/public/operations.md index 21f3743575..b6ebf611de 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -120,13 +120,13 @@ Add `--system` to both commands for a system service. ## Message queue settings -Open **Settings > Task Behavior > Message Queue** to manage install-wide queue behavior. The default capacity is `10`; `0` means unlimited. Admin saves apply immediately to later admissions. Lowering the limit does not prune rows already waiting, so a queue at or above the new limit rejects new work until messages run or are removed. Delivery retries for work accepted before the change are not discarded by the lower cap. +Open **Settings > Task Behavior > Message Queue** to manage install-wide queue behavior. The default capacity is `10`; `0` means unlimited. Admin saves apply immediately to later admissions. Lowering the limit does not prune rows already waiting. At or above the new limit, only an eligible direct automatic fold into the existing tail can still succeed; other work is rejected until messages run or are removed. Staged attachments are rejected before a fold or claim. Delivery retries for work accepted before the change are not discarded by the lower cap. `KANDEV_QUEUE_MAX_PER_SESSION` has higher precedence than the saved capacity. A valid environment value makes only that field read-only; zero or a negative value means unlimited. Invalid text is logged and ignored in favor of the saved setting or default. Environment changes require a backend restart, while UI changes do not. -**Automatically merge consecutive messages** is on by default. Capacity is checked before any fold, so a full queue still rejects a compatible message. After admission, a new row folds only into its immediate pending predecessor when both rows have the same strict source and compatible task, model, mode, metadata, attachments, and references. Any mismatch or combined limit leaves the new row separate. The earlier row survives and its ID is returned. Turning the switch on does not compact existing rows. This setting is independent from **Enable queued message merging**, which controls the manual queue action. +**Automatically merge consecutive messages** is on by default. It is the fallback for every session that has not changed its compact **Auto-merge** queue pill. Untouched sessions follow later global changes. The first per-session change creates an explicit override that survives empty queues and restarts and remains independent for that session's lifetime. An eligible direct admission may fold into the immediate pending predecessor even at capacity because it does not create another row; staged-attachment admissions cannot use this path. Both rows must have the same strict source and compatible task, model, mode, metadata, attachments, and references. Any mismatch or combined limit leaves the new row separate when capacity permits. The earlier row survives and its ID is returned. Turning either setting on does not compact existing rows. This behavior is independent from **Enable queued message merging**, which controls the manual queue action. -To recover capacity in one session, expand its queue chip in the task workbench. **Remove** deletes one visible pending row and **Clear all** deletes all visible pending rows, including user-, agent-, workflow-, and server-origin work. After removal, merge, or drain, displayed positions immediately compact to `#1` through `#N`; durable FIFO ordering is unchanged. A row already reserved for delivery is hidden and is not cancelled by either action. +To recover capacity in one session, expand its queue chip in the task workbench. The compact header contains **Auto-run** and **Auto-merge** pills beside the count. **Remove** deletes one visible pending row and **Clear all** deletes all visible pending rows, including user-, agent-, workflow-, and server-origin work. After removal, merge, or drain, displayed positions immediately compact to `#1` through `#N`; durable FIFO ordering is unchanged. A row already reserved for delivery is hidden and is not cancelled by either action. ## State and storage diff --git a/docs/public/sessions-and-review.md b/docs/public/sessions-and-review.md index 1e45a8cc44..ff8cf943b2 100644 --- a/docs/public/sessions-and-review.md +++ b/docs/public/sessions-and-review.md @@ -63,11 +63,11 @@ Stopping is not deletion. Resume succeeds only while the executor still has the Stopping a turn does not itself run the next queued message. If pending rows remain, Kandev sets their session's **Auto-run** switch to OFF. Expand the queue and turn Auto-run ON when you want FIFO processing to continue. -The expanded queue also lets you pause or discard stale work. **Remove** is available for every visible pending row, including messages from users, peer agents, workflows, and server actions; **Clear all** removes all visible pending rows in that session. Only user-origin rows remain editable. A message already reserved for delivery is hidden from the queue and cannot be cancelled with these controls. +The expanded queue also lets you pause or discard stale work. Its compact header places the **Auto-run** and **Auto-merge** pills beside the queue count. **Remove** is available for every visible pending row, including messages from users, peer agents, workflows, and server actions; **Clear all** removes all visible pending rows in that session. Only user-origin rows remain editable. A message already reserved for delivery is hidden from the queue and cannot be cancelled with these controls. -Use **Auto-run** for normal queue motion. ON runs one eligible row per turn in FIFO order; OFF lets the current response finish and holds later rows. The setting belongs to the session and survives an empty queue, reload, and backend restart. A pending clarification or another lifecycle guard can leave the queue waiting while the switch remains ON. +Use **Auto-run** for normal queue motion. ON runs one eligible row per turn in FIFO order; OFF lets the current response finish and holds later rows. The setting belongs to the session and survives an empty queue, reload, and backend restart. A pending clarification or another lifecycle guard can leave the queue waiting while the switch remains ON. **Auto-merge** controls automatic folding of later compatible admissions. It initially follows the install-wide value and follows later global changes until you change the pill. That first change creates a session override that remains independent for the session's lifetime. -Every row has **Send Now** for targeted priority. It sends that row directly when the session is promptable or replaces the captured active turn after backend cancellation acknowledgement. A successful Send Now turns Auto-run ON, runs the selected row first, then continues the remaining rows as separate FIFO turns without ordinary Cancel side effects. **Clear all** discards the visible queue. The chat toolbar's **Cancel** immediately stops the active turn, sends no queued prompt, parks any pending backlog with Auto-run OFF, and may complete an eligible workflow step or move the task to review. +Every row has **Send Now** for targeted priority. It sends that row directly when the session is promptable or replaces the captured active turn after backend cancellation acknowledgement. A successful Send Now turns Auto-run ON, runs the selected row first, then continues the remaining rows as separate FIFO turns without ordinary Cancel side effects. **Clear all** discards the visible queue. The chat toolbar's **Cancel** immediately stops the active turn, sends no queued prompt, parks any pending backlog by turning Auto-run OFF, and can complete the workflow step or move the task to review. A CLI-passthrough profile displays the agent's native terminal interface in a PTY. It still belongs to the task, but it does not provide Kandev's structured chat messages and tool-call presentation. @@ -101,9 +101,9 @@ Delivery follows the target state: Without an explicit session ID the message goes to the primary session, and falls back to the newest session that can still take a message when the primary is cancelled or failed. A session named explicitly is never redirected. When every session is terminal the call fails and names `spawn_session_kandev`. -The default pending-message limit is 10 per session. An admin can change it live under **Settings > Task Behavior > Message Queue**; `0` removes the cap. A valid `KANDEV_QUEUE_MAX_PER_SESSION` value takes precedence and makes only the capacity field read-only; changing the environment still requires a restart. Malformed environment values are logged and ignored, so the saved setting or default applies instead. Lowering the saved limit does not delete entries already waiting. +The default pending-message limit is 10 per session. An admin can change it live under **Settings > Task Behavior > Message Queue**; `0` removes the cap. A valid `KANDEV_QUEUE_MAX_PER_SESSION` value takes precedence and makes only the capacity field read-only; changing the environment still requires a restart. Malformed environment values are logged and ignored, so the saved setting or default applies instead. Lowering the saved limit does not delete entries already waiting. An eligible direct automatic fold may still succeed at or above capacity because it does not add a row; other admissions are rejected, and staged attachments are rejected before any fold or claim. -The same card enables **Automatically merge consecutive messages** by default. Once capacity admission succeeds, compatible consecutive messages from the same strict source fold into the earlier pending entry; incompatible messages remain separate. The earlier entry's ID survives, and only admissions after the setting is read are eligible. This behavior is independent from the manual **Enable queued message merging** switch. Interrupt delivery is restricted to a direct parent task messaging its child. Other senders must queue. +The same card enables **Automatically merge consecutive messages** by default. Untouched sessions inherit this value and later changes to it. Changing a session's **Auto-merge** pill creates an explicit override for that session's lifetime. Compatible consecutive messages from the same strict source fold into the earlier pending entry; incompatible messages remain separate when capacity permits. The earlier entry's ID survives, and only admissions after the effective setting is read are eligible. This behavior is independent from the manual **Enable queued message merging** switch. Interrupt delivery is restricted to a direct parent task messaging its child. Other senders always queue, and only user-origin rows may be edited or manually merged. For urgent replacement work, the parent should use `message_task_kandev` with `delivery_mode: "interrupt"`; this cancels the current approach and immediately tries to dispatch the new prompt, with a safe queued fallback. Use `stop_task_kandev` only for halt-only intent. A successful stop marks every accepted live child session `CANCELLED` and schedules graceful teardown asynchronously. Kandev then attempts to move an eligible unarchived, non-Office task from `IN_PROGRESS` or `SCHEDULING` to `REVIEW`; other task states remain unchanged. A child with no live execution returns idempotent `not_running`, and its worktrees, environment, commits, task record, descendants, and queued messages are preserved. See [Coordination](coordination.md) for the complete authority and lifecycle contract. diff --git a/docs/public/websocket-api.md b/docs/public/websocket-api.md index fab6b6c474..cc3794ab2c 100644 --- a/docs/public/websocket-api.md +++ b/docs/public/websocket-api.md @@ -226,6 +226,8 @@ If `intent` is omitted, the backend infers it from those fields. That inference A successful response contains `success`, `task_id`, `state`, and usually `session_id`; it can also contain `agent_execution_id`, `worktree_path`, and `worktree_branch`. Session states use uppercase values such as `CREATED`, `STARTING`, `RUNNING`, `WAITING_FOR_INPUT`, `COMPLETED`, `FAILED`, and `CANCELLED`. +Every task-session response includes immutable `queue_incarnation_id`. Kandev generates a new value when a session is created, including when a deleted textual session ID is reused. Queue clients must retain this value with the task and session IDs rather than looking up a replacement after starting an operation. + ### Search work-item references over HTTP The structured chat composer's `#` search calls `GET /api/v1/workspaces/:workspaceId/mentions/search?q=&limit=&exclude_task_id=`. `q` is required after trimming and accepts 1–200 Unicode characters. `limit` defaults to 5 and is clamped to 1–10. When supplied, `exclude_task_id` must belong to the requested workspace. @@ -253,9 +255,13 @@ A successful response returns the normalized query and an ordered `groups` array `message.queue.add` accepts the same optional array. `message.queue.update` also accepts it and treats the array as a replacement: send `[]` after removing every generated reference link so stale metadata is cleared. Queue status and message responses return validated entries under `metadata.entity_references`. -Ordinary `message.queue.add` admissions use the default-on automatic-merge setting. Capacity is enforced first. The newly admitted row folds only into its immediate pending predecessor when both rows have the same strict source and compatible task, model, plan mode, metadata, attachments, context files, and entity references; the merged context-file union is capped at 200 descriptors. Otherwise it remains separate. A successful fold returns the surviving earlier row's `entry_id`, while a fallback returns the new row's ID. The final state is published through `message.queue.status_changed`. Automatic merging applies only to later ordinary admissions, not existing rows, explicit append/coalesce/restore/retry/send-now operations, or the independent manual merge action. +Every browser queue read or mutation requires the exact `task_id`, `session_id`, and `session_incarnation_id` triplet. This applies to `message.queue.get`, `message.queue.add`, `message.queue.append`, `message.queue.cancel`, `message.queue.drain`, `message.queue.update`, `message.queue.remove`, `message.queue.merge`, `message.queue.reorder`, `message.queue.send_now`, `message.queue.auto_run.set`, and `message.queue.auto_merge.set`. The gateway authorizes the task/session pair and the queue layer compares the supplied incarnation before acting. A deleted or replaced session returns the same non-enumerating not-found response as an inaccessible session; the request is not redirected to a new session with the same textual ID and clients must not retry it against a freshly looked-up identity. -`message.queue.merge` folds a queued entry into the entry directly above it. The payload requires `session_id` and `entry_id` (the source entry being merged away); `user_id` is optional and defaults to `user`. On success the response carries the surviving merged entry's `entry_id` (the target's id) and the server broadcasts an updated `message.queue.status_changed`. The request is rejected with `entry_not_found` when the entry was already drained or is not owned by the caller, with a validation error when no mergeable entry exists above it, and with `merge_reference_overflow` when the combined entity-reference lists would exceed the per-message cap (the merge is rejected atomically; neither row changes). +Ordinary `message.queue.add` admissions use one effective automatic-merge policy snapshot. An untouched session inherits the current install-wide value and its global revision. `message.queue.auto_merge.set` accepts required `enabled` and creates a persistent explicit session override; later global changes no longer affect that session. The newly admitted row folds only into its immediate pending predecessor when both rows have the same strict source and compatible task, model, plan mode, metadata, attachments, context files, and entity references; the merged context-file union is capped at 200 descriptors. An eligible direct fold can succeed at or above capacity because it adds no row. A staged-attachment admission at or above capacity is rejected before a fold or attachment claim. Otherwise, capacity or incompatibility leaves the admission rejected or separate as applicable. A successful fold returns the surviving earlier row's `entry_id`, while a separate admission returns the new row's ID. Automatic merging applies only to later ordinary admissions, not existing rows, explicit append/coalesce/restore/retry/send-now operations, or the independent manual merge action. + +`message.queue.get`, the Auto-merge mutation response, and `message.queue.status_changed` expose `task_id`, `session_id`, `session_incarnation_id`, `status_epoch`, `status_generation`, `auto_merge_available`, `auto_merge_enabled`, `auto_merge_source`, and `auto_merge_revision` with the queue entries and existing Auto-run/manual-merge fields. `status_epoch` is the session incarnation. Within an epoch, clients apply only increasing status generations. `auto_merge_available:false` means the backend could not establish authoritative policy and the control must remain disabled; it is not an OFF value. A `session` source is an explicit override; a `global` source remains inherited. Session and global revisions are monotonic within their own source. Internal task-only recount events have explicit task scope and are not session status. + +`message.queue.merge` folds a queued entry into the entry directly above it. In addition to the required identity triplet, its payload requires `entry_id` (the source entry being merged away); `user_id` is optional and defaults to `user`. On success the response carries the surviving merged entry's `entry_id` (the target's ID) and the server broadcasts an updated `message.queue.status_changed`. The request is rejected with `entry_not_found` when the entry was already drained or is not owned by the caller, with a validation error when no mergeable entry exists above it, and with `merge_reference_overflow` when the combined entity-reference lists would exceed the per-message cap. The overflow rejection is atomic; neither row changes. If the agent is busy, use the `message.queue.*` operations rather than retrying `message.add`. Permission prompts are represented in persisted/session message data; answer one with `permission.respond`. Its payload requires `session_id` and `pending_id`, plus `option_id` unless `cancelled:true`; optional `rejected:true` distinguishes an explicit denial from dismissing the prompt. @@ -354,11 +360,15 @@ message.add message.list message.queue.add message.queue.append +message.queue.auto_merge.set +message.queue.auto_run.set message.queue.cancel message.queue.drain message.queue.get message.queue.merge message.queue.remove +message.queue.reorder +message.queue.send_now message.queue.update message.search diff --git a/docs/specs/ui/README.md b/docs/specs/ui/README.md index 115a162e17..ca735dd063 100644 --- a/docs/specs/ui/README.md +++ b/docs/specs/ui/README.md @@ -61,12 +61,14 @@ UI owns responsive behavior; other systems own behavior/state. - [Mermaid Rendering](requirements/mermaid-rendering.md) - [Message favorite star mobile sizing](requirements/message-favorite-star-mobile-size.md) - [Metadata dialog](requirements/message-metadata-overflow.md) -- [Queue auto-merge](requirements/message-queue-auto-merge.md) +- [Queue auto-merge (deprecated)](requirements/message-queue-auto-merge.md) +- [Queue auto-merge session overrides (active)](requirements/message-queue-auto-merge-session-overrides.md) +- [Message queue automation controls (active)](requirements/message-queue-automation-controls.md) - [Message queue management](requirements/message-queue-management.md) - [Merge queued](requirements/message-queue-merge.md) - [Pin the Message Queue Panel](requirements/message-queue-pin.md) - [Reorder Queued Messages](requirements/message-queue-reorder.md) -- [Pending auto-run](requirements/message-queue-run.md) +- [Pending auto-run (deprecated)](requirements/message-queue-run.md) - [Send Queued Messages Now](requirements/message-queue-send-now.md) - [Mobile topbar actions](requirements/mobile-quick-chat-topbar.md) - [Mobile Task Chrome](requirements/mobile-task-chrome.md) @@ -165,6 +167,7 @@ UI owns responsive behavior; other systems own behavior/state. - [File Tree Chat](system-design/file-tree-chat-context.md) - [MCP results](system-design/kandev-mcp-tool-results.md) - [Message queue row controls](system-design/message-queue-row-controls.md) +- [Message Queue Automation Controls (current)](system-design/message-queue-automation-controls.md) - [Native HTML File Preview](system-design/native-html-preview.md) - [Mobile Task Chrome](system-design/mobile-task-chrome.md) - [Persistent status motion](system-design/persistent-status-motion.md) diff --git a/docs/specs/ui/requirements/message-queue-auto-merge-session-overrides.md b/docs/specs/ui/requirements/message-queue-auto-merge-session-overrides.md new file mode 100644 index 0000000000..cbcaffd955 --- /dev/null +++ b/docs/specs/ui/requirements/message-queue-auto-merge-session-overrides.md @@ -0,0 +1,189 @@ +--- +status: active +system: ui +created: 2026-09-04 +owners: + - kandev +--- + +# Per-session Automatic Message Merge Override Requirements + +## Overview + +Automatic message merging reduces avoidable agent turns by folding compatible +consecutive prompts from one source. Install administrators set the default, +while each task session can retain an explicit override from the expanded queue +panel. + +This requirement supersedes +`REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-001` when activated. It preserves the existing +compatibility and content/entity fold semantics while replacing the +install-wide-only policy, aligning queued-time behavior with the shipped +automatic fold, adding a session control, and allowing a compatible direct fold +at or above the persisted-row capacity. + +## Terminology + +- **Global value:** The current install-wide automatic-merge setting. +- **Session override:** An explicit ON or OFF value saved after a user changes + the Auto-merge switch for one task session. +- **Effective value:** The session override when one exists; otherwise the + current global value. +- **Staged-attachment admission:** An admission that must create a source row + before it can claim uploaded attachments into session ownership. + +## Requirements + +### REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001: Per-session Automatic Message Merge Override + +**Intent:** Let users override the install default for the session they are +actively managing without making untouched sessions stale copies of that +default. + +#### Acceptance criteria + +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.1:** \*\*Settings > Task Behavior + > Message Queue** shall expose an **Automatically merge consecutive + > messages\*\* switch whose install-wide default is ON. Its localized description + > shall explain that sessions inherit this value until changed in the session's + > queue panel. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.2:** A session without an + explicit override shall use the current global value. When the global value + changes, existing sessions that have never changed Auto-merge shall use the + new value for later admissions. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.3:** The first successful + Auto-merge change in a session shall persist an explicit ON or OFF override + for that session. Later global changes shall not alter its effective value. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.4:** The expanded queue header + shall expose the effective value through the labeled Auto-merge switch pill + defined by `REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001`. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.5:** The effective value shall + apply only to messages admitted after it is resolved. Changing it shall not + compact or split entries already pending. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.6:** When effective Auto-merge + is ON, a newly admitted message shall fold into the pending tail only when the + entries are consecutive and compatible under + `REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-001`. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.7:** At or above a positive + persisted-row limit, a compatible admission that does not require a staged + attachment claim shall fold directly into the tail without increasing the + row count. An incompatible admission or an admission with effective + Auto-merge OFF shall fail with `queue_full` and leave the queue unchanged. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.8:** At or above a positive + persisted-row limit, a staged-attachment admission shall fail with + `queue_full` before claiming the attachment or changing the prior tail. It + shall not use the direct candidate-fold path. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.9:** A successful automatic fold + shall preserve the earlier tail's identity, FIFO position, task, model, + plan-mode value, and sender identity while appending compatible content and + additive context in target-then-source order. Its queued time shall become + the later of the target and source queued times so pending activity reflects + the newest accepted prompt. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.10:** Automatic merging shall + remain independent from manual **Merge with above**. The two effective + switches may differ without enabling or disabling each other's behavior. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.11:** A saved session override + shall survive an empty queue, navigation, reload, and backend restart. A new + task session shall have no override and shall inherit the current global + value until changed. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.12:** A failed override mutation + shall restore authoritative state and show a localized error. If the backend + cannot read a session override during admission, that admission shall fail + closed by skipping automatic merge rather than falling back to ON. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.13:** A delayed queue snapshot, + successful status, or unavailable-status result shall not replace or disable + newer authoritative Auto-merge state for the same task-session incarnation. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.14:** Desktop and mobile shall + expose the same effective state, mutation, persistence, error, and global + precedence outcomes without document horizontal overflow or nested queue + scrolling. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.15:** If the backend cannot + resolve Auto-merge while reading queue status, it shall still reconcile queue + entries but shall report the policy as unavailable rather than inventing an + effective value. The client shall apply availability only from the newest + status read, preserve any last authoritative value, and disable Auto-merge + until a newer successful status restores availability. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.16:** An override shall belong + to the exact task-session identity. Deleting that session, deleting its task, + or deleting its workspace shall remove its queue-owned state atomically with + the destructive lifecycle operation, so later reuse of the identity starts + without an override. Archiving a task shall preserve the state because its + session identity remains; workflow transfer shall remove the source state + without copying its override over the destination's independent state. A + post-delete queue recount shall be task-scoped, shall not carry the deleted + session ID, and shall not be delivered to browser session reconciliation. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.17:** Every created task session + shall receive an opaque server-generated incarnation identity that remains + stable for that row's lifetime and changes if the same session ID is reused. + Legacy rows shall receive distinct identities through a replay-safe migration + that preserves them through every task-session table rebuild. + Session-scoped queue status shall carry that identity. The client shall + discard status for a different incarnation, clear queue metadata when the + session is removed, and reset source/revision ordering when the current + session ID is recreated with a new incarnation. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.18:** Every ordinary queue + admission and Auto-merge override mutation shall carry the caller's expected + session incarnation into the persistence boundary. Browser + `message.queue.add` and `message.queue.append` requests shall require that + token, and every internal ordinary or lifecycle-coalescing producer capable + of inserting a row shall use the same typed identity contract. The backend + shall acquire the owning task lock before the queue session lock, confirm the + exact task/session/incarnation still exists, and reject a missing or changed + incarnation through the existing non-enumerating session-not-found response + without appending, inserting, replacing, folding, claiming attachments, or + writing policy. Session deletion shall use the same lock order, validation, + and atomic queue-state cleanup. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.19:** A browser queue read shall + carry its expected task/session/incarnation through authorization and the + identity-bound snapshot. Every session-scoped status shall label entries and + policy with only that validated identity. Deletion or recreation during the + read shall yield an old-incarnation snapshot or no payload, never replacement + state or old rows labeled with the replacement incarnation. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.20:** Queue mutation/loading + state shall belong to an exact session incarnation and operation generation. + Completion or cleanup from an older incarnation or operation shall not clear + a newer operation or re-enable controls for a replacement session that + reused the same textual ID. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.21:** Queue persistence shall + fail closed unless an injected task-session authority validates identities + inside its mutation transaction or memory critical section. A session + transfer shall validate exact source and destination task/session/incarnation + identities after locking sorted task rows and then sorted queue-session locks + in one transaction. A mismatch shall move no rows or policy, and concurrent + transfer and destructive lifecycle operations shall not deadlock or leave + old-incarnation rows. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.22:** An admission that claims + staged attachments shall validate its exact session identity, insert the + queue row, claim attachments, and perform any automatic fold in one + task-row-then-session-lock transaction or memory critical section. Any claim + or identity failure shall roll back every queue and attachment change. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.23:** Deferred transition apply + shall atomically CAS the captured task/session/incarnation, pending move, + expected source, and target, then return an immutable effect token containing + the committed task transition generation and queue-session identity. + Post-commit task effects shall CAS that generation; session/queue effects + shall validate the token's identity and generation. Recreation or a successor + move shall leave its state unchanged. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.24:** Every session-scoped queue + mutation and its resulting response or status shall retain the caller's + expected task/session/incarnation. A stale operation shall not add, append, + reserve, acknowledge, drain, send, update, remove, merge, reorder, clear, + restore, change policy, or publish replacement-session state. +- **AC-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001.25:** Queue work, including + deferred-transition work, that continues after a claim or transaction shall + retain immutable session identity and operation generation through + cancellation, prompt dispatch, restore, acknowledgement, and terminal + cleanup. Delayed old work shall not prompt, restore into, block, or clear + in-flight ownership for a replacement session. + +## Exclusions + +- No workspace-level, task-level, or user-level override. +- No Follow global or reset-to-global control. +- No inherited-versus-explicit badge in the queue header. +- No retroactive compaction or reconstruction of pending messages. +- No transfer of an override from one task session to another. + +## Related decision + +[Inherit Queue Auto-merge Until Overridden](../../../decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md) diff --git a/docs/specs/ui/requirements/message-queue-auto-merge.md b/docs/specs/ui/requirements/message-queue-auto-merge.md index f7a15bd82a..7a920baa2c 100644 --- a/docs/specs/ui/requirements/message-queue-auto-merge.md +++ b/docs/specs/ui/requirements/message-queue-auto-merge.md @@ -1,16 +1,19 @@ --- -status: active +status: deprecated system: ui created: 2026-08-12 owners: - kandev --- + # Automatically Merge Consecutive Queued Messages Requirements ## Overview Users and coordinating agents often send several short follow-ups while a session is busy. Delivering every follow-up as a separate turn adds latency and queue noise even when the messages came from the same source and can safely be handled together. +Deprecated by [Per-session Automatic Message Merge Override](message-queue-auto-merge-session-overrides.md), which preserves compatibility rules while adding inheritance, session overrides, and the direct-fold capacity exception. + ## Requirements ### REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-001: Automatically Merge Consecutive Queued Messages diff --git a/docs/specs/ui/requirements/message-queue-automation-controls.md b/docs/specs/ui/requirements/message-queue-automation-controls.md new file mode 100644 index 0000000000..a0aff6c900 --- /dev/null +++ b/docs/specs/ui/requirements/message-queue-automation-controls.md @@ -0,0 +1,81 @@ +--- +status: active +system: ui +created: 2026-09-04 +owners: + - kandev +--- + +# Message Queue Automation Controls Requirements + +## Overview + +The expanded queue panel must present queue progress and both session automation +policies without spending a second row on one control. Auto-run retains its +backend-owned behavior while Auto-merge gains a session-scoped control. + +This requirement supersedes `REQ-UI-MESSAGE-QUEUE-RUN-001` when activated. It +preserves that requirement's queue execution behavior while replacing its +header composition and visible-help contract. + +## Requirements + +### REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001: Message Queue Automation Controls + +**Intent:** Keep queue automation visible, compact, accessible, and equivalent +on desktop and mobile. + +#### Acceptance criteria + +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.1:** When the queue panel is + expanded, one responsive header composition shall contain the queue summary, + labeled Auto-run and Auto-merge switch pills, and the existing queue actions. + Layouts with enough width shall keep the groups on the first visual line; + narrower layouts may wrap groups but shall not render a dedicated second + Auto-run row. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.2:** The Auto-run switch shall + expose its checked state, disabled state, keyboard focus, and localized ON or + OFF behavior description to assistive technology. Its state-specific + description need not remain visually present after the compact header + replaces the prior helper row. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.3:** Auto-run ON shall allow the + backend to start the FIFO head whenever the session is eligible and continue + with one distinct turn per entry after each preceding turn completes. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.4:** Auto-run OFF shall prevent + every later automatic FIFO take without cancelling, truncating, or altering + an active turn or a queued handoff accepted before OFF won the race. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.5:** Turning Auto-run ON while + the session is promptable shall attempt the FIFO head immediately. When the + session is busy or another lifecycle guard applies, ON shall persist and + delivery shall resume at the next eligible trigger. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.6:** Auto-run shall remain + backend-owned per session, default to ON, survive an empty queue, navigation, + reload, and backend restart, and remain OFF until an explicit resume changes + it. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.7:** Every visible queue row + shall retain Send Now. A successful Send Now shall enable Auto-run, dispatch + the selected entry first, and preserve the relative FIFO order of the + remainder. The first-party header shall not add a bulk Send Now action. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.8:** A switch shall be disabled + while its mutation or another conflicting queue or cancellation operation is + pending. A failed mutation shall restore authoritative state and show a + localized error. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.9:** Desktop and mobile shall + expose the same controls and outcomes. Header groups shall wrap without + document horizontal overflow; coarse-pointer effective targets shall be at + least 44 CSS pixels in each dimension. +- **AC-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001.10:** Reorder, edit, manual + merge, Remove, Clear all, pin, collapse, and the queue panel's single internal + scroll owner shall retain their existing behavior. + +## Exclusions + +- No mobile-only drawer or alternate queue route. +- No aggregate first-party Send Now control. +- No inherited-versus-explicit badge or reset-to-global control. + +## Related requirements + +- [Control Pending Message Auto-run](message-queue-run.md) +- [Per-session Automatic Message Merge Overrides](message-queue-auto-merge-session-overrides.md) +- [Send Queued Messages Now](message-queue-send-now.md) diff --git a/docs/specs/ui/requirements/message-queue-management.md b/docs/specs/ui/requirements/message-queue-management.md index a891ee565f..f99fa60e03 100644 --- a/docs/specs/ui/requirements/message-queue-management.md +++ b/docs/specs/ui/requirements/message-queue-management.md @@ -5,6 +5,7 @@ created: 2026-08-03 owners: - kandev --- + # Manage Pending Message Queues Requirements ## Overview @@ -75,9 +76,11 @@ remains full. Those same rows have no individual remove action. automatic merge switches. - The default is `10`. A positive integer sets a cap; `0` means unlimited. - A saved setting applies immediately to later admissions. Existing entries - are never trimmed. If a queue already exceeds a newly lowered limit, new - messages are rejected with `queue_full` until its persisted count is below - the limit. + are never trimmed. At or above a positive cap, an eligible direct automatic + fold into the pending tail may still succeed because it does not add a row. + Other admissions are rejected with `queue_full` until the persisted count is + below the limit. A staged-attachment admission cannot use the direct-fold + exception and is rejected before attachment claim or tail mutation. - Previously accepted work may be restored or retried after a delivery failure even when the new cap is lower. Capacity limits new work; it does not turn a failed delivery into message loss. diff --git a/docs/specs/ui/requirements/message-queue-run.md b/docs/specs/ui/requirements/message-queue-run.md index 849807ba43..885ab06ded 100644 --- a/docs/specs/ui/requirements/message-queue-run.md +++ b/docs/specs/ui/requirements/message-queue-run.md @@ -1,16 +1,19 @@ --- -status: active +status: deprecated system: ui created: 2026-08-16 owners: - kandev --- + # Control Pending Message Auto-run Requirements ## Overview The queue header presents **Run next** and bulk **Send Now** as separate ways to move pending work. **Run next** directly takes only one FIFO head, but normal turn completion then keeps taking later entries. The label describes one backend step rather than the user's real outcome: putting the queue in motion. +Deprecated by [Message Queue Automation Controls](message-queue-automation-controls.md), which preserves the execution behavior and replaces the expanded-header contract. + ## Requirements ### REQ-UI-MESSAGE-QUEUE-RUN-001: Control Pending Message Auto-run diff --git a/docs/specs/ui/system-design/message-queue-automation-controls.md b/docs/specs/ui/system-design/message-queue-automation-controls.md new file mode 100644 index 0000000000..d025bba8eb --- /dev/null +++ b/docs/specs/ui/system-design/message-queue-automation-controls.md @@ -0,0 +1,562 @@ +--- +status: current +system: ui +requirements: + - REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001 + - REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001 +--- + +# Message Queue Automation Controls System Design + +## Purpose and boundaries + +This design owns the task-workbench Auto-run and Auto-merge controls and their +end-to-end queue policy projection. The UI owns the operator outcomes; the +backend message queue remains the source of truth. + +It changes no compatibility rule, manual merge operation, capacity value, Send +Now behavior, or Auto-run state machine. An eligible message can fold into the +tail at or above the positive row limit; staged attachments still require +insertion capacity. + +## Requirement mapping + +| Requirement | Design sections | +| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `REQ-UI-MESSAGE-QUEUE-AUTOMATION-CONTROLS-001` | [Queue header composition](#queue-header-composition), [Frontend state and mutation flow](#frontend-state-and-mutation-flow) | +| `REQ-UI-MESSAGE-QUEUE-AUTO-MERGE-OVERRIDE-001` | [Policy resolution](#policy-resolution), [Persistence](#persistence), [Admission flow](#admission-flow), [Status ordering](#status-ordering) | + +## Components and responsibilities + +| Component | Responsibility | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `internal/system/queuesettings.Service` | Persists the install-wide automatic-merge value and revision together, then applies that committed policy to the queue service. | +| `internal/orchestrator/messagequeue.Service` | Owns one atomic immutable global-policy snapshot and one process status epoch/generation source, resolves the effective session value, serializes session override writes with admissions, and applies the snapshotted value to automatic folding. | +| `internal/orchestrator/messagequeue.Repository` | Persists the nullable session override and its monotonic revision for memory, SQLite, and PostgreSQL implementations. | +| Session-scoped queue-status producers | Receive an expected `QueueSessionIdentity` and publish only its identity-bound snapshot. | +| `internal/orchestrator/handlers.QueueHandlers` | Authorizes the supplied task/session pair, passes the unchanged triplet to queue operations, and publishes status for that identity. | +| `pkg/websocket` | Defines queue action names. | +| `useQueue` and the queue session slice | Supply current identities and order, reconcile, preserve, and disable queue state. | +| `QueuePanelHeader` | Renders the compact automation pills with responsive and accessible interaction. | + +## Policy resolution + +Automatic merge has two inputs: + +```text +explicit session override present -> use override +no session override -> use current global value +session override read fails -> disable merge for this admission +``` + +`messagequeue.Service` replaces its independent automatic-merge boolean with one +immutable `GlobalAutoMergePolicy { Enabled, Revision }` stored through an +`atomic.Pointer`. `SetGlobalAutoMergePolicy(enabled, revision)` constructs and +publishes one new snapshot only after the settings value and revision commit. +Admissions and status reads load that pointer once; they cannot observe a value +paired with a revision from another settings generation. Reads allocate +nothing, while the infrequent settings update allocates one immutable snapshot. + +At process startup, `backendapp/orchestrator.go` receives the persisted enabled +value and global revision from `queuesettings.Resolution` and calls +`SetGlobalAutoMergePolicy` before queue admissions become available. Startup +never reconstructs the runtime snapshot from the boolean alone. Reconnect tests +restart with a persisted nonzero revision and verify that the first status uses +the same pair. + +Kandev has one orchestrator owner per installation, so the immutable global +snapshot is process-local under that deployment invariant. Each ordinary +admission carries `{task_id, session_id, expected_incarnation}` and loads the +global snapshot once before calling the repository operation that owns the +durable policy linearization point. SQL first locks the active task row, then +the `queue_session_locks` row, revalidates the exact task/session/incarnation, +and reads the nullable override plus inserts or directly folds in that +transaction. Override writes receive the same identity tuple and use the same +lock order and CAS validation. The result carries the one effective snapshot +used by later post-insert work. The service admission gate preserves +in-process call order but is not the cross-connection correctness boundary. + +A session override committed before the repository admission lock is acquired +governs that admission. An override committed after the policy-and-insert +linearization point governs the next admission, even when the current admission +still has a staged-attachment claim and post-insert fold to complete. A +concurrent in-process global update is likewise ordered by the single atomic +load; it never changes policy halfway through one admission. + +Updating the global snapshot does not scan or write session rows. Consequently +every session without an override uses the new committed global policy on its +next admission and successful queue-status read, while an overridden session +stays independent. + +A user mutation always writes an explicit boolean. There is no clear-override +WebSocket operation or third UI state. + +## Persistence + +Queue-owned state gains a nullable override and revision: + +```sql +CREATE TABLE queue_session_state ( + session_id TEXT PRIMARY KEY, + auto_run INTEGER NOT NULL DEFAULT 1, + auto_merge_override INTEGER NULL, + auto_merge_revision INTEGER NOT NULL DEFAULT 0 +); +``` + +`NULL` or a missing row means inherit. `0` means explicit OFF and `1` means +explicit ON. `auto_merge_revision` increments atomically on every successful +session override write. Existing rows migrate to `NULL` and revision `0`, so +upgrades preserve global inheritance. Fresh schema creation and the replay-safe +`ALTER TABLE` path define both columns using the repository's established +cross-dialect binding and duplicate-column handling. + +`task_sessions` gains an opaque `queue_incarnation_id`. `CreateTaskSession` +generates a UUID once; it never changes while that row exists. The migration is +registered after both existing `task_sessions` table-rebuild migrations and +then adds the column and backfills every empty legacy row with a distinct +server-generated UUID. A restart resumes empty-row backfill without replacing +completed values. Fresh schema requires a non-empty value, and any future table +rebuild must project the column unchanged. Recreating a deleted textual session +ID therefore produces a different incarnation without retaining queue policy +state. + +The persisted install-wide message-queue settings envelope also gains an +internal `auto_merge_revision`. It defaults to `0` for legacy values and +increments atomically only when `auto_merge_enabled` changes. Public settings +responses retain their existing shape; the revision is queue-status ordering +metadata, not an editable setting. + +The memory repository stores override presence, value, and revision under its +existing mutex. SQL override writes increment and return the revision while +holding the durable session lock used by policy-aware admission. The settings +store saves the global value and revision in one write. Only after that commit +does the settings service publish the same immutable pair to +`messagequeue.Service`; a failed save leaves the prior runtime snapshot +unchanged. + +`queuesettings.Resolution` carries the internal global revision alongside its +public configured and effective views. The backend application startup wiring +hydrates `messagequeue.Service` with that persisted pair. Live settings saves +and startup therefore use the same typed application boundary rather than a +boolean-only initialization path. + +The queue package defines required authority seams instead of querying task +tables itself. `SQLSessionAuthority` exposes +`LockQueueTaskRows(ctx, tx, taskIDs)`, +`ValidateQueueSessions(ctx, tx, identities)`, and +`ClaimQueueAttachments(ctx, tx, identity, claim)`. +`QueueAttachmentClaim` carries authenticated owner/workspace and staged IDs. +`MemorySessionAuthority.WithQueueSessionIdentities` supplies an equivalent +claim-capable boundary to its callback. The task SQLite repository implements +the SQL seam on the queue transaction; production injects it into +`NewSQLiteRepository`. SQL fixtures use task/session/attachment rows, while +memory fixtures inject a seeded locked authority. Neither constructor accepts +nil; `tasksTablePresent` and missing-schema fail-open behavior are removed. + +The common SQL transaction helper locks sorted unique task rows through the +authority, then sorted repository-local and durable queue-session locks, then +revalidates every exact identity before queue access. PostgreSQL uses ordered +`FOR UPDATE`; SQLite uses its writer fence. The service's outer admission mutex +is never acquired by lifecycle transactions and does not replace this order. +For memory, the authority read guard encloses validation and sorted bucket +locks; deletion removes identity under its write guard before post-commit +cleanup. Admission therefore finishes before deletion or sees a missing +incarnation. + +The handler resolves authenticated claim metadata before admission and passes a +`QueueAttachmentClaim`, never a side-effect callback. +`QueueMessageWithMetadataAfterInsert` is removed. The policy-aware repository +operation validates identity, inserts, claims through the authority on the same +transaction, then folds if eligible. Claim failure rolls back queue and +attachment writes. Memory performs the same sequence inside its authority/bucket +critical section. At capacity, a staged claim still fails before insert or +claim. A two-connection test pauses inside the claim seam: deletion either wins +before validation and no claim occurs, or waits and removes the committed row +and claim without orphaning replacement state. + +Queue policy state belongs to the exact task-session incarnation. SQL session +deletion receives the task ID and expected incarnation, locks the owning task +row before the durable session lock, revalidates the exact row, and atomically +deletes the task session, queued rows, and `queue_session_state`. Task and +workspace deletion already lock their authoritative task rows before acquiring +affected session locks in stable order and apply the same cleanup before +cascade removal. Admissions retain task-row-then-session-lock order and reject +a missing or mismatched session, preventing a waiter from inserting after +deletion or into a recreated textual ID. + +The task repository adds a deletion-only post-commit callback carrying complete +`[]QueueSessionIdentity{TaskID, SessionID, SessionIncarnationID}` values. It is +separate from archive's queue purger/recount notifier. The memory repository +tags each bucket with the complete identity and CAS-purges only exact matches. +Its policy-aware admission validates the expected identity through the task +session authority while holding the bucket mutex, so validation and the +deletion callback order the old bucket. Archive never calls the deletion-only +callback and therefore preserves Auto-run and Auto-merge policy. + +`DeleteSession` relies on this repository cleanup instead of calling +`messageQueue.CancelAll` after the session row is gone. After commit it +publishes only `publishTaskQueueStatusEvent(taskID, "")`, an internal +task-scoped recount. It never publishes a session-scoped queue status for the +deleted textual ID. + +Workflow transfer supplies exact source and destination identities. In one +transaction it locks sorted unique task rows, then sorted source/destination +repository-local and durable session locks, and revalidates both incarnations +before reading or moving queue state. The move preserves existing Auto-run +semantics and the destination Auto-merge override/revision, while deleting the +source state. Source or destination mismatch returns session-not-found with no +row or policy change. Opposite-direction transfer and transfer versus +session/task/workspace delete/recreate tests use two connections to prove this +ordering is deadlock-free and leaves no old-incarnation rows. Snapshot/restore +also validates captured identities and never overwrites either policy. + +`pending_moves` gains non-null `session_incarnation_id`; migration backfills +current identities and deletes unmatched legacy rows. `PendingMoveRecord` and +every set/read/take/CAS, retry, reaper, snapshot/restore, and transfer operation +retain `QueueSessionIdentity` and use the common authority/lock boundary. +Transfer writes the validated destination incarnation. + +`applyPendingMove` passes captured identity, pending move ID, expected source, +and target to one deferred-transition transaction. It locks task then session, +revalidates, applies every direct write, increments the task workflow-transition +generation, and returns an immutable `TransitionEffectToken` containing that +generation and identity. The same-target marker uses this operation. + +Every post-commit task effect CASes the token generation and committed target; +session/queue effects also validate the token identity and operation generation. +Publication, step history, exit/enter, prompt cleanup, drain, and pull-next fail +closed after recreation or a successor move. Tests pause before commit and +after commit before each effect, then race session/task/workspace recreation and +successor moves in both orders. + +## WebSocket contract + +The dispatcher adds: + +```text +message.queue.auto_merge.set +``` + +Request: + +```json +{ + "task_id": "task-id", + "session_id": "session-id", + "session_incarnation_id": "incarnation-uuid", + "enabled": false +} +``` + +Successful response: + +```json +{ + "session_id": "session-id", + "session_incarnation_id": "incarnation-uuid", + "queue_status_epoch": "server-epoch", + "queue_status_generation": 42, + "auto_merge_available": true, + "auto_merge": false, + "auto_merge_source": "session", + "auto_merge_revision": 1 +} +``` + +`task_id`, `session_id`, `session_incarnation_id`, and `enabled` are required. +The gateway and handler authorize the supplied task/session pair; persistence +revalidates the unchanged triplet under task-row-then-session-lock order. + +Every browser queue request requires the triplet: `get`, `add`, `append`, +`cancel`, `update`, `remove`, `merge`, `reorder`, `drain`, `send_now`, +`auto_run.set`, and `auto_merge.set`. A stale identity returns the same +non-enumerating session-not-found response, changes nothing, and publishes no +status. The frontend refetches current state but never retries the operation. + +`message.queue.get` and every `message.queue.status_changed` producer report +policy availability. A successful resolution carries the complete tuple: + +```json +{ + "session_incarnation_id": "incarnation-uuid", + "queue_status_epoch": "server-epoch", + "queue_status_generation": 43, + "auto_merge_available": true, + "auto_merge": true, + "auto_merge_source": "global", + "auto_merge_revision": 4 +} +``` + +`auto_merge_source` is `global` while the session has no override and `session` +after its first successful override write. The revision belongs to that source: +the current persisted global revision for inherited values, or the session +override revision for explicit values. The transport exposes source only for +ordering; the UI does not display inheritance state or offer reset. + +Every session-scoped response/event uses an expected identity captured before +the operation: browser requests supply it; internal work retains it. The +producer allocates a generation, then calls the identity-bound snapshot. SQL +locks task then session, revalidates, and reads entries and policies together; +memory validates and reads under its bucket lock. The payload uses only the +expected identity. Mismatch returns session-not-found and publishes nothing, so +mutation commit followed by recreation cannot expose replacement state. + +The payload receives a `queue_status_epoch` from the random process-start +identity and the allocated generation. The generation comes from one +process-local atomic counter and increases across sessions; it need not persist +because a restart changes the epoch. + +If override resolution fails, the payload carries the same status ordering +fields plus `"auto_merge_available": false` and omits `auto_merge`, +`auto_merge_source`, and `auto_merge_revision`. Queue entries and independent +metadata still publish. The unavailable marker is not an authoritative OFF +value, and its generation lets clients reject a delayed older failure. + +Older notifications without status ordering or `auto_merge_available` use a +complete policy tuple, when present, as an available rolling-upgrade payload; +otherwise they preserve prior frontend policy state. Current session-scoped +producers always encode status ordering and either a complete available tuple +or the unavailable marker. + +Task-only `message.queue.status_changed` events used to trigger status-summary +recounts have no `session_id` and cannot resolve a session policy. They carry +`"queue_status_scope": "task"` and `task_id`, are exempt from session status +ordering and policy fields, and remain on the internal event bus for the +status-summary projector. `routeBroadcast` detects this scope and returns +without sending the event to a session or workspace browser channel. A +session-scoped publisher must provide a non-empty `session_id`, matching +`session_incarnation_id`, and the complete status encoder; an empty-session +queue event without explicit task scope is invalid and is dropped rather than +falling through to `BroadcastToWorkspace`. Producer and gateway tests cover all +three branches. + +## Admission flow + +`QueueSessionIdentity` is mandatory on every service/repository operation that +can insert, append, replace, restore, reserve, acknowledge, consume, update, +remove, merge, reorder, clear, transfer, or change policy. No mutating overload +accepts textual IDs. Browser callers supply current task-session identity; +handlers authorize its pair and persistence revalidates it. Internal and +deferred callers retain captured identity. + +Reservations, `SendNowClaim`, queued-dispatch/cancellation records, and deferred +transition dispatch carry identity plus operation generation. Queue restore/ack +uses the common locked identity boundary. In-flight cleanup CASes identity and +generation. Before cancel or prompt, orchestration verifies task session, +execution incarnation, and operation ownership without holding a database lock +across agent I/O. Stale drain, Auto-run, or Send Now work cannot prompt, restore +into, block, or clear a replacement. Channel-controlled tests recreate after +claim and before prompt/restore for manual drain, Auto-run ON, and both Send Now +scopes. Restore/retry/transfer retains both endpoint identities. + +1. Resolve the authoritative task session and carry its task ID, session ID, + and immutable incarnation in `QueueSessionIdentity`. Load the current global + policy snapshot once. +2. Call the repository's policy-aware admission operation. SQL locks the active + task row, then the durable session lock, and confirms the exact + task/session/incarnation still exists before reading policy or queue rows. + The memory implementation performs identity validation and admission under + its incarnation-tagged bucket mutex. +3. Under that boundary, read the nullable override, resolve it against the + loaded global snapshot, and validate and insert the candidate under the + current queue-capacity rules. +4. If insertion reports `queue_full`, effective Auto-merge is ON, and admission + does not require a staged attachment claim, use the same transaction to try + the existing direct candidate fold into the compatible tail. This applies + when the row count is equal to or greater than the positive limit. If the + fold skips, retain `queue_full`. +5. If override resolution fails, roll back that policy-aware attempt and retry + only ordinary insertion with automatic merge disabled. The fallback repeats + task/session/incarnation validation and the same lock order, never enters the + direct-fold path, and still returns `queue_full` at capacity. +6. A staged-attachment admission at or above the positive limit returns + `queue_full` before claiming the attachment or changing the prior tail. +7. For a successful ordinary insert, claim staged attachments in the same + transaction, then apply the compatibility fold only when its policy snapshot + is ON. Claim or identity failure rolls back both queue and attachment writes. + A later override applies only to the next admission. +8. A successful fold preserves the target row identity and FIFO position, but + assigns the later target/source queued time, matching the shipped + `latestQueuedAt` behavior so pending activity reflects the newest prompt. +9. Publish final authoritative queue status. + +Changing either global or session policy never scans or rewrites pending rows. +Lifecycle coalescing, retry, restore, and transfer keep their existing +coalesce, lifecycle-generation, FIFO, claim, and policy-copy semantics but pass +through the same required session-identity guard before replacing, restoring, +moving, or inserting a row; only ordinary admissions participate in automatic +merge. Manual merge and Send Now retain their existing explicit behavior. + +## Frontend state and mutation flow + +Wire-derived frontend types preserve the backend JSON names: +`TaskSession.queue_incarnation_id`, `TaskSession.task_id`, and +`session_incarnation_id` on queue payloads. There is no camel-case mapping +layer. Local function parameter names may be camel case, but every normalized +session ownership join and status comparison reads the snake-case +`TaskSession` properties. + +`QueueStatus`, `QueueStatusChangedPayload`, and `QueueMeta` gain the +incarnation, status epoch/generation, policy availability, and effective +Auto-merge value, source, and revision. Every `setQueueEntries` call that +synthesizes metadata preserves the complete known incarnation, ordering, and +policy state alongside `autoRun` and `mergeEnabled` so optimistic clear and +reorder operations cannot reset or regress the control. + +`queue-api.ts` requires `taskId` and `sessionIncarnationId` for every queue +request, including status reads. `useQueue` supplies the current `TaskSession` +identity. Stale outcomes refetch current state but never retry old work. + +`QueueState` replaces the textual-session boolean loading entry with +`activeOperationBySessionId[sessionId]`, whose value contains +`sessionIncarnationId` and a monotonically allocated client +`operationGeneration`. `beginQueueOperation(sessionId, incarnation)` verifies +the current `TaskSession.queue_incarnation_id`, rejects a conflicting active +operation, installs a fresh token, and returns it. +`finishQueueOperation(sessionId, token)` deletes state only when both token +fields and the current session incarnation still match. + +An independent status refetch begins and finishes its own token. A refetch +inside a mutation receives and retains the mutation's token instead of +installing another one, so the mutation and its authoritative refetch are one +terminal operation. Success, failure, cancellation, and early returns finish +through the same compare-and-clear API. Session removal deletes the token; a +replacement operation receives a new generation. A delayed old success, error, +refetch, or `finally` therefore cannot clear the replacement's token or +re-enable its controls. + +One shared per-session predicate is true only for the active token matching the +current incarnation. It disables Auto-run, Auto-merge, row Send Now, and +reorder together. Auto-merge additionally disables when policy availability is +false. + +The WebSocket handler applies the status-ordering rules below when an available +tuple is present. An unavailable payload marks the control unavailable but +preserves its last authoritative tuple; entries and unrelated metadata still +reconcile. This keeps refresh, reconnect, and live mutation paths consistent. + +## Status ordering + +Each producer allocates `queue_status_generation` and snapshots its expected +`QueueSessionIdentity`. Available and unavailable results use only that +incarnation; mismatch publishes nothing. + +Frontend policy ordering is: + +1. Reject the whole payload when current `TaskSession.queue_incarnation_id` + differs, including queue entries. +2. A matching incarnation that differs from `QueueMeta` resets prior epoch, + generation, availability, source, and revision. +3. Within one epoch/incarnation, apply availability only at or above the highest + generation. A new non-retired process epoch retires the prior epoch; ignore + later payloads from retired epochs. +4. Accept the first complete tuple, same-source revisions at or above the + current revision, and the irreversible `global` to `session` transition. + Never accept `session` back to `global` for that incarnation. +5. An accepted unavailable status preserves the tuple and disables Auto-merge; + a newer complete tuple restores availability. + +All session- and task-removal reducers delete queue entries, `QueueMeta`, and +the incarnation/generation-tagged operation token for each removed session. +`workspace.deleted` first snapshots affected session IDs by joining the +normalized current `TaskSession.task_id` map to locally cached tasks for that +workspace, then calls the same session cleanup before removing workspace/task +indexes. Queue metadata can exist only for a session present in that normalized +map because incarnation validation precedes status application, making this the +authoritative local cleanup set even for an empty queue. + +A newly created `TaskSession` establishes `queue_incarnation_id` before queue +status is requested. Mutation responses stay narrow, and a post-mutation +refetch uses the same incarnation comparison instead of unconditionally +overwriting event state. Focused tests cover failure before success, a +lower-generation delayed failure after success, restart/reconnect with a new +epoch, session/task/workspace removal, and delete/recreate of the same textual +session ID while old-incarnation status, mutation success, error, refetch, and +`finally` completion are delayed. In the cleanup-race case a new-incarnation +mutation remains pending and all controls stay disabled until its own matching +operation generation completes. + +## Queue header composition + +`QueuePanelHeader` keeps one responsive header region with three logical groups: + +1. queue icon, **Queued**, and capacity text; +2. compact bordered Auto-run and Auto-merge pills, each containing its visible + label and shared `Switch`; +3. Clear all, the desktop-only pin, and collapse. + +The groups use wrapping rather than truncating or horizontally scrolling the +controls. The former dedicated Auto-run help row is removed. Its localized ON +and OFF explanations remain available as the Auto-run switch's accessible +description, while Auto-merge has a localized accessible name. Both labels are +clickable. + +Desktop retains the current dense header sizing. Coarse pointers enlarge each +pill's effective target to at least 44 CSS pixels without inflating desktop +controls. The existing inline panel remains the presentation on mobile; it owns +one internal message-list scroll region, uses dynamic viewport height, and does +not introduce a drawer, fixed footer, safe-area control, or document-level +horizontal overflow. + +Mobile reuses `QueueChip` pill geometry and the current Auto-run coarse-pointer +target in this inline surface, not a compressed desktop workbench. + +The global settings switch remains admin-only. Its localized description and +five locale catalogs explain that sessions inherit the default until changed. + +## Failure and recovery + +- **Write or identity failure:** publish no success; stale identity returns + non-enumerating session-not-found with no mutation or retry. Refetch and show + the localized error when applicable. +- **Admission policy read fails:** log and admit separately only when capacity + permits; never auto-merge. +- **Status policy read fails:** publish ordered + `auto_merge_available: false` without a tuple. Reconcile entries, preserve the + last tuple, and disable Auto-merge until a newer success. +- **Global change:** advance persisted revision; later admission/status uses it + without session migration or notification fan-out. +- **Delayed/legacy status:** epoch, generation, source, and revision reject + regressions; compatibility preservation retains unrelated queue state. + +## Security + +The gateway and handler authorize the supplied task/session pair for every +browser queue request. They pass that pair plus the server-issued expected +incarnation unchanged to the identity-bound read or mutation. Persistence +revalidates under task-row-then-session-lock order, so authorization cannot +retarget a replacement. Existing admin-only global-setting permissions remain. + +## Observability + +Queue mutation failures and override-read failures use structured logs with +`session_id` and the operation. Successful status publication continues through +`message.queue.status_changed`. No new metric is required because this is a user +preference, not a background retry or availability loop. + +## Verification strategy + +- Queue service/repository tests cover policy, capacity, claims, timestamps, + fail-closed reads, and required identities for every mutation category. +- Deterministic SQLite/PostgreSQL tests cover admission, policy, status, + transfer, and deferred transition against lifecycle recreation. They pause + before writes and after commit before every effect, race successor moves, + assert task-then-session lock order, and prove no stale state or deadlock. +- Drain, Auto-run, deferred dispatch, and Send Now tests pause after claim and + before prompt/restore. They prove identity/generation CAS protects replacement + execution, queue, and in-flight ownership. +- Migration/lifecycle tests cover incarnation backfill, deletion cleanup, + archive, transfer, task-only recount routing, and global-policy hydration. +- Handler and producer tests cover triplet authorization, expected-identity + status snapshots, stale errors, and task-only browser suppression. +- Frontend tests cover wire fields, status ordering/removal, and operation-token + CAS. Raw E2E helpers and sidebar, desktop, and mobile Playwright cover + complete get/add/cancel payloads, mutation, persistence, layout, and + stale-token rejection. + +## Related decisions + +- [Keep Queue Auto-run Server Owned](../../../decisions/2026-08-16-server-owned-queue-auto-run.md) +- [Inherit Queue Auto-merge Until Overridden](../../../decisions/2026-09-04-inherit-queue-auto-merge-until-overridden.md)