From bd7974dc8da3fd60d094ad654b7085bcb8e4d414 Mon Sep 17 00:00:00 2001 From: Henry Date: Sun, 13 Sep 2026 17:24:16 +0800 Subject: [PATCH] fix(office): make repeat task reassignment work again after 24 hours (#3533) Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio --- .../internal/backendapp/adapters_office.go | 8 +- .../backendapp/adapters_office_wakeup_test.go | 97 +++ .../office/approvals/handler_security_test.go | 5 +- .../internal/office/approvals/service.go | 6 +- .../internal/office/approvals/service_test.go | 5 +- .../internal/office/channels/service.go | 2 +- .../internal/office/dashboard/handler_test.go | 1 + .../internal/office/dashboard/service.go | 10 +- .../office/dashboard/service_tasks.go | 25 +- .../dashboard/session_termination_test.go | 50 +- .../internal/office/onboarding/service.go | 7 +- .../repository/sqlite/base_migrations.go | 8 +- ...ons_priority_assignment_generation_test.go | 79 +++ .../office/repository/sqlite/tasks.go | 44 +- .../repository/sqlite/tasks_ops_test.go | 62 +- .../office/repository/sqlite/tasks_test.go | 1 + .../run_dedup_generation_dispatch_test.go | 90 +++ .../run_dedup_generation_keys_test.go | 104 ++++ .../internal/office/routines/service.go | 70 ++- .../internal/office/runtime/actions.go | 33 +- .../internal/office/runtime/actions_test.go | 5 +- .../backend/internal/office/runtime/errors.go | 6 + .../internal/office/runtime/handler.go | 2 +- .../runtime/spawn_agent_run_key_test.go | 217 +++++++ .../office/scheduler/approval_adapter.go | 2 +- .../office/scheduler/dashboard_adapter.go | 13 +- .../scheduler/queue_run_dedup_outcome_test.go | 72 +++ .../internal/office/scheduler/reactivity.go | 175 ++++-- .../reactivity_apply_mutation_test.go | 138 +++++ .../reactivity_blockers_resolved_test.go | 117 ++++ .../reactivity_children_completed_test.go | 2 +- .../reactivity_producer_key_audit_test.go | 372 ++++++++++++ apps/backend/internal/office/scheduler/run.go | 64 +- .../service/agent_working_status_test.go | 2 +- .../internal/office/service/base_test.go | 1 + .../blockers_resolved_key_convergence_test.go | 192 ++++++ .../office/service/budget_admission_test.go | 46 +- .../office/service/budget_metrics_test.go | 22 +- .../internal/office/service/channels.go | 2 +- .../continuation_summary_reader_test.go | 4 +- .../office/service/event_subscribers.go | 50 +- .../event_subscribers_decision_test.go | 16 +- .../service/event_subscribers_engine_test.go | 4 +- .../event_subscribers_run_output_test.go | 18 +- .../office/service/event_subscribers_test.go | 71 ++- .../internal/office/service/failure.go | 6 +- .../internal/office/service/failure_test.go | 4 +- apps/backend/internal/office/service/retry.go | 7 +- .../service/retry_ceo_self_escalation_test.go | 2 +- .../office/service/retry_ratelimit_test.go | 6 +- apps/backend/internal/office/service/run.go | 30 +- .../service/run_lifecycle_events_test.go | 14 +- .../internal/office/service/run_test.go | 81 ++- .../scheduler_checkout_contention_test.go | 2 +- .../service/scheduler_checkout_error_test.go | 2 +- .../scheduler_checkout_inactive_agent_test.go | 4 +- .../scheduler_checkout_release_test.go | 6 +- .../office/service/scheduler_features_test.go | 18 +- .../scheduler_integration_routing_test.go | 12 +- .../service/scheduler_integration_test.go | 14 +- .../office/service/scheduler_recovery.go | 4 +- .../service/scheduler_run_outcome_test.go | 16 +- .../office/service/scheduler_runs_test.go | 8 +- .../service/scheduler_taskless_launch_test.go | 16 +- .../task_assigned_generation_key_test.go | 180 ++++++ .../internal/office/service/task_assignee.go | 10 +- .../office/service/task_starter_test.go | 16 +- .../wo46_idle_skip_routine_dispatch_test.go | 4 +- .../internal/office/shared/interfaces.go | 8 +- .../orchestrator/event_handlers_workflow.go | 12 +- ...handlers_workflow_office_autostart_test.go | 14 + .../internal/runs/dedupkeys/dedupkeys.go | 38 ++ .../internal/runs/dedupkeys/dedupkeys_test.go | 43 ++ apps/backend/internal/runs/service/dedup.go | 116 ++++ .../internal/runs/service/dedup_test.go | 140 +++++ .../internal/runs/service/metrics_vars.go | 75 +++ .../runs/service/queue_outcome_none_test.go | 22 + apps/backend/internal/runs/service/service.go | 58 +- .../task/repository/sqlite/base_migrations.go | 6 + .../internal/task/repository/sqlite/task.go | 8 + ...w_fk_removal_assignment_generation_test.go | 117 ++++ .../internal/task/service/service_tasks.go | 17 +- ...ervice_tasks_assignment_generation_test.go | 125 ++++ .../internal/workflow/engine/adapters.go | 6 + .../plans/office-run-dedup-generation/plan.md | 47 ++ .../task-01-run-dedup-generation.md | 74 +++ .../requirements/run-dedup-generation.md | 336 +++++++++++ .../system-design/run-dedup-generation-01.md | 568 ++++++++++++++++++ .../system-design/run-dedup-generation-02.md | 398 ++++++++++++ .../system-design/run-dedup-generation-03.md | 504 ++++++++++++++++ 90 files changed, 5149 insertions(+), 365 deletions(-) create mode 100644 apps/backend/internal/backendapp/adapters_office_wakeup_test.go create mode 100644 apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go create mode 100644 apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go create mode 100644 apps/backend/internal/office/routines/run_dedup_generation_keys_test.go create mode 100644 apps/backend/internal/office/runtime/spawn_agent_run_key_test.go create mode 100644 apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go create mode 100644 apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go create mode 100644 apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go create mode 100644 apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go create mode 100644 apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go create mode 100644 apps/backend/internal/office/service/task_assigned_generation_key_test.go create mode 100644 apps/backend/internal/runs/dedupkeys/dedupkeys.go create mode 100644 apps/backend/internal/runs/dedupkeys/dedupkeys_test.go create mode 100644 apps/backend/internal/runs/service/dedup.go create mode 100644 apps/backend/internal/runs/service/dedup_test.go create mode 100644 apps/backend/internal/runs/service/metrics_vars.go create mode 100644 apps/backend/internal/runs/service/queue_outcome_none_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go create mode 100644 apps/backend/internal/task/service/service_tasks_assignment_generation_test.go create mode 100644 docs/plans/office-run-dedup-generation/plan.md create mode 100644 docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md create mode 100644 docs/specs/office/requirements/run-dedup-generation.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-01.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-02.md create mode 100644 docs/specs/office/system-design/run-dedup-generation-03.md diff --git a/apps/backend/internal/backendapp/adapters_office.go b/apps/backend/internal/backendapp/adapters_office.go index 87041e86e5..0ce41e4646 100644 --- a/apps/backend/internal/backendapp/adapters_office.go +++ b/apps/backend/internal/backendapp/adapters_office.go @@ -17,6 +17,7 @@ import ( officeroutines "github.com/kandev/kandev/internal/office/routines" officeservice "github.com/kandev/kandev/internal/office/service" officewakeup "github.com/kandev/kandev/internal/office/wakeup" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/task/models" tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" taskservice "github.com/kandev/kandev/internal/task/service" @@ -267,7 +268,12 @@ func (a *routineWakeupAdapter) CreateWakeupRequest( } if err := a.repo.CreateWakeupRequest(ctx, row); err != nil { if errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) { - return officeroutines.ErrWakeupAlreadyRequested + runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, req.Reason, req.IdempotencyKey, req.AgentProfileID) + // Wraps both sentinels so a caller can check either: routines + // callers key off ErrWakeupAlreadyRequested to treat this as + // success by another route, while errors.Is against the + // sqlite-layer ErrWakeupIdempotencyConflict still matches. + return fmt.Errorf("%w: %w", officeroutines.ErrWakeupAlreadyRequested, err) } return err } diff --git a/apps/backend/internal/backendapp/adapters_office_wakeup_test.go b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go new file mode 100644 index 0000000000..cc58c77ba8 --- /dev/null +++ b/apps/backend/internal/backendapp/adapters_office_wakeup_test.go @@ -0,0 +1,97 @@ +package backendapp + +import ( + "context" + "errors" + "expvar" + "strings" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + officeroutines "github.com/kandev/kandev/internal/office/routines" +) + +func newTestRoutineWakeupAdapter(t *testing.T) *routineWakeupAdapter { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store init: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new office repo: %v", err) + } + return &routineWakeupAdapter{repo: repo} +} + +// AC-OFFICE-RUN-DEDUP-004.6: a durable idempotency conflict on the wakeup +// queue moves office_run_dedup_total{queue="wakeup",kind="durable"} through +// ReportDurableDedup, not ReportInsertResult - the adapter does its own +// errors.Is on the repository's sentinel and reports the conflict itself. +func TestRoutineWakeupAdapter_CreateWakeupRequest_DurableConflictReportsCounter(t *testing.T) { + adapter := newTestRoutineWakeupAdapter(t) + ctx := context.Background() + reason := "test_routine_wakeup_conflict_" + t.Name() + key := "routine:r1:t1:tick:1234567890" + + first := &officeroutines.WakeupRequest{ + ID: "wakeup-1", AgentProfileID: "agent-1", Source: "cron", + Reason: reason, IdempotencyKey: key, + } + if err := adapter.CreateWakeupRequest(ctx, first); err != nil { + t.Fatalf("first create: %v", err) + } + + second := &officeroutines.WakeupRequest{ + ID: "wakeup-2", AgentProfileID: "agent-1", Source: "cron", + Reason: reason, IdempotencyKey: key, + } + err := adapter.CreateWakeupRequest(ctx, second) + if !errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) { + t.Fatalf("second create error = %v, want ErrWakeupIdempotencyConflict", err) + } + + // reason isn't in runs/service's bounded metricReasons allowlist, so the + // label buckets it to "custom" rather than carrying it verbatim. + if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=wakeup") { + t.Fatal("expected office_run_dedup_total to carry a custom/durable/wakeup entry") + } +} + +// counterHasLabel reports whether the named expvar.Map has an entry whose +// key contains every given substring. Counters are process-global expvar +// state (no reset hook), so tests assert presence/growth rather than exact +// values that could collide with other tests in the same binary run. +func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a *expvar.Map", mapName) + } + found := false + m.Do(func(kv expvar.KeyValue) { + matches := true + for _, s := range substrs { + if !strings.Contains(kv.Key, s) { + matches = false + break + } + } + if matches { + found = true + } + }) + return found +} diff --git a/apps/backend/internal/office/approvals/handler_security_test.go b/apps/backend/internal/office/approvals/handler_security_test.go index 76d7bac69a..7cb8c50de4 100644 --- a/apps/backend/internal/office/approvals/handler_security_test.go +++ b/apps/backend/internal/office/approvals/handler_security_test.go @@ -18,6 +18,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // approvalHandlerFixture wires the minimal stack needed to exercise the @@ -72,7 +73,9 @@ func (s *silentActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _, type silentRunQueuer struct{} -func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil } +func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) { + return runsservice.QueueOutcomeQueued, nil +} // seedApprovalAgent creates an agent_profiles row with the given role // and permissions, returning the persisted instance. diff --git a/apps/backend/internal/office/approvals/service.go b/apps/backend/internal/office/approvals/service.go index aaef3d2a01..5408f7645a 100644 --- a/apps/backend/internal/office/approvals/service.go +++ b/apps/backend/internal/office/approvals/service.go @@ -9,6 +9,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" "go.uber.org/zap" ) @@ -29,7 +30,7 @@ type AgentWriter interface { // RunQueuer enqueues run requests for agent instances. type RunQueuer interface { - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // ApprovalService handles approval CRUD and decide logic. @@ -215,6 +216,7 @@ func (s *ApprovalService) queueApprovalRun(ctx context.Context, approval *Approv approval.ID, approval.Type, approval.Status, approval.DecisionNote, ) idempotencyKey := "approval:" + approval.ID - return s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID, + _, err := s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID, "approval_resolved", payload, idempotencyKey) + return err } diff --git a/apps/backend/internal/office/approvals/service_test.go b/apps/backend/internal/office/approvals/service_test.go index 1186011164..33206ea7b7 100644 --- a/apps/backend/internal/office/approvals/service_test.go +++ b/apps/backend/internal/office/approvals/service_test.go @@ -11,6 +11,7 @@ import ( "github.com/kandev/kandev/internal/office/approvals" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // noopActivityLogger implements shared.ActivityLogger without importing shared. @@ -23,7 +24,9 @@ func (n *noopActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _, _ // noopRunQueuer implements approvals.RunQueuer as a no-op. type noopRunQueuer struct{} -func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil } +func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) { + return runsservice.QueueOutcomeQueued, nil +} type fakeAgentWriter struct { statuses map[string]string diff --git a/apps/backend/internal/office/channels/service.go b/apps/backend/internal/office/channels/service.go index 5311fa31d6..7971ffdf6d 100644 --- a/apps/backend/internal/office/channels/service.go +++ b/apps/backend/internal/office/channels/service.go @@ -103,7 +103,7 @@ func (s *ChannelService) SetupChannel(ctx context.Context, channel *models.Chann // workflow_step_participants. Channel tasks have no workflow_step_id; // the participant row is keyed at (step_id="", task_id) and the // runner projection still resolves it. - if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { + if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { _ = s.repo.DeleteChannel(ctx, channel.ID) return fmt.Errorf("set channel task assignee: %w", err) } diff --git a/apps/backend/internal/office/dashboard/handler_test.go b/apps/backend/internal/office/dashboard/handler_test.go index 9391245fb7..e445ba7248 100644 --- a/apps/backend/internal/office/dashboard/handler_test.go +++ b/apps/backend/internal/office/dashboard/handler_test.go @@ -141,6 +141,7 @@ func newTestDeps(t *testing.T) *testDeps { project_id TEXT DEFAULT '', assignee_agent_profile_id TEXT DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, labels TEXT DEFAULT '[]', metadata TEXT DEFAULT '{}', identifier TEXT DEFAULT '', diff --git a/apps/backend/internal/office/dashboard/service.go b/apps/backend/internal/office/dashboard/service.go index 1e2a3bdcea..b32621113b 100644 --- a/apps/backend/internal/office/dashboard/service.go +++ b/apps/backend/internal/office/dashboard/service.go @@ -74,8 +74,11 @@ type Repository interface { GetRunsByCommentIDs(ctx context.Context, commentIDs []string) (map[string]sqlite.CommentRunStatus, error) UpdateTaskState(ctx context.Context, taskID, state string) error GetTaskExecutionFields(ctx context.Context, taskID string) (*sqlite.TaskExecutionFields, error) + // UpdateTaskAssignee returns the task's assignment_generation after the + // bump, read back inside the same transaction that wrote the runner + // seat (see the sqlite implementation's doc comment). + UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) UpdateTaskStateIfWorkflowStep(ctx context.Context, taskID, expectedStepID, state string) (bool, error) - UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error UpdateTaskPriority(ctx context.Context, taskID, priority string) error UpdateTaskProjectID(ctx context.Context, taskID, projectID string) error GetTaskProjectID(ctx context.Context, taskID string) (string, error) @@ -270,6 +273,11 @@ type MarkFixedHandler interface { type TaskReactivityChange struct { NewStatus *string NewAssigneeID *string + // AssignmentGeneration is the value UpdateTaskAssignee's transaction + // committed and read back, carried here rather than re-read. Nil means + // the caller could not supply one (e.g. the read-back itself failed); + // the pipeline then enqueues the task_assigned wake keyless. + AssignmentGeneration *int64 // PrevAssigneeID is the assignee BEFORE the mutation. Required when // NewAssigneeID is set so the pipeline can detect a real change and // hand off the previous assignee's session. diff --git a/apps/backend/internal/office/dashboard/service_tasks.go b/apps/backend/internal/office/dashboard/service_tasks.go index 5b821ec342..b4fe507d46 100644 --- a/apps/backend/internal/office/dashboard/service_tasks.go +++ b/apps/backend/internal/office/dashboard/service_tasks.go @@ -920,7 +920,8 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge zap.String("task_id", taskID), zap.Error(err)) } } - if err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID); err != nil { + generation, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + if err != nil { return err } @@ -928,14 +929,16 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge // Reactivity pipeline — wakes the new assignee with task_assigned // and hard-cancels the previous assignee's active session. - s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID) + s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID, generation) return nil } // runReactivityForAssigneeChange invokes the reactivity pipeline for an // assignee change. Best-effort — failures are logged, never propagated. +// generation is the value UpdateTaskAssignee's transaction just committed +// and read back; it is carried onto the mutation rather than re-read. func (s *DashboardService) runReactivityForAssigneeChange( - ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string, + ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string, generation int64, ) { if s.reactivity == nil { return @@ -945,10 +948,11 @@ func (s *DashboardService) runReactivityForAssigneeChange( actorType = "agent" } change := TaskReactivityChange{ - NewAssigneeID: &newAssigneeID, - PrevAssigneeID: prevAssigneeID, - ActorID: callerAgentID, - ActorType: actorType, + NewAssigneeID: &newAssigneeID, + AssignmentGeneration: &generation, + PrevAssigneeID: prevAssigneeID, + ActorID: callerAgentID, + ActorType: actorType, } // preStatus="" — assignee changes don't depend on the prev status. result, err := s.reactivity.ApplyTaskMutation(ctx, taskID, "", change) @@ -963,7 +967,10 @@ func (s *DashboardService) runReactivityForAssigneeChange( // Flip the prev assignee's office session row to COMPLETED so it leaves // the active sessions list. The reactivity pipeline already hard-cancels // the running execution above; this is the persistent-row counterpart. - if prevAssigneeID != "" && s.sessionTerm != nil { + // A same-agent reassignment is not a handoff — the pipeline above never + // interrupts it — so this must not terminate the agent's own live + // session out from under it. + if prevAssigneeID != "" && prevAssigneeID != newAssigneeID && s.sessionTerm != nil { if err := s.sessionTerm.TerminateOfficeSession(ctx, taskID, prevAssigneeID, sessionTermReasonReassigned); err != nil { s.logger.Warn("terminate prev-assignee office session failed", zap.String("task_id", taskID), @@ -975,7 +982,7 @@ func (s *DashboardService) runReactivityForAssigneeChange( // the user isn't asked to triage a failure they already worked // around by reassigning. Counter is intentionally not reset — the // root cause may still be unfixed for the old agent. - if prevAssigneeID != "" && s.failureNotifier != nil { + if prevAssigneeID != "" && prevAssigneeID != newAssigneeID && s.failureNotifier != nil { s.failureNotifier.OnAssigneeChanged(ctx, taskID, prevAssigneeID) } } diff --git a/apps/backend/internal/office/dashboard/session_termination_test.go b/apps/backend/internal/office/dashboard/session_termination_test.go index 0ee901e7f2..5679188310 100644 --- a/apps/backend/internal/office/dashboard/session_termination_test.go +++ b/apps/backend/internal/office/dashboard/session_termination_test.go @@ -13,6 +13,18 @@ type recordingTerminator struct { calls []termCall } +type recordingFailureNotifier struct { + calls []struct { + taskID, agentID string + } +} + +func (r *recordingFailureNotifier) OnAssigneeChanged(_ context.Context, taskID, agentID string) { + r.calls = append(r.calls, struct { + taskID, agentID string + }{taskID: taskID, agentID: agentID}) +} + type termCall struct { taskID, agentID, reason string } @@ -85,12 +97,14 @@ func (r *recordingReactivity) ApplyTaskMutation(_ context.Context, _ string, _ s func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) { deps := newTestDeps(t) rt := &recordingTerminator{} + fn := &recordingFailureNotifier{} deps.svc.SetSessionTerminator(rt) + deps.svc.SetFailureNotifier(fn) deps.svc.SetReactivityApplier(&recordingReactivity{result: &dashboard.TaskReactivityResult{}}) insertTestTask(t, deps.db, "task-r", "ws-r", "Reassign", "todo", 2) // Seed prev assignee directly via the underlying repo update. - if err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil { + if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil { t.Fatalf("seed prev assignee: %v", err) } @@ -105,4 +119,38 @@ func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) { if got.taskID != "task-r" || got.agentID != "agent-prev" { t.Errorf("term call: got %+v", got) } + if len(fn.calls) != 1 || fn.calls[0].taskID != "task-r" || fn.calls[0].agentID != "agent-prev" { + t.Fatalf("expected prior-assignee failure notification, got %+v", fn.calls) + } +} + +// TestSetTaskAssignee_SameAgent_DoesNotTerminateSession is the regression +// test for Review round 3 Finding 5: a repeat assignment to the agent that +// already holds the seat must not flip that agent's own live session row to +// COMPLETED. The reactivity pipeline correctly declines to hard-cancel this +// case (AC-OFFICE-RUN-DEDUP-001.3 treats it as a real occurrence, not an +// interrupt), and the persisted-row side effect must agree. +func TestSetTaskAssignee_SameAgent_DoesNotTerminateSession(t *testing.T) { + deps := newTestDeps(t) + rt := &recordingTerminator{} + fn := &recordingFailureNotifier{} + deps.svc.SetSessionTerminator(rt) + deps.svc.SetFailureNotifier(fn) + deps.svc.SetReactivityApplier(&recordingReactivity{result: &dashboard.TaskReactivityResult{}}) + + insertTestTask(t, deps.db, "task-same", "ws-r", "Reassign", "todo", 2) + if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-same", "agent-x"); err != nil { + t.Fatalf("seed prev assignee: %v", err) + } + + if err := deps.svc.SetTaskAssigneeAsAgent(context.Background(), "", "task-same", "agent-x"); err != nil { + t.Fatalf("set assignee: %v", err) + } + + if len(rt.calls) != 0 { + t.Fatalf("same-agent reassignment must not terminate the agent's own session, got %d calls (%+v)", len(rt.calls), rt.calls) + } + if len(fn.calls) != 0 { + t.Fatalf("same-agent reassignment must not dismiss its failure inbox entry, got %+v", fn.calls) + } } diff --git a/apps/backend/internal/office/onboarding/service.go b/apps/backend/internal/office/onboarding/service.go index 88ca65afbf..aff0ca4eef 100644 --- a/apps/backend/internal/office/onboarding/service.go +++ b/apps/backend/internal/office/onboarding/service.go @@ -17,6 +17,7 @@ import ( "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/routing" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" taskservice "github.com/kandev/kandev/internal/task/service" "go.uber.org/zap" @@ -349,7 +350,11 @@ func (s *OnboardingService) maybeCreateOnboardingTask( return "" } if s.runQueuer != nil { - if wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned, + // A third task_assigned producer alongside queueTaskAssignedRun; it is + // never handed the assigning transaction's generation, so it enqueues + // keyless rather than deriving a divergent key. + runsservice.ReportKeylessEnqueue(runReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "onboarding_no_generation") + if _, wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned, fmt.Sprintf(`{"task_id":%q}`, taskID), ""); wakeErr != nil { s.logger.Warn("enqueue onboarding run failed", zap.Error(wakeErr)) } diff --git a/apps/backend/internal/office/repository/sqlite/base_migrations.go b/apps/backend/internal/office/repository/sqlite/base_migrations.go index 42253d7a0f..56238f2c7c 100644 --- a/apps/backend/internal/office/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/office/repository/sqlite/base_migrations.go @@ -625,6 +625,7 @@ func (r *Repository) runTaskPriorityRecreate() error { {"external_id", `ALTER TABLE tasks ADD COLUMN external_id TEXT COLLATE BINARY`}, {"external_id_settled_at", `ALTER TABLE tasks ADD COLUMN external_id_settled_at TIMESTAMP`}, {"assignee_user_id", `ALTER TABLE tasks ADD COLUMN assignee_user_id TEXT NOT NULL DEFAULT ''`}, + {"assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`}, } for _, column := range legacyColumns { if _, err := conn.ExecContext(ctx, column.stmt); err != nil && !db.IsDuplicateColumnError(err) { @@ -687,7 +688,8 @@ func taskPriorityMigrationStatements() []string { checkout_run_id TEXT, external_id TEXT COLLATE BINARY, external_id_settled_at TIMESTAMP, - assignee_user_id TEXT NOT NULL DEFAULT '' + assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0 )`, // archived_by_cascade_id and external_id/external_id_settled_at are // added to the task schema by task/repository/sqlite/base.go @@ -707,7 +709,7 @@ func taskPriorityMigrationStatements() []string { origin, project_id, labels, identifier, checkout_agent_id, checkout_at, checkout_run_id, - external_id, external_id_settled_at, assignee_user_id + external_id, external_id_settled_at, assignee_user_id, assignment_generation ) SELECT id, COALESCE(workspace_id,''), COALESCE(workflow_id,''), COALESCE(workflow_step_id,''), title, COALESCE(description,''), @@ -722,7 +724,7 @@ func taskPriorityMigrationStatements() []string { COALESCE(labels,'[]'), identifier, checkout_agent_id, checkout_at, checkout_run_id, external_id, external_id_settled_at, - COALESCE(assignee_user_id,'') + COALESCE(assignee_user_id,''), COALESCE(assignment_generation,0) FROM tasks`, `DROP TABLE tasks`, `ALTER TABLE tasks_priority_new RENAME TO tasks`, diff --git a/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go b/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go new file mode 100644 index 0000000000..f1cc53e87e --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/migrations_priority_assignment_generation_test.go @@ -0,0 +1,79 @@ +package sqlite_test + +import ( + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + "github.com/kandev/kandev/internal/office/repository/sqlite" +) + +// TestMigrate_PriorityRebuildPreservesAssignmentGeneration is the regression +// test for the same rebuild hazard TestMigrate_PriorityRebuildPreservesAssignee +// covers one column over: taskPriorityMigrationStatements' INSERT ... SELECT +// carries tasks.assignment_generation through +// COALESCE(assignment_generation,0) (docs/specs/office/system-design/ +// run-dedup-generation-02.md#failure-and-recovery). That expression had zero +// coverage before this test — the sibling assignee test seeds a table where +// runTaskPriorityRecreate's own defensive ALTER TABLE ADD COLUMN supplies the +// column fresh (every row defaults to 0), which never exercises copying a +// genuinely non-zero value through the recreate. Here the legacy table +// already carries the column with a non-zero value, as an install upgrading +// mid-feature (assignment_generation shipped, priority rebuild still pending) +// would. +func TestMigrate_PriorityRebuildPreservesAssignmentGeneration(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + "?_journal_mode=WAL" + db, err := sqlx.Open("sqlite3", dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + if _, err := db.Exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + workflow_id TEXT NOT NULL DEFAULT '', + workflow_step_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + description TEXT DEFAULT '', + state TEXT DEFAULT 'TODO', + priority INTEGER DEFAULT 0, + position INTEGER DEFAULT 0, + metadata TEXT DEFAULT '{}', + is_ephemeral INTEGER NOT NULL DEFAULT 0, + parent_id TEXT DEFAULT '', + archived_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + origin TEXT DEFAULT 'manual', + project_id TEXT DEFAULT '', + labels TEXT DEFAULT '[]', + identifier TEXT, + assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0 + ); + `); err != nil { + t.Fatalf("seed legacy schema: %v", err) + } + if _, err := db.Exec(` + INSERT INTO tasks (id, workspace_id, title, assignment_generation) + VALUES ('task-1', 'ws-1', 'reassigned task', 3) + `); err != nil { + t.Fatalf("seed reassigned task: %v", err) + } + + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init office repo (run migrations): %v", err) + } + + var generation int64 + if err := db.Get(&generation, `SELECT assignment_generation FROM tasks WHERE id = 'task-1'`); err != nil { + t.Fatalf("tasks.assignment_generation missing after priority rebuild: %v", err) + } + if generation != 3 { + t.Fatalf("assignment_generation lost by priority rebuild: got %d, want 3", generation) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/tasks.go b/apps/backend/internal/office/repository/sqlite/tasks.go index 34004a9b4a..09a0b53e46 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks.go +++ b/apps/backend/internal/office/repository/sqlite/tasks.go @@ -165,21 +165,32 @@ func (r *Repository) UpdateTaskStateIfWorkflowStep( // task_id) so the projection's per-task runner clause still resolves it // (the (step_id="" / step_id="") match holds because the SELECT joins // step_id = task.workflow_step_id which is also ""). -func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error { +// +// This is one of the two assignment_generation bump sites (the other is +// insertTaskTx -> upsertRunnerInTx on create). It increments +// tasks.assignment_generation unconditionally on every committed call — +// including a repeat assignment to the agent that already holds the seat, +// which is a real occurrence, not a no-op — and reads the new value back +// inside this same transaction before Commit, returning it so callers carry +// it forward instead of re-reading it later (a later re-read could observe a +// different, more recent occurrence's value). A read-back failure rolls the +// whole assignment back rather than commit a write whose generation could +// not be reported. +func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) { var stepID string err := r.ro.QueryRowxContext(ctx, r.ro.Rebind( `SELECT COALESCE(workflow_step_id, '') FROM tasks WHERE id = ?`), taskID).Scan(&stepID) if err != nil { if err == sql.ErrNoRows { - return fmt.Errorf("task not found: %s", taskID) + return 0, fmt.Errorf("task not found: %s", taskID) } - return err + return 0, err } tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return err + return 0, err } defer func() { _ = tx.Rollback() }() @@ -188,7 +199,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID DELETE FROM workflow_step_participants WHERE step_id = ? AND task_id = ? AND role = 'runner' `), stepID, taskID); err != nil { - return err + return 0, err } } else { var existing string @@ -201,7 +212,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID if _, err := tx.ExecContext(ctx, tx.Rebind( `UPDATE workflow_step_participants SET agent_profile_id = ? WHERE id = ?`), assigneeID, existing); err != nil { - return err + return 0, err } case sql.ErrNoRows: if _, err := tx.ExecContext(ctx, tx.Rebind(` @@ -209,19 +220,30 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID (id, step_id, task_id, role, agent_profile_id, decision_required, position, created_at) VALUES (?, ?, ?, 'runner', ?, 0, 0, ?) `), newParticipantUUID(), stepID, taskID, assigneeID, time.Now().UTC()); err != nil { - return err + return 0, err } default: - return probeErr + return 0, probeErr } } if _, err := tx.ExecContext(ctx, tx.Rebind(` - UPDATE tasks SET updated_at = CURRENT_TIMESTAMP WHERE id = ? + UPDATE tasks SET assignment_generation = assignment_generation + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ? `), taskID); err != nil { - return err + return 0, err } - return tx.Commit() + + var generation int64 + if err := tx.QueryRowxContext(ctx, tx.Rebind( + `SELECT assignment_generation FROM tasks WHERE id = ?`), + taskID).Scan(&generation); err != nil { + return 0, err + } + + if err := tx.Commit(); err != nil { + return 0, err + } + return generation, nil } // TaskBasicInfo contains the minimal task fields needed for prompt building. diff --git a/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go b/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go index 18ff7cce14..d0012a48aa 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go +++ b/apps/backend/internal/office/repository/sqlite/tasks_ops_test.go @@ -330,9 +330,13 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Fatalf("seed task: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one"); err != nil { + gen1, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one") + if err != nil { t.Fatalf("UpdateTaskAssignee (insert): %v", err) } + if gen1 != 1 { + t.Fatalf("generation = %d, want 1 for the first assignment", gen1) + } if got := taskRunner(t, repo, "as-1"); got != "agent-one" { t.Fatalf("runner = %q, want agent-one", got) } @@ -340,10 +344,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Fatalf("runner rows = %d, want 1", n) } - // A second assignment updates the existing row instead of adding one. - if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two"); err != nil { + // A second assignment updates the existing row instead of adding one, + // and bumps the generation again. + gen2, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two") + if err != nil { t.Fatalf("UpdateTaskAssignee (update): %v", err) } + if gen2 != 2 { + t.Fatalf("generation = %d, want 2 for the second assignment", gen2) + } if got := taskRunner(t, repo, "as-1"); got != "agent-two" { t.Errorf("runner = %q, want agent-two", got) } @@ -351,10 +360,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Errorf("runner rows = %d, want the row updated in place", n) } - // An empty assignee deletes the runner row. - if err := repo.UpdateTaskAssignee(ctx, "as-1", ""); err != nil { + // An empty assignee deletes the runner row and still bumps the + // generation: an unassignment is itself an assignment write. + gen3, err := repo.UpdateTaskAssignee(ctx, "as-1", "") + if err != nil { t.Fatalf("UpdateTaskAssignee (clear): %v", err) } + if gen3 != 3 { + t.Fatalf("generation = %d, want 3 for the clear", gen3) + } if n := runnerRowCount(t, repo, "as-1"); n != 0 { t.Errorf("runner rows = %d, want 0 after clearing", n) } @@ -362,7 +376,7 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { t.Errorf("runner = %q, want empty after clearing", got) } - err := repo.UpdateTaskAssignee(ctx, "missing", "agent-one") + _, err = repo.UpdateTaskAssignee(ctx, "missing", "agent-one") if err == nil { t.Fatal("UpdateTaskAssignee(missing) = nil error, want not-found") } @@ -371,6 +385,34 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) { } } +// TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration covers AC-001.3: a +// repeat assignment to the agent that already holds the seat is a real +// occurrence (the operator is asking for the work again), not a no-op, so it +// must mint a new generation rather than leave the counter unchanged. +func TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration(t *testing.T) { + repo := newSearchTestRepo(t) + ctx := context.Background() + + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id, title, created_at, updated_at) + VALUES ('as-repeat', 'ws-1', 'step-1', 'Repeat assign', datetime('now'), datetime('now')) + `); err != nil { + t.Fatalf("seed task: %v", err) + } + + gen1, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a") + if err != nil { + t.Fatalf("first assignment: %v", err) + } + gen2, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a") + if err != nil { + t.Fatalf("repeat assignment: %v", err) + } + if gen2 != gen1+1 { + t.Fatalf("repeat assignment generation = %d, want %d (one more than %d)", gen2, gen1+1, gen1) + } +} + // Tasks created outside a workflow have no workflow_step_id; the runner row // is keyed on the empty step so the projection still resolves it. func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) { @@ -384,7 +426,7 @@ func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) { t.Fatalf("seed task: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil { t.Fatalf("UpdateTaskAssignee: %v", err) } if got := taskRunner(t, repo, "as-nostep"); got != "agent-channel" { @@ -405,13 +447,13 @@ func TestUpdateTaskAssignee_LeavesOtherTasksAlone(t *testing.T) { t.Fatalf("seed tasks: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil { t.Fatalf("assign as-a: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil { t.Fatalf("assign as-b: %v", err) } - if err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil { + if _, err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil { t.Fatalf("clear as-b: %v", err) } diff --git a/apps/backend/internal/office/repository/sqlite/tasks_test.go b/apps/backend/internal/office/repository/sqlite/tasks_test.go index 26dcf32f5e..c76bf8ebf0 100644 --- a/apps/backend/internal/office/repository/sqlite/tasks_test.go +++ b/apps/backend/internal/office/repository/sqlite/tasks_test.go @@ -36,6 +36,7 @@ func newSearchTestRepo(t *testing.T) *sqlite.Repository { workflow_step_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, description TEXT DEFAULT '', state TEXT DEFAULT 'TODO', priority TEXT NOT NULL DEFAULT 'medium' CHECK (priority IN ('critical','high','medium','low')), diff --git a/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go b/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go new file mode 100644 index 0000000000..b17ba118fe --- /dev/null +++ b/apps/backend/internal/office/routines/run_dedup_generation_dispatch_test.go @@ -0,0 +1,90 @@ +package routines_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +// AC-OFFICE-RUN-DEDUP-001.9: the cron dispatch path threads the trigger's +// claimed tick into the wakeup-request's idempotency key rather than +// re-reading trigger.next_run_at after TickScheduledTriggers has already +// advanced it to the next slot. Driving this through TickScheduledTriggers +// (not the private key builder) is what actually pins the wiring: a future +// change that re-reads the trigger row after the advance would still pass +// the unit-level key-format tests but fail here. +func TestTickScheduledTriggers_WakeupKeyNamesClaimedTick_NotAdvancedRow(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, + Kind: "cron", + CronExpression: "* * * * *", + Timezone: "UTC", + Enabled: true, + } + if err := svc.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + triggers, err := svc.ListRoutineTriggers(ctx, routine.ID) + if err != nil || len(triggers) != 1 { + t.Fatalf("list triggers: %v (n=%d)", err, len(triggers)) + } + claimedTick := *triggers[0].NextRunAt + + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + if err := svc.TickScheduledTriggers(ctx, claimedTick.Add(2*time.Minute)); err != nil { + t.Fatalf("tick scheduled triggers: %v", err) + } + if len(enq.created) != 1 { + t.Fatalf("expected 1 wakeup-request created, got %d", len(enq.created)) + } + + want := fmt.Sprintf("routine:%s:%s:tick:%d", routine.ID, triggers[0].ID, claimedTick.Unix()) + got := enq.created[0].IdempotencyKey + if got != want { + t.Fatalf("idempotency key = %q, want %q (the claimed tick, not the row's advanced next_run_at)", got, want) + } +} + +// AC-OFFICE-RUN-DEDUP-001.2: two manual "Fire now" calls inside the same +// minute are two distinct occurrences (an operator asking twice) and must +// mint two distinct keys - the exact collision the old unix-minute key +// format reached (both fires share triggerID == "" and can share a minute). +func TestFireManual_TwoFiresInSameMinute_MintDistinctKeys(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + routine.ConcurrencyPolicy = "always_create" + if err := svc.UpdateRoutine(ctx, routine); err != nil { + t.Fatalf("update routine: %v", err) + } + + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "one"}); err != nil { + t.Fatalf("first fire: %v", err) + } + if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "two"}); err != nil { + t.Fatalf("second fire: %v", err) + } + if len(enq.created) != 2 { + t.Fatalf("expected 2 wakeup-requests created, got %d", len(enq.created)) + } + first, second := enq.created[0].IdempotencyKey, enq.created[1].IdempotencyKey + if first == "" || second == "" { + t.Fatalf("expected non-empty keys, got %q and %q", first, second) + } + if first == second { + t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first) + } +} diff --git a/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go new file mode 100644 index 0000000000..97cae86d23 --- /dev/null +++ b/apps/backend/internal/office/routines/run_dedup_generation_keys_test.go @@ -0,0 +1,104 @@ +package routines + +import ( + "fmt" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/shared" +) + +// AC-OFFICE-RUN-DEDUP-001.1 / .4: a cron fire's key names the claimed +// scheduled slot, and redelivering the same slot (the same claimedTick) +// reproduces the same key byte for byte. +func TestBuildRoutineIdempotencyKey_Cron_UsesClaimedTick(t *testing.T) { + tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-ignored") + + want := fmt.Sprintf("routine:%s:%s:tick:%d", "routine-1", "trigger-1", tick.Unix()) + if key != want { + t.Fatalf("key = %q, want %q", key, want) + } + + // A second claim of the identical slot (redelivery) mints the same key. + repeat := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-different") + if repeat != key { + t.Fatalf("redelivery of the same slot produced %q, want %q (identical to the first)", repeat, key) + } +} + +// AC-OFFICE-RUN-DEDUP-001.2: two distinct cron slots (a genuinely later +// tick) mint two distinct keys. +func TestBuildRoutineIdempotencyKey_Cron_DistinctTicksDiffer(t *testing.T) { + tickOne := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + tickTwo := tickOne.Add(time.Minute) + + keyOne := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickOne, "run-a") + keyTwo := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickTwo, "run-b") + + if keyOne == keyTwo { + t.Fatalf("distinct cron slots must mint distinct keys, both got %q", keyOne) + } +} + +// AC-OFFICE-RUN-DEDUP-003.3: a cron fire with no claimed tick (the +// exported entry point called directly with a nil tick, bypassing the +// live processCronTrigger guard) has no occurrence identity and goes +// keyless with cause=unresolved. +func TestBuildRoutineIdempotencyKey_Cron_NoClaimedTickGoesKeyless(t *testing.T) { + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", nil, "run-1") + if key != "" { + t.Fatalf("key = %q, want empty (keyless) for a cron fire with no claimed tick", key) + } +} + +// Manual and webhook fires claim no slot, so two fires of one routine are +// two distinct occurrences by design: each RoutineRun.ID mints its own key. +func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing.T) { + for _, source := range []string{"manual", "webhook"} { + key := buildRoutineIdempotencyKey(source, "routine-1", "", "", nil, "run-1") + want := "routine:routine-1:run:run-1" + if key != want { + t.Errorf("source=%q key = %q, want %q", source, key, want) + } + } + + // Two distinct manual fires -> two distinct keys (the collision the + // old unix-minute key format could reach). + first := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-a") + second := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-b") + if first == second { + t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first) + } +} + +// An unrecognised RoutineRun.Source names no occurrence this table covers +// and goes keyless with cause=unresolved, the same direction as a cron +// fire with no claimed tick. +func TestBuildRoutineIdempotencyKey_UnrecognisedSourceGoesKeyless(t *testing.T) { + key := buildRoutineIdempotencyKey("some_future_source", "routine-1", "", "", nil, "run-1") + if key != "" { + t.Fatalf("key = %q, want empty (keyless) for an unrecognised source", key) + } +} + +// An explicit request key (a webhook delivery header, say) always wins, +// even over a claimed cron tick, so a caller-supplied idempotency key never +// collides with the source-derived identity scheme. +func TestBuildRoutineIdempotencyKey_ExplicitKeyTakesPriority(t *testing.T) { + tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + + key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "delivery-abc", &tick, "run-1") + want := fmt.Sprintf("routine:%s:%s:%s", "routine-1", shared.RoutineSourceCron, "delivery-abc") + if key != want { + t.Fatalf("key = %q, want %q (explicit key must take priority over claimedTick)", key, want) + } + + // Two webhook deliveries with distinct explicit keys mint distinct keys. + first := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-1", nil, "run-a") + second := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-2", nil, "run-a") + if first == second { + t.Fatalf("two distinct explicit keys must mint distinct keys, both got %q", first) + } +} diff --git a/apps/backend/internal/office/routines/service.go b/apps/backend/internal/office/routines/service.go index 3575cf6aa1..f22c8c57e0 100644 --- a/apps/backend/internal/office/routines/service.go +++ b/apps/backend/internal/office/routines/service.go @@ -15,6 +15,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/service" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" taskservice "github.com/kandev/kandev/internal/task/service" ) @@ -445,6 +446,11 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin zap.String("trigger_id", trigger.ID), zap.Error(err)) return err } + // The claimed tick: UpdateTriggerNextRun below advances the trigger + // row's next_run_at to the next slot, so trigger.NextRunAt is captured + // here, before that write, and carried through to the key builder + // rather than re-read afterward. + claimedTick := trigger.NextRunAt if err := s.repo.UpdateTriggerNextRun(ctx, trigger.ID, &advanceTo); err != nil { s.logger.Warn("update trigger next_run_at failed", zap.String("trigger_id", trigger.ID), zap.Error(err)) @@ -458,7 +464,7 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin // runCount-1 as missed_ticks ("you missed N since the last fire"). missedForPayload = runCount - 1 } - _, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload) + _, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload, claimedTick) return err } @@ -530,7 +536,7 @@ func (s *RoutineService) DispatchRoutineRun( source string, provided map[string]string, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0, nil) } // DispatchRoutineRunWithIdempotencyKey dispatches a fire with an explicit @@ -544,7 +550,7 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey( provided map[string]string, idempotencyKey string, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0, nil) } // DispatchRoutineRunWithMissed is the cron-tick entry point that @@ -554,6 +560,12 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey( // learns "you missed N-1 ticks" via wakeup.RoutinePayload.MissedTicks. // Manual fires (UI / API) call DispatchRoutineRun directly with no // missed-tick attribution. +// +// claimedTick is the scheduled tick processCronTrigger claimed off the +// trigger row before advancing it — the dedup key's occurrence identity +// for a cron fire. It travels as a parameter rather than being re-read +// from the trigger row, which by dispatch time already names the next +// slot. func (s *RoutineService) DispatchRoutineRunWithMissed( ctx context.Context, routine *Routine, @@ -561,8 +573,9 @@ func (s *RoutineService) DispatchRoutineRunWithMissed( source string, provided map[string]string, missedTicks int, + claimedTick *time.Time, ) (*RoutineRun, error) { - return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks) + return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks, claimedTick) } func (s *RoutineService) dispatchRoutineRun( @@ -573,6 +586,7 @@ func (s *RoutineService) dispatchRoutineRun( provided map[string]string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) (*RoutineRun, error) { now := time.Now().UTC() defaults := parseDeclaredDefaults(routine.Variables) @@ -640,7 +654,7 @@ func (s *RoutineService) dispatchRoutineRun( return run, nil } - if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil { + if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks, claimedTick); err != nil { return run, err } // A nil error only means materialiseRoutineRun's terminal write @@ -683,11 +697,12 @@ func (s *RoutineService) materialiseRoutineRun( source string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) error { if tmpl.Title != "" && s.workflowEnsurer != nil && s.taskCreator != nil { return s.materialiseHeavyRoutineRun(ctx, routine, run, title, description) } - return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks) + return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks, claimedTick) } // materialiseHeavyRoutineRun creates a real task in the routine system @@ -746,11 +761,12 @@ func (s *RoutineService) materialiseLightweightRoutineRun( source string, idempotencyKey string, missedTicks int, + claimedTick *time.Time, ) error { if s.wakeup == nil || routine.AssigneeAgentProfileID == "" { return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusDone) } - idemKey := buildRoutineIdempotencyKey(routine.ID, run.TriggerID, source, run.ID, idempotencyKey, run.StartedAt) + idemKey := buildRoutineIdempotencyKey(source, routine.ID, run.TriggerID, idempotencyKey, claimedTick, run.ID) payloadStr, _ := marshalRoutinePayload(routine.ID, vars, missedTicks) req := &WakeupRequest{ ID: uuid.New().String(), @@ -805,27 +821,37 @@ func (s *RoutineService) finalizeLightweightRun( } // buildRoutineIdempotencyKey composes the source-level dedup key for a -// routine fire. Cron fires use the trigger and minute bucket. Manual and -// webhook fires use a unique run identity unless the caller supplies an -// explicit request key, so distinct event deliveries never collide. +// routine fire. An explicit request key (a webhook delivery header, say) +// always wins, so distinct event deliveries never collide. Otherwise the +// occurrence identity depends on the source: a cron fire claims a scheduled +// slot off the trigger row (ClaimTrigger's compare-and-swap means exactly +// one RoutineRun exists per slot, so the slot - not the run - is the +// occurrence, and a catch-up collapsing several missed ticks into one fire +// is still one occurrence). A manual or webhook fire claims no slot, so +// RoutineRun.ID is its only durable distinguishing identity, and two such +// fires are two distinct occurrences by design. +// +// A cron source with no claimed tick, or a source this table does not +// recognise, has no occurrence identity to name and goes keyless. func buildRoutineIdempotencyKey( - routineID, triggerID, source, runID, explicitKey string, startedAt *time.Time, + source, routineID, triggerID, explicitKey string, claimedTick *time.Time, routineRunID string, ) string { if explicitKey != "" { return fmt.Sprintf("routine:%s:%s:%s", routineID, source, explicitKey) } - if source != shared.RoutineSourceCron { - return fmt.Sprintf("routine:%s:%s:%s", routineID, source, runID) + switch source { + case shared.RoutineSourceCron: + if claimedTick == nil { + runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "cron_no_claimed_tick") + return "" + } + return fmt.Sprintf("routine:%s:%s:tick:%d", routineID, triggerID, claimedTick.Unix()) + case "manual", "webhook": + return fmt.Sprintf("routine:%s:run:%s", routineID, routineRunID) + default: + runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "unrecognised_routine_source") + return "" } - now := time.Now().UTC() - if startedAt != nil { - now = *startedAt - } - minute := now.Unix() / 60 - if triggerID == "" { - return fmt.Sprintf("routine:%s:%s:%d", routineID, source, minute) - } - return fmt.Sprintf("routine:%s:%s:%d", routineID, triggerID, minute) } // marshalRoutinePayload renders the wakeup-request payload for a diff --git a/apps/backend/internal/office/runtime/actions.go b/apps/backend/internal/office/runtime/actions.go index bd955da02a..b046f3c61b 100644 --- a/apps/backend/internal/office/runtime/actions.go +++ b/apps/backend/internal/office/runtime/actions.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/kandev/kandev/internal/office/models" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // CommentWriter is the comment mutation dependency used by runtime actions. @@ -254,7 +255,7 @@ type ApprovalRequester interface { // RunSpawner is the run queue dependency used by runtime actions. type RunSpawner interface { - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // AgentModifier is the agent update dependency used by runtime actions. @@ -565,7 +566,22 @@ type SpawnAgentRunInput struct { IdempotencyKey string `json:"idempotency_key"` } +// maxSpawnAgentRunReasonLength bounds SpawnAgentRunInput.Reason. Reason is +// agent-supplied and reaches office_run_dedup_total / +// office_run_dedup_keyless_total as an expvar.Map label; those maps never +// evict, so an unbounded Reason would let a caller grow them without limit. +const maxSpawnAgentRunReasonLength = 100 + // SpawnAgentRun queues a run for an agent in the same workspace. +// +// A non-empty agent-supplied key is prefixed with the calling run's id +// (agent::) so a retry of the same run reuses the run id +// and still dedupes, while a later run gets a different prefix and is not +// suppressed. With no caller run id the request enqueues keyless +// (cause=unresolved) rather than risk colliding across runs. An empty key is +// NOT prefixed: the agent expressed no dedup intent (cause=by_design), and +// prefixing it would collapse every no-dedup-intent call inside one run onto +// a single key, suppressing every call after the first. func (a *Actions) SpawnAgentRun( ctx context.Context, runCtx RunContext, @@ -577,6 +593,9 @@ func (a *Actions) SpawnAgentRun( if a.deps.Runs == nil || a.deps.AgentModifier == nil { return fmt.Errorf("%w: runs", ErrRuntimeDependencyMissing) } + if len(input.Reason) > maxSpawnAgentRunReasonLength { + return ErrReasonTooLong + } target, err := a.deps.AgentModifier.GetAgentInstance(ctx, input.AgentID) if err != nil { return err @@ -592,7 +611,17 @@ func (a *Actions) SpawnAgentRun( if err != nil { return err } - return a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), input.IdempotencyKey) + key := "" + switch { + case input.IdempotencyKey == "": + runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseByDesign, "") + case runCtx.RunID != "": + key = fmt.Sprintf("agent:%s:%s", runCtx.RunID, input.IdempotencyKey) + default: + runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseUnresolved, "no_caller_run") + } + _, err = a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), key) + return err } // ModifyAgentInput contains agent fields an authorized runtime may update. diff --git a/apps/backend/internal/office/runtime/actions_test.go b/apps/backend/internal/office/runtime/actions_test.go index 6115385abc..9e922ab77d 100644 --- a/apps/backend/internal/office/runtime/actions_test.go +++ b/apps/backend/internal/office/runtime/actions_test.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) func TestCapabilitiesMarshalProjectCapabilityKeys(t *testing.T) { @@ -1221,14 +1222,14 @@ type spawnRunCall struct { func (r *recordingRunSpawner) QueueRun( _ context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { r.calls = append(r.calls, spawnRunCall{ AgentID: agentInstanceID, Reason: reason, Payload: payload, IdempotencyKey: idempotencyKey, }) - return nil + return runsservice.QueueOutcomeQueued, nil } type recordingAgentModifier struct { diff --git a/apps/backend/internal/office/runtime/errors.go b/apps/backend/internal/office/runtime/errors.go index 1e61e69cd7..d08f4c3823 100644 --- a/apps/backend/internal/office/runtime/errors.go +++ b/apps/backend/internal/office/runtime/errors.go @@ -20,4 +20,10 @@ var ( // either) in a workspace that has at least one project to choose from. // Caller-correctable: the caller should retry with an explicit project_id. ErrProjectRequired = fmt.Errorf("project_id is required") + // ErrReasonTooLong is returned when SpawnAgentRunInput.Reason exceeds + // maxSpawnAgentRunReasonLength. Reason is agent-supplied and becomes a + // label on the process-global office_run_dedup_total / + // office_run_dedup_keyless_total expvar maps, which never evict entries; + // an unbounded value lets a caller grow those maps without limit. + ErrReasonTooLong = fmt.Errorf("reason exceeds max length of %d characters", maxSpawnAgentRunReasonLength) ) diff --git a/apps/backend/internal/office/runtime/handler.go b/apps/backend/internal/office/runtime/handler.go index c315fb866b..2d779c4026 100644 --- a/apps/backend/internal/office/runtime/handler.go +++ b/apps/backend/internal/office/runtime/handler.go @@ -478,7 +478,7 @@ func (h *Handler) respondRuntimeError( targetID string, err error, ) { - if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) { + if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) || errors.Is(err, ErrReasonTooLong) { h.appendDeniedRunEvent(c.Request.Context(), runCtx, action, targetType, targetID, err) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return diff --git a/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go new file mode 100644 index 0000000000..2e9e0e5ece --- /dev/null +++ b/apps/backend/internal/office/runtime/spawn_agent_run_key_test.go @@ -0,0 +1,217 @@ +package runtime + +import ( + "context" + "errors" + "expvar" + "strings" + "testing" + + "github.com/kandev/kandev/internal/office/models" +) + +// spawnAgentRunKeylessCounterHasLabel reports whether the process-global +// office_run_dedup_keyless_total expvar map carries an entry for the given +// reason and cause. Mirrors internal/runs/service's own counterHasLabel test +// helper — that one lives in a different package (service_test) and cannot +// be imported here. +func spawnAgentRunKeylessCounterHasLabel(t *testing.T, reason, cause string) bool { + t.Helper() + v := expvar.Get("office_run_dedup_keyless_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_keyless_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") + } + found := false + m.Do(func(kv expvar.KeyValue) { + if strings.Contains(kv.Key, "reason="+reason) && strings.Contains(kv.Key, "cause="+cause) { + found = true + } + }) + return found +} + +// TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID is the AC-001.7 +// regression: a non-empty agent-supplied key is prefixed with the calling +// run's id (agent::) before reaching QueueRun, so a retry +// of the same run reuses the run id and still dedupes, while a later run +// (different RunID) produces a different key and is not suppressed. Before +// this test, the only coverage touching SpawnAgentRun's key logic was the +// unrelated cross-workspace denial test — the prefix itself was unpinned. +func TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-1", + Capabilities: Capabilities{ + CanSpawnAgentRun: true, + }, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "custom_reason", + IdempotencyKey: "custom-key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + want := "agent:run-caller-1:custom-key" + if got := runs.calls[0].IdempotencyKey; got != want { + t.Fatalf("idempotency key = %q, want %q", got, want) + } +} + +// TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey proves +// the prefix's other half: the SAME agent-supplied key from a DIFFERENT +// calling run does not collide, because the caller run id varies the key. +func TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + caps := Capabilities{CanSpawnAgentRun: true} + + for _, callerRunID := range []string{"run-a", "run-b"} { + runCtx := RunContext{WorkspaceID: "ws-1", RunID: callerRunID, Capabilities: caps} + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "custom_reason", + IdempotencyKey: "same-key", + }); err != nil { + t.Fatalf("SpawnAgentRun (caller %s): %v", callerRunID, err) + } + } + + if len(runs.calls) != 2 { + t.Fatalf("run spawner calls = %d, want 2", len(runs.calls)) + } + if runs.calls[0].IdempotencyKey == runs.calls[1].IdempotencyKey { + t.Fatalf("two different caller runs produced the same key %q", runs.calls[0].IdempotencyKey) + } +} + +// TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign proves the third +// branch: the agent expressed no dedup intent (empty IdempotencyKey), so the +// call enqueues keyless without a prefix — prefixing an empty key would +// collapse every no-dedup-intent call inside one run onto a single key, +// suppressing every call after the first. +func TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-2", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "test_spawn_empty_key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + if got := runs.calls[0].IdempotencyKey; got != "" { + t.Fatalf("idempotency key = %q, want empty (keyless by design)", got) + } + // Reason isn't in runs/service's bounded metricReasons allowlist, so it + // buckets to "custom" on the label rather than surviving verbatim. + if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "by_design") { + t.Fatal("expected office_run_dedup_keyless_total to carry a custom/by_design entry") + } +} + +// TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved proves +// the fourth branch: a non-empty agent-supplied key with no caller run id to +// prefix it enqueues keyless (cause=unresolved) rather than risk colliding +// across unrelated runs. +func TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: "test_spawn_no_caller_run", + IdempotencyKey: "custom-key", + }); err != nil { + t.Fatalf("SpawnAgentRun: %v", err) + } + + if len(runs.calls) != 1 { + t.Fatalf("run spawner calls = %d, want 1", len(runs.calls)) + } + if got := runs.calls[0].IdempotencyKey; got != "" { + t.Fatalf("idempotency key = %q, want empty (keyless, no caller run to prefix with)", got) + } + // Reason isn't in runs/service's bounded metricReasons allowlist, so it + // buckets to "custom" on the label rather than surviving verbatim. + if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "unresolved") { + t.Fatal("expected office_run_dedup_keyless_total to carry a custom/unresolved entry") + } +} + +// TestActionsSpawnAgentRun_ReasonTooLong_Rejected proves Reason is bounded +// before it can reach office_run_dedup_total / +// office_run_dedup_keyless_total as an expvar.Map label: those maps never +// evict entries, so an unvalidated agent-supplied Reason would let a caller +// grow them without limit. An over-length Reason is rejected with +// ErrReasonTooLong and never reaches the run spawner. +func TestActionsSpawnAgentRun_ReasonTooLong_Rejected(t *testing.T) { + agents := &recordingAgentModifier{ + agents: map[string]*models.AgentInstance{ + "agent-2": {ID: "agent-2", WorkspaceID: "ws-1"}, + }, + } + runs := &recordingRunSpawner{} + actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents}) + runCtx := RunContext{ + WorkspaceID: "ws-1", + RunID: "run-caller-3", + Capabilities: Capabilities{CanSpawnAgentRun: true}, + } + + err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{ + AgentID: "agent-2", + Reason: strings.Repeat("a", maxSpawnAgentRunReasonLength+1), + }) + if !errors.Is(err, ErrReasonTooLong) { + t.Fatalf("SpawnAgentRun error = %v, want ErrReasonTooLong", err) + } + if len(runs.calls) != 0 { + t.Fatalf("run spawner calls = %d, want 0 (rejected before enqueue)", len(runs.calls)) + } +} diff --git a/apps/backend/internal/office/scheduler/approval_adapter.go b/apps/backend/internal/office/scheduler/approval_adapter.go index 3c96ef96aa..f7d414a63c 100644 --- a/apps/backend/internal/office/scheduler/approval_adapter.go +++ b/apps/backend/internal/office/scheduler/approval_adapter.go @@ -45,7 +45,7 @@ func (a *DashboardApprovalAdapter) QueueApprovalRuns( DecisionComment: w.DecisionComment, IdempotencyKey: w.IdempotencyKey, } - if err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil { + if _, err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil { a.scheduler.logger.Warn("approval run failed: " + err.Error()) } } diff --git a/apps/backend/internal/office/scheduler/dashboard_adapter.go b/apps/backend/internal/office/scheduler/dashboard_adapter.go index 82187d003a..e5cdc2e9ab 100644 --- a/apps/backend/internal/office/scheduler/dashboard_adapter.go +++ b/apps/backend/internal/office/scheduler/dashboard_adapter.go @@ -75,12 +75,13 @@ func (a *DashboardReactivityAdapter) ApplyTaskMutation( // two packages. func convertChangeToMutation(c dashboard.TaskReactivityChange) TaskMutation { out := TaskMutation{ - NewStatus: c.NewStatus, - NewAssigneeID: c.NewAssigneeID, - ReopenIntent: c.ReopenIntent, - ResumeIntent: c.ResumeIntent, - ActorID: c.ActorID, - ActorType: c.ActorType, + NewStatus: c.NewStatus, + NewAssigneeID: c.NewAssigneeID, + AssignmentGeneration: c.AssignmentGeneration, + ReopenIntent: c.ReopenIntent, + ResumeIntent: c.ResumeIntent, + ActorID: c.ActorID, + ActorType: c.ActorType, } if c.Comment != nil { out.Comment = &MutationComment{ diff --git a/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go b/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go new file mode 100644 index 0000000000..2524711d79 --- /dev/null +++ b/apps/backend/internal/office/scheduler/queue_run_dedup_outcome_test.go @@ -0,0 +1,72 @@ +package scheduler + +import ( + "context" + "expvar" + "testing" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// windowedDedupCounterValue reads the current value of one +// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the +// label has never been reported. Callers snapshot before/after and assert +// the exact delta. +func windowedDedupCounterValue(t *testing.T, reason string) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs") + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason) + } + return i.Value() +} + +// TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome closes a +// phantom-green gap on the office-owned queue implementation the design +// names as least observable (run-dedup-generation-02.md#test-strategy): +// no test asserted SchedulerService.QueueRun actually calls +// ReportWindowedDedup, so deleting that call would leave every existing +// test green while AC-OFFICE-RUN-DEDUP-004.1 silently went unmet on the +// path a reassignment travels. +func TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-queue-run-outcome") + ctx := context.Background() + + key := "idem-key-scheduler-outcome" + first, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("first enqueue: %v", err) + } + if first != runsservice.QueueOutcomeQueued { + t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first) + } + + before := windowedDedupCounterValue(t, RunReasonTaskAssigned) + + second, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("second enqueue: %v", err) + } + if second != runsservice.QueueOutcomeDeduped { + t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second) + } + + after := windowedDedupCounterValue(t, RunReasonTaskAssigned) + if after != before+1 { + t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", RunReasonTaskAssigned, after, before+1) + } +} diff --git a/apps/backend/internal/office/scheduler/reactivity.go b/apps/backend/internal/office/scheduler/reactivity.go index 0b57aa1589..fc4d10f8ff 100644 --- a/apps/backend/internal/office/scheduler/reactivity.go +++ b/apps/backend/internal/office/scheduler/reactivity.go @@ -10,6 +10,9 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/commentkeys" + "github.com/kandev/kandev/internal/runs/dedupkeys" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // Canonical lowercase status values used inside the pipeline. Backend @@ -46,10 +49,17 @@ type TaskMutation struct { // What's changing. NewStatus *string // nil = unchanged NewAssigneeID *string - NewPriority *string - Comment *MutationComment // user/agent comment if this mutation includes one - ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress) - ResumeIntent bool // explicit resume=true (validated upstream to require comment) + // AssignmentGeneration is the value the assigning transaction committed, + // carried here rather than re-read (a producer reading the task's + // current generation after the fact can observe a later occurrence's + // value — see docs/specs/office/system-design/run-dedup-generation-01.md + // #carrying-the-generation). Nil means the caller could not supply one; + // reactToAssigneeChange then enqueues keyless rather than guess. + AssignmentGeneration *int64 + NewPriority *string + Comment *MutationComment // user/agent comment if this mutation includes one + ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress) + ResumeIntent bool // explicit resume=true (validated upstream to require comment) // Who is acting. ActorID string @@ -115,13 +125,17 @@ func (ss *SchedulerService) ApplyTaskMutation( return } seen[key] = struct{}{} - if err := ss.QueueRunCtx(ctx, agentID, c); err != nil { + outcome, err := ss.QueueRunCtx(ctx, agentID, c) + if err != nil { ss.logger.Error("reactivity run failed", zap.String("agent", agentID), zap.String("reason", c.Reason), zap.Error(err)) return } + if outcome != runsservice.QueueOutcomeQueued { + return + } res.Runs = append(res.Runs, QueuedRunSummary{ AgentID: agentID, Reason: c.Reason, TaskID: c.TaskID, }) @@ -133,7 +147,12 @@ func (ss *SchedulerService) ApplyTaskMutation( } // --- Assignee handoff --- - if change.NewAssigneeID != nil && *change.NewAssigneeID != task.AssigneeAgentProfileID { + // Fires on every non-nil NewAssigneeID, including a repeat assignment to + // the agent that already holds the seat: that is a real occurrence (the + // operator is asking for the work again), not a no-op, so this no longer + // gates on the two ids differing. reactToAssigneeChange itself guards the + // session interrupt. + if change.NewAssigneeID != nil { ss.reactToAssigneeChange(task, *change.NewAssigneeID, change, queue, res) } @@ -180,7 +199,13 @@ func (ss *SchedulerService) reactToStatusChange( ss.cascadeReviewRequested(ctx, task, change, queue) case prev == statusBlocked && next != statusBlocked: - // Unblocked — wake assignee with task_unblocked. + // Unblocked — no durable occurrence row to key on: keyless by design. + // Reported only when there's an assignee to enqueue for — queue() + // silently drops an empty agent id, so reporting unconditionally + // would count an enqueue that never happened. + if task.AssigneeAgentProfileID != "" { + runsservice.ReportKeylessEnqueue(RunReasonTaskUnblocked, runsservice.KeylessCauseByDesign, "") + } queue(task.AssigneeAgentProfileID, RunContext{ Reason: RunReasonTaskUnblocked, TaskID: task.ID, @@ -190,7 +215,7 @@ func (ss *SchedulerService) reactToStatusChange( }) case (prev == statusDone || prev == statusCancelled) && (next == statusTodo || next == statusInProgress): - // Reopen — different reason if a comment was attached. + // Reopen — different reason (and dedup identity) if a comment was attached. reason := RunReasonTaskReopened commentID := "" if change.Comment != nil { @@ -201,13 +226,23 @@ func (ss *SchedulerService) reactToStatusChange( // Explicit resume always uses the comment-flavoured reason. reason = RunReasonTaskReopenedComment } + var key string + if reason == RunReasonTaskReopenedComment { + key = fmt.Sprintf("%s:%s:%s", RunReasonTaskReopenedComment, commentID, task.AssigneeAgentProfileID) + } else if task.AssigneeAgentProfileID != "" { + // Silent reopen (no comment) — no durable occurrence row: keyless + // by design. Reported only when there's an assignee to enqueue + // for, matching the task_unblocked case above. + runsservice.ReportKeylessEnqueue(RunReasonTaskReopened, runsservice.KeylessCauseByDesign, "") + } queue(task.AssigneeAgentProfileID, RunContext{ - Reason: reason, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: change.ActorID, - ActorType: change.ActorType, - CommentID: commentID, + Reason: reason, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: change.ActorID, + ActorType: change.ActorType, + CommentID: commentID, + IdempotencyKey: key, }) } } @@ -222,6 +257,13 @@ func (ss *SchedulerService) reactToStatusChange( // // The previous assignee is NOT separately notified — interrupting their // run is the signal that they're no longer in charge. +// +// ApplyTaskMutation now calls this for every non-nil NewAssigneeID, +// including a repeat assignment to the agent that already holds the seat. +// The interrupt must NOT fire for that case — it would hard-cancel the +// agent's own in-flight run — so unlike the removed caller-side equality +// gate, this comparison stays local to the interrupt decision and does not +// also guard the wake. func (ss *SchedulerService) reactToAssigneeChange( task *TaskSnapshot, newAssigneeID string, @@ -229,25 +271,36 @@ func (ss *SchedulerService) reactToAssigneeChange( queue func(string, RunContext), res *ApplyTaskMutationResult, ) { - // Cancel the prior assignee's session if there was one. We re-use - // InterruptSessionID — status→cancelled also sets it; either reason - // for hard-cancelling produces the same downstream call. - if task.AssigneeAgentProfileID != "" && res.InterruptSessionID == "" { + if task.AssigneeAgentProfileID != "" && newAssigneeID != task.AssigneeAgentProfileID { res.InterruptSessionID = task.ID } - // Wake the new assignee. + if newAssigneeID == "" { + // Unassignment: the interrupt above (if any) already fired; queue's + // own empty-agent-id guard would catch this too, but there is no key + // to build or keyless cause to report for a run that will never be + // attempted. + return + } + commentID := "" if change.Comment != nil { commentID = change.Comment.ID } + var key string + if change.AssignmentGeneration != nil { + key = dedupkeys.AssignmentKey(task.ID, newAssigneeID, *change.AssignmentGeneration) + } else { + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "nil_mutation_generation") + } queue(newAssigneeID, RunContext{ - Reason: RunReasonTaskAssigned, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: change.ActorID, - ActorType: change.ActorType, - CommentID: commentID, + Reason: RunReasonTaskAssigned, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: change.ActorID, + ActorType: change.ActorType, + CommentID: commentID, + IdempotencyKey: key, }) } @@ -262,13 +315,15 @@ func (ss *SchedulerService) reactToComment( // Assignee wake — skip if self-comment or task is closed. if !comment.SkipAssigneeWake && !selfComment && !closed { + key := commentkeys.TaskComment(comment.ID) + ":" + task.AssigneeAgentProfileID queue(task.AssigneeAgentProfileID, RunContext{ - Reason: RunReasonTaskComment, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: comment.AuthorID, - ActorType: comment.AuthorType, - CommentID: comment.ID, + Reason: RunReasonTaskComment, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: comment.AuthorID, + ActorType: comment.AuthorType, + CommentID: comment.ID, + IdempotencyKey: key, }) } @@ -286,13 +341,15 @@ func (ss *SchedulerService) reactToComment( if agentID == comment.AuthorID { continue } + key := fmt.Sprintf("%s:%s:%s", RunReasonTaskMentioned, comment.ID, agentID) queue(agentID, RunContext{ - Reason: RunReasonTaskMentioned, - TaskID: task.ID, - WorkspaceID: task.WorkspaceID, - ActorID: comment.AuthorID, - ActorType: comment.AuthorType, - CommentID: comment.ID, + Reason: RunReasonTaskMentioned, + TaskID: task.ID, + WorkspaceID: task.WorkspaceID, + ActorID: comment.AuthorID, + ActorType: comment.AuthorType, + CommentID: comment.ID, + IdempotencyKey: key, }) } } @@ -320,6 +377,7 @@ func (ss *SchedulerService) cascadeReviewRequested( zap.String("task_id", task.ID), zap.Error(err)) return } + reported := map[string]struct{}{} for _, p := range parts { if p.AgentProfileID == "" { continue @@ -327,6 +385,14 @@ func (ss *SchedulerService) cascadeReviewRequested( if p.Role != models.ParticipantRoleReviewer && p.Role != models.ParticipantRoleApprover { continue } + // No durable occurrence row to key on: keyless by design, reported + // once per distinct agent rather than per role — an agent seated as + // both reviewer and approver is one recipient, and queue()'s own + // seen-map dedup only attempts one enqueue for it. + if _, dup := reported[p.AgentProfileID]; !dup { + reported[p.AgentProfileID] = struct{}{} + runsservice.ReportKeylessEnqueue(RunReasonTaskReviewRequested, runsservice.KeylessCauseByDesign, "") + } queue(p.AgentProfileID, RunContext{ Reason: RunReasonTaskReviewRequested, TaskID: task.ID, @@ -352,19 +418,34 @@ func (ss *SchedulerService) cascadeBlockersResolved( } for _, blockedID := range blockedTaskIDs { // Verify all OTHER blockers are also resolved. - ready, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID) - if err != nil || !ready { + ready, blockers, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID) + if err != nil { + ss.logger.Error("check blockers resolved failed", + zap.String("task_id", blockedID), zap.Error(err)) + continue + } + if !ready { continue } assignee, err := ss.repo.GetTaskAssignee(ctx, blockedID) if err != nil || assignee == "" { continue } + blockerIDs := make([]string, 0, len(blockers)) + for _, b := range blockers { + blockerIDs = append(blockerIDs, b.BlockerTaskID) + } + if len(blockerIDs) == 0 { + continue + } + key := fmt.Sprintf("%s:%s:%s:%s", + RunReasonTaskBlockersResolved, blockedID, assignee, dedupkeys.BlockerDigest(blockerIDs)) queue(assignee, RunContext{ Reason: RunReasonTaskBlockersResolved, TaskID: blockedID, WorkspaceID: task.WorkspaceID, ResolvedBlockerTaskID: task.ID, + IdempotencyKey: key, }) } } @@ -443,13 +524,17 @@ func childrenCompletedIdempotencyKey(parentID, agentID string, children []sqlite } // allBlockersResolvedExcept returns true if every blocker on `taskID` -// other than `excludeBlockerID` is in a terminal step. +// other than `excludeBlockerID` is in a terminal step, alongside the full +// blocker-task-id set it read to decide — the caller digests that same +// slice for the wake's dedup key rather than re-reading it (AC-001.9 +// applied to a set: a second read could observe a different set than the +// one this readiness decision was actually made against). func (ss *SchedulerService) allBlockersResolvedExcept( ctx context.Context, taskID, excludeBlockerID string, -) (bool, error) { +) (bool, []*models.TaskBlocker, error) { blockers, err := ss.repo.ListTaskBlockers(ctx, taskID) if err != nil { - return false, err + return false, nil, err } for _, b := range blockers { if b.BlockerTaskID == excludeBlockerID { @@ -457,13 +542,13 @@ func (ss *SchedulerService) allBlockersResolvedExcept( } done, err := ss.repo.IsTaskInTerminalStep(ctx, b.BlockerTaskID) if err != nil { - return false, err + return false, nil, err } if !done { - return false, nil + return false, blockers, nil } } - return true, nil + return true, blockers, nil } // normalisedStatus maps both backend uppercase task states (TODO, diff --git a/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go b/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go new file mode 100644 index 0000000000..1c82bce74b --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_apply_mutation_test.go @@ -0,0 +1,138 @@ +package scheduler + +import ( + "context" + "testing" +) + +// TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt +// pins the reactivity gate relaxation this feature made: ApplyTaskMutation +// now calls reactToAssigneeChange for EVERY non-nil NewAssigneeID, including +// a repeat assignment to the agent that already holds the seat, because that +// is a real occurrence (the operator asking for the work again) rather than +// a no-op. Before this feature the caller-side equality gate suppressed it +// entirely. The relocated interrupt-comparison guard must still hold: a +// same-agent repeat must NOT hard-cancel the agent's own in-flight session. +// Before this test, ApplyTaskMutation/reactToStatusChange/ +// reactToAssigneeChange were at 0.0% coverage. +func TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-repeat") + + task := &TaskSnapshot{ + ID: "task-repeat-assign", + WorkspaceID: "ws-1", + AssigneeAgentProfileID: "agent-repeat", // already holds the seat + } + newAssignee := "agent-repeat" + gen := int64(1) + change := TaskMutation{ + NewAssigneeID: &newAssignee, + AssignmentGeneration: &gen, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if res.InterruptSessionID != "" { + t.Fatalf("same-agent repeat assignment set InterruptSessionID = %q, want empty (must not cancel its own run)", res.InterruptSessionID) + } + found := false + for _, r := range res.Runs { + if r.AgentID == "agent-repeat" && r.Reason == RunReasonTaskAssigned { + found = true + } + } + if !found { + t.Fatalf("expected a task_assigned run queued for agent-repeat, got %+v", res.Runs) + } +} + +// TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee +// is the positive counterpart: reassigning to a DIFFERENT agent must still +// hard-cancel the previous assignee's session — the interrupt guard's +// comparison (moved inside reactToAssigneeChange by this feature) must not +// have accidentally suppressed the case it was already handling. +func TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-old") + createChildrenCompletedAgent(t, repo, "agent-new") + + task := &TaskSnapshot{ + ID: "task-reassign", + WorkspaceID: "ws-1", + AssigneeAgentProfileID: "agent-old", + } + newAssignee := "agent-new" + gen := int64(1) + change := TaskMutation{ + NewAssigneeID: &newAssignee, + AssignmentGeneration: &gen, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if res.InterruptSessionID != task.ID { + t.Fatalf("InterruptSessionID = %q, want %q (previous assignee's session must be cancelled)", res.InterruptSessionID, task.ID) + } + found := false + for _, r := range res.Runs { + if r.AgentID == "agent-new" && r.Reason == RunReasonTaskAssigned { + found = true + } + } + if !found { + t.Fatalf("expected a task_assigned run queued for agent-new, got %+v", res.Runs) + } +} + +// TestApplyTaskMutation_StatusChangeAndCommentDispatch drives +// ApplyTaskMutation's NewStatus and Comment branches together (an unblock +// plus an attached comment), covering the dispatcher itself rather than only +// the reactToStatusChange/reactToComment helpers it delegates to. +func TestApplyTaskMutation_StatusChangeAndCommentDispatch(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-dispatch") + + task := &TaskSnapshot{ + ID: "task-dispatch", + WorkspaceID: "ws-1", + State: "BLOCKED", + AssigneeAgentProfileID: "agent-dispatch", + } + newStatus := "todo" + change := TaskMutation{ + NewStatus: &newStatus, + Comment: &MutationComment{ID: "comment-dispatch", AuthorType: "user", AuthorID: "user-1"}, + ActorID: "user-1", + ActorType: "user", + } + + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + reasons := map[string]bool{} + for _, r := range res.Runs { + reasons[r.Reason] = true + } + if !reasons[RunReasonTaskUnblocked] { + t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskUnblocked, res.Runs) + } + if !reasons[RunReasonTaskComment] { + t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskComment, res.Runs) + } +} diff --git a/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go b/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go new file mode 100644 index 0000000000..380bc5310e --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_blockers_resolved_test.go @@ -0,0 +1,117 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +// insertBlockerRelationship records that taskID is blocked by +// blockerTaskID, and gives blockerTaskID the given (already-terminal or +// not) state. +func insertBlockerRelationship( + t *testing.T, repo *officesqlite.Repository, taskID, blockerTaskID, blockerState string, +) { + t.Helper() + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + blockerTaskID, blockerState, + ); err != nil { + t.Fatalf("insert blocker task %s: %v", blockerTaskID, err) + } + if _, err := repo.ExecRaw(ctx, + `INSERT INTO task_blockers (task_id, blocker_task_id, created_at) VALUES (?, ?, CURRENT_TIMESTAMP)`, + taskID, blockerTaskID, + ); err != nil { + t.Fatalf("insert task_blockers row (%s blocked by %s): %v", taskID, blockerTaskID, err) + } +} + +// TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey is the +// regression test for both blocker-resolution producers converging on the +// same key shape (AC-OFFICE-RUN-DEDUP-002.1, SR-59's resolution: only the +// digest need be byte-identical, not the whole key/opID — see +// docs/specs/office/system-design/run-dedup-generation-03.md#the-shared-key-builders). +// Before this test, office/scheduler.cascadeBlockersResolved and +// allBlockersResolvedExcept had zero coverage: nothing exercised the +// blocked task's wake at all, let alone the specific +// dedupkeys.BlockerDigest call. +func TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-1") + queue := newChildrenCompletedQueue(t, ss) + ctx := context.Background() + + // blocked-1 is blocked by blocker-a (already done) and blocker-b (the + // one resolving now). setupChildrenCompletedParent gives blocked-1 a + // runner participant row so GetTaskAssignee resolves it to agent-1 — + // reused here even though it's named for the children-completed tests, + // because it does exactly what a blocked task's assignee wiring needs. + setupChildrenCompletedParent(t, ss, "blocked-1", "agent-1") + insertBlockerRelationship(t, repo, "blocked-1", "blocker-a", "COMPLETED") + insertBlockerRelationship(t, repo, "blocked-1", "blocker-b", "IN_PROGRESS") + + // blocker-b finishes: it's the second and last outstanding blocker. + if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-b'`); err != nil { + t.Fatalf("complete blocker-b: %v", err) + } + ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-b", WorkspaceID: "ws-1"}, queue) + + runs, err := repo.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + var got *string + for _, run := range runs { + if run.Reason == RunReasonTaskBlockersResolved && run.AgentProfileID == "agent-1" { + got = run.IdempotencyKey + break + } + } + if got == nil { + t.Fatalf("no %s run persisted for agent-1: runs=%#v", RunReasonTaskBlockersResolved, runs) + } + + // The key's digest segment must be exactly what the OTHER blocker + // producer (office/service.resolveAndWakeIfUnblocked) would derive + // for the identical blocker set, in either read order — that's the + // structural convergence AC-002.1 requires. + digestAscending := dedupkeys.BlockerDigest([]string{"blocker-a", "blocker-b"}) + digestDescending := dedupkeys.BlockerDigest([]string{"blocker-b", "blocker-a"}) + if digestAscending != digestDescending { + t.Fatalf("BlockerDigest is not order-independent: %q vs %q", digestAscending, digestDescending) + } + want := fmt.Sprintf("%s:blocked-1:agent-1:%s", RunReasonTaskBlockersResolved, digestAscending) + if *got != want { + t.Fatalf("idempotency key = %q, want %q", *got, want) + } +} + +// TestCascadeBlockersResolved_StillBlocked_DoesNotQueue proves the +// negative: cascadeBlockersResolved must not wake the blocked task while +// another blocker is still outstanding. +func TestCascadeBlockersResolved_StillBlocked_DoesNotQueue(t *testing.T) { + repo := newReactivityTestRepo(t) + ss := newChildrenCompletedTestScheduler(t, repo) + createChildrenCompletedAgent(t, repo, "agent-1") + queue := newChildrenCompletedQueue(t, ss) + ctx := context.Background() + + setupChildrenCompletedParent(t, ss, "blocked-2", "agent-1") + insertBlockerRelationship(t, repo, "blocked-2", "blocker-c", "IN_PROGRESS") // never finishes + insertBlockerRelationship(t, repo, "blocked-2", "blocker-d", "IN_PROGRESS") + + if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-d'`); err != nil { + t.Fatalf("complete blocker-d: %v", err) + } + ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-d", WorkspaceID: "ws-1"}, queue) + + if got := runsCountForReason(t, ss, RunReasonTaskBlockersResolved); got != 0 { + t.Fatalf("persisted runs = %d, want 0 (blocker-c is still unresolved)", got) + } +} diff --git a/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go b/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go index 01ae6fbabe..d80caafc53 100644 --- a/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go +++ b/apps/backend/internal/office/scheduler/reactivity_children_completed_test.go @@ -50,7 +50,7 @@ func createChildrenCompletedAgent(t *testing.T, repo *officesqlite.Repository, i func newChildrenCompletedQueue(t *testing.T, ss *SchedulerService) func(string, RunContext) { t.Helper() return func(agentID string, c RunContext) { - if err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil { + if _, err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil { t.Fatalf("QueueRunCtx: %v", err) } } diff --git a/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go new file mode 100644 index 0000000000..1dd0e3944b --- /dev/null +++ b/apps/backend/internal/office/scheduler/reactivity_producer_key_audit_test.go @@ -0,0 +1,372 @@ +package scheduler + +import ( + "context" + "expvar" + "testing" + + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/commentkeys" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +// newAuditScheduler wires a real repo + real *service.Service (reused from +// reactivity_children_completed_test.go's newChildrenCompletedTestScheduler) +// so every producer under audit — including the two (task_blockers_resolved, +// task_review_requested) that read participants/blockers off the repo, and +// task_mentioned, which resolves mentions via ss.svc.ListAgentInstances — +// can run unmodified. +func newAuditScheduler(t *testing.T) (*SchedulerService, *officesqlite.Repository) { + t.Helper() + repo := newReactivityTestRepo(t) + return newChildrenCompletedTestScheduler(t, repo), repo +} + +// requireOneCallKey finds the single recorded queue call for (reason, agentID) +// and returns its IdempotencyKey, failing the test if no such call exists. +func requireOneCallKey(t *testing.T, calls []recordedQueueCall, reason, agentID string) string { + t.Helper() + for _, c := range calls { + if c.agentID == agentID && c.ctx.Reason == reason { + return c.ctx.IdempotencyKey + } + } + t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls) + return "" +} + +// requireOneCallKeyless is requireOneCallKey's keyless counterpart: it +// asserts the matching call carries NO idempotency key, rather than +// asserting a value. +func requireOneCallKeyless(t *testing.T, calls []recordedQueueCall, reason, agentID string) { + t.Helper() + for _, c := range calls { + if c.agentID == agentID && c.ctx.Reason == reason { + if c.ctx.IdempotencyKey != "" { + t.Fatalf("%s call for agent %q carries key %q, want keyless", reason, agentID, c.ctx.IdempotencyKey) + } + return + } + } + t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls) +} + +// auditKeylessCounterValue returns the current value of the process-global +// office_run_dedup_keyless_total expvar map (internal/runs/service) for the +// exact "reason=;cause=" label, or 0 if that label has never +// been incremented. Duplicated locally rather than imported: the counter map +// is unexported and the sibling helper in internal/runs/service/dedup_test.go +// lives in a different package. Callers snapshot before and after a producer +// call and assert the delta, rather than merely that the label exists — a +// mere-existence check would pass even if this specific call site failed to +// report, as long as some other test already set that label. +func auditKeylessCounterValue(t *testing.T, reason, cause string) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_keyless_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_keyless_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";cause=" + cause) + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;cause=%s is not *expvar.Int", reason, cause) + } + return i.Value() +} + +// TestReactivityProducerKeyAudit is the Part 2-required "producer-audit +// key-format table test" (docs/specs/office/system-design/ +// run-dedup-generation-02.md#test-strategy): a table over every reactivity +// producer named in the "Generation sources per producer" table +// (…-01.md#generation-sources-per-producer), asserting each one derives the +// spec's exact key shape, or reports the spec's exact keyless cause when the +// reason has no durable occurrence to key on. Review round 1 found 6 of this +// file's 8 run reasons silently doing neither — this is the design's own +// named enforcement mechanism whose absence is why that gap shipped +// undetected, and what stops the next reason added to reactivity.go from +// reintroducing it silently. +func TestReactivityProducerKeyAudit(t *testing.T) { + t.Run("task_assigned", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + task := &TaskSnapshot{ID: "task-audit-assigned", WorkspaceID: "ws-1"} + gen := int64(3) + res := &ApplyTaskMutationResult{} + ss.reactToAssigneeChange(task, "agent-assigned", TaskMutation{ + AssignmentGeneration: &gen, ActorID: "user-1", ActorType: "user", + }, queue, res) + + key := requireOneCallKey(t, calls, RunReasonTaskAssigned, "agent-assigned") + if want := dedupkeys.AssignmentKey(task.ID, "agent-assigned", gen); key != want { + t.Fatalf("task_assigned key = %q, want %q", key, want) + } + }) + + t.Run("task_blockers_resolved", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + setupChildrenCompletedParent(t, ss, "blocked-audit", "agent-blocked") + insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-a", "COMPLETED") + insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-b", "COMPLETED") + + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.cascadeBlockersResolved(context.Background(), + &TaskSnapshot{ID: "blocker-audit-b", WorkspaceID: "ws-1"}, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskBlockersResolved, "agent-blocked") + digest := dedupkeys.BlockerDigest([]string{"blocker-audit-a", "blocker-audit-b"}) + want := RunReasonTaskBlockersResolved + ":blocked-audit:agent-blocked:" + digest + if key != want { + t.Fatalf("task_blockers_resolved key = %q, want %q", key, want) + } + }) + + t.Run("task_comment", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-comment", WorkspaceID: "ws-1", AssigneeAgentProfileID: "agent-assignee"} + comment := &MutationComment{ID: "comment-audit-1", Body: "status update", AuthorType: "user", AuthorID: "user-1"} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.reactToComment(context.Background(), task, comment, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskComment, "agent-assignee") + want := commentkeys.TaskComment("comment-audit-1") + ":agent-assignee" + if key != want { + t.Fatalf("task_comment key = %q, want %q", key, want) + } + }) + + t.Run("task_mentioned", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + createChildrenCompletedAgent(t, repo, "agent-mentioned") + task := &TaskSnapshot{ID: "task-audit-mentioned", WorkspaceID: "ws-1"} + comment := &MutationComment{ + ID: "comment-audit-2", AuthorType: "user", AuthorID: "user-1", + // The mention token regex allows internal spaces (for multi-word + // agent names), so it greedily consumes everything after the "@" + // up to end of string or a disallowed character. Keeping the + // mention last with nothing trailing bounds the capture to + // exactly the agent name. + Body: "cc @agent-mentioned", + // Isolate the mention path from the assignee-wake path (task has + // no assignee here, but SkipAssigneeWake documents the intent). + SkipAssigneeWake: true, + } + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + ss.reactToComment(context.Background(), task, comment, queue) + + key := requireOneCallKey(t, calls, RunReasonTaskMentioned, "agent-mentioned") + want := RunReasonTaskMentioned + ":comment-audit-2:agent-mentioned" + if key != want { + t.Fatalf("task_mentioned key = %q, want %q", key, want) + } + }) + + t.Run("task_reopened_via_comment", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-comment", WorkspaceID: "ws-1", State: "DONE", AssigneeAgentProfileID: "agent-reopen"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + change := TaskMutation{ + Comment: &MutationComment{ID: "comment-audit-3", AuthorType: "user", AuthorID: "user-1"}, + ActorID: "user-1", + ActorType: "user", + } + ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res) + + key := requireOneCallKey(t, calls, RunReasonTaskReopenedComment, "agent-reopen") + want := RunReasonTaskReopenedComment + ":comment-audit-3:agent-reopen" + if key != want { + t.Fatalf("task_reopened_via_comment key = %q, want %q", key, want) + } + }) + + t.Run("task_unblocked", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-unblocked", WorkspaceID: "ws-1", State: "BLOCKED", AssigneeAgentProfileID: "agent-unblocked"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design") + change := TaskMutation{ActorID: "user-1", ActorType: "user"} + ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res) + + requireOneCallKeyless(t, calls, RunReasonTaskUnblocked, "agent-unblocked") + if got, want := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want %d", got, want) + } + }) + + t.Run("task_unblocked_no_assignee_does_not_report", func(t *testing.T) { + // Regression test for Review round 2 Finding 3: an unassigned task + // must not increment the keyless counter for an enqueue that never + // happens. Goes through ApplyTaskMutation (not reactToStatusChange + // directly) so the real queue closure's empty-agent-id guard + // (reactivity.go's `if agentID == "" { return }`) is exercised — + // that guard is exactly what silently swallowed the enqueue while + // FINDING 3's ReportKeylessEnqueue call still fired unconditionally. + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-unblocked-noassignee", WorkspaceID: "ws-1", State: "BLOCKED"} + before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design") + newStatus := "todo" + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if len(res.Runs) != 0 { + t.Fatalf("expected no queued runs for an unassigned task_unblocked, got %#v", res.Runs) + } + if got := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"); got != before { + t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before) + } + }) + + t.Run("task_reopened", func(t *testing.T) { + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-silent", WorkspaceID: "ws-1", State: "CANCELLED", AssigneeAgentProfileID: "agent-silent-reopen"} + res := &ApplyTaskMutationResult{} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design") + change := TaskMutation{ActorID: "user-1", ActorType: "user"} + ss.reactToStatusChange(context.Background(), task, "in_progress", change, queue, res) + + requireOneCallKeyless(t, calls, RunReasonTaskReopened, "agent-silent-reopen") + if got, want := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want %d", got, want) + } + }) + + t.Run("task_reopened_no_assignee_does_not_report", func(t *testing.T) { + // Regression test for Review round 2 Finding 3, silent-reopen branch. + // Goes through ApplyTaskMutation for the same reason as the + // task_unblocked case above: the empty-agent-id guard lives in the + // real queue closure, not in reactToStatusChange itself. + ss, _ := newAuditScheduler(t) + task := &TaskSnapshot{ID: "task-audit-reopen-silent-noassignee", WorkspaceID: "ws-1", State: "CANCELLED"} + before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design") + newStatus := "in_progress" + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(context.Background(), task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + if len(res.Runs) != 0 { + t.Fatalf("expected no queued runs for an unassigned silent reopen, got %#v", res.Runs) + } + if got := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"); got != before { + t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before) + } + }) + + t.Run("task_review_requested", func(t *testing.T) { + ss, repo := newAuditScheduler(t) + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review', 'ws-1', 'step-audit') + `); err != nil { + t.Fatalf("insert task: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-reviewer', 'step-audit', 'task-audit-review', 'reviewer', 'agent-reviewer-audit', 1, 0) + `); err != nil { + t.Fatalf("insert participant: %v", err) + } + + task := &TaskSnapshot{ID: "task-audit-review", WorkspaceID: "ws-1"} + var calls []recordedQueueCall + queue := func(agentID string, c RunContext) { + calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c}) + } + before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design") + ss.cascadeReviewRequested(ctx, task, TaskMutation{ActorID: "user-1", ActorType: "user"}, queue) + + requireOneCallKeyless(t, calls, RunReasonTaskReviewRequested, "agent-reviewer-audit") + if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d", got, want) + } + }) + + t.Run("task_review_requested_dual_role_recipient_counts_once", func(t *testing.T) { + // Regression test for Review round 3 Finding 4: an agent seated as + // both reviewer and approver on the same task is one recipient, not + // two. Driven through the real ApplyTaskMutation (not a bare + // closure) so the real queue closure's seen-map dedup — which + // silently absorbs the second queue() call for the same agent — + // is what proves the counter must match the number of enqueue + // attempts actually made, not the number of participant rows read. + ss, repo := newAuditScheduler(t) + ctx := context.Background() + createChildrenCompletedAgent(t, repo, "agent-dual-role") + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review-dual', 'ws-1', 'step-audit') + `); err != nil { + t.Fatalf("insert task: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-dual-reviewer', 'step-audit', 'task-audit-review-dual', 'reviewer', 'agent-dual-role', 1, 0) + `); err != nil { + t.Fatalf("insert reviewer participant: %v", err) + } + if _, err := repo.ExecRaw(ctx, ` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position) + VALUES ('p-audit-dual-approver', 'step-audit', 'task-audit-review-dual', 'approver', 'agent-dual-role', 1, 1) + `); err != nil { + t.Fatalf("insert approver participant: %v", err) + } + + task := &TaskSnapshot{ID: "task-audit-review-dual", WorkspaceID: "ws-1", State: statusTodo} + before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design") + newStatus := statusInReview + change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"} + res, err := ss.ApplyTaskMutation(ctx, task, change) + if err != nil { + t.Fatalf("ApplyTaskMutation: %v", err) + } + + queued := 0 + for _, r := range res.Runs { + if r.Reason == RunReasonTaskReviewRequested && r.AgentID == "agent-dual-role" { + queued++ + } + } + if queued != 1 { + t.Fatalf("expected exactly 1 queued task_review_requested run for the dual-role agent, got %d (%#v)", queued, res.Runs) + } + if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want { + t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d (one recipient, not one row per role)", got, want) + } + }) +} diff --git a/apps/backend/internal/office/scheduler/run.go b/apps/backend/internal/office/scheduler/run.go index 87d57ab4ec..47ab506196 100644 --- a/apps/backend/internal/office/scheduler/run.go +++ b/apps/backend/internal/office/scheduler/run.go @@ -19,6 +19,7 @@ import ( "github.com/kandev/kandev/internal/office/routing" "github.com/kandev/kandev/internal/office/service" "github.com/kandev/kandev/internal/office/shared" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // ErrRoutingNotSupported is returned by TaskStarter.StartTaskWithRoute @@ -107,13 +108,13 @@ type RunContext struct { // task_changes_requested has the context inline. DecisionComment string `json:"decision_comment,omitempty"` - // IdempotencyKey, when non-empty, overrides the default - // "{reason}:{taskID}:{agentID}" key QueueRunCtx mints. The default - // key is permanently unique per (reason, task, agent) — fine for - // reasons that fire at most once per task, wrong for a reason that - // can legitimately recur (e.g. task_children_completed across - // repeated delegation waves). Callers that recur build their own - // key that changes with the thing that makes each occurrence + // IdempotencyKey is the dedup identity QueueRunCtx passes through + // verbatim. Empty means no dedup — the run enqueues keyless — not + // "derive one for me": QueueRunCtx no longer synthesises a + // "{reason}:{taskID}:{agentID}" default, which was permanently unique + // per (reason, task, agent) and silently swallowed every later + // legitimate occurrence for the same triple. Callers that want dedup + // build a key that changes with the thing that makes each occurrence // distinct. Excluded from the JSON payload: it must never change // encodeRunContext's output shape, which CoalesceRun compares for // equality and taskIDFromPayload parses. @@ -248,32 +249,30 @@ func (ss *SchedulerService) SetProjectSkillDirResolver(fn func(agentTypeID strin func (ss *SchedulerService) QueueRun( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if err := ss.guardAgentStatus(ctx, agentInstanceID); err != nil { - return err + return runsservice.QueueOutcomeNone, err } if idempotencyKey != "" { dup, err := ss.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours) if err != nil { - return fmt.Errorf("idempotency check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - ss.logger.Debug("run skipped (idempotent)", - zap.String("key", idempotencyKey)) - return nil + return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil } } coalesced, err := ss.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload) if err != nil { - return fmt.Errorf("coalesce check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { ss.logger.Debug("run coalesced", zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeCoalesced, nil } var idemKeyPtr *string @@ -290,36 +289,37 @@ func (ss *SchedulerService) QueueRun( IdempotencyKey: idemKeyPtr, RequestedAt: time.Now().UTC(), } - if err := ss.repo.CreateRun(ctx, req); err != nil { - return fmt.Errorf("enqueue run: %w", err) + insertErr := ss.repo.CreateRun(ctx, req) + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr) + if err != nil { + return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err) + } + if outcome == runsservice.QueueOutcomeDeduped { + return outcome, nil } ss.logger.Info("run queued", zap.String("id", req.ID), zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeQueued, nil } -// QueueRunCtx is the typed variant of QueueRun that takes a -// structured RunContext. The context is JSON-encoded into the -// payload column so the agent runtime can deserialise it. The -// idempotency key is c.IdempotencyKey when the caller set one; -// otherwise it defaults to "{reason}:{taskID}:{agentID}" so the same -// agent never gets two runs for the same task+reason within the -// idempotency window. +// QueueRunCtx is the typed variant of QueueRun that takes a structured +// RunContext. The context is JSON-encoded into the payload column so the +// agent runtime can deserialise it. The idempotency key is c.IdempotencyKey +// verbatim — an empty key enqueues with no dedup identity rather than +// falling back to a "{reason}:{taskID}:{agentID}" default that would be +// permanently unique per (reason, task, agent) and silently swallow every +// later legitimate occurrence for the same triple. func (ss *SchedulerService) QueueRunCtx( ctx context.Context, agentInstanceID string, c RunContext, -) error { +) (runsservice.QueueOutcome, error) { payload, err := encodeRunContext(c) if err != nil { - return fmt.Errorf("encode run context: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("encode run context: %w", err) } - idempotencyKey := c.IdempotencyKey - if idempotencyKey == "" { - idempotencyKey = fmt.Sprintf("%s:%s:%s", c.Reason, c.TaskID, agentInstanceID) - } - return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, idempotencyKey) + return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, c.IdempotencyKey) } func encodeRunContext(c RunContext) (string, error) { diff --git a/apps/backend/internal/office/service/agent_working_status_test.go b/apps/backend/internal/office/service/agent_working_status_test.go index 45922722d6..de77b6e125 100644 --- a/apps/backend/internal/office/service/agent_working_status_test.go +++ b/apps/backend/internal/office/service/agent_working_status_test.go @@ -40,7 +40,7 @@ func launchedWorkingAgent( svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, project_id, title, created_at, updated_at) VALUES (?, 'ws-1', ?, 'Working status task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, taskID, project.ID) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } diff --git a/apps/backend/internal/office/service/base_test.go b/apps/backend/internal/office/service/base_test.go index cd74500580..9f10932b6f 100644 --- a/apps/backend/internal/office/service/base_test.go +++ b/apps/backend/internal/office/service/base_test.go @@ -54,6 +54,7 @@ func newTestService(t *testing.T, overrides ...service.ServiceOptions) *service. state TEXT NOT NULL DEFAULT 'TODO', title TEXT DEFAULT '', assignee_user_id TEXT NOT NULL DEFAULT '', + assignment_generation INTEGER NOT NULL DEFAULT 0, description TEXT DEFAULT '', identifier TEXT DEFAULT '', workflow_id TEXT DEFAULT '', diff --git a/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go b/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go new file mode 100644 index 0000000000..487a2281fe --- /dev/null +++ b/apps/backend/internal/office/service/blockers_resolved_key_convergence_test.go @@ -0,0 +1,192 @@ +package service + +import ( + "context" + "sync" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/runs/dedupkeys" + "github.com/kandev/kandev/internal/workflow/engine" +) + +// newBlockersConvergenceTestRepo builds a real office repository (real +// migrations, so task_blockers is the production schema) plus the minimal +// hand-rolled tasks table resolveAndWakeIfUnblocked reads state from — +// tasks is owned by the task package's schema, which office's migrations +// only reference by foreign key. +func newBlockersConvergenceTestRepo(t *testing.T) *sqlite.Repository { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store: %v", err) + } + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + ctx := context.Background() + if _, err := repo.ExecRaw(ctx, ` + CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT DEFAULT '', + state TEXT DEFAULT '' + ) + `); err != nil { + t.Fatalf("create tasks table: %v", err) + } + return repo +} + +// convergenceDispatcher records every HandleTrigger call so the test can +// assert the exact operation id resolveAndWakeIfUnblocked derived. +type convergenceDispatcher struct { + mu sync.Mutex + calls []convergenceCall +} + +type convergenceCall struct { + taskID string + trigger engine.Trigger + payload any + opID string +} + +func (d *convergenceDispatcher) HandleTrigger( + _ context.Context, taskID string, trigger engine.Trigger, payload any, opID string, +) error { + d.mu.Lock() + defer d.mu.Unlock() + d.calls = append(d.calls, convergenceCall{taskID, trigger, payload, opID}) + return nil +} + +func newConvergenceTestService(t *testing.T, repo *sqlite.Repository, dispatcher *convergenceDispatcher) *Service { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "console"}) + if err != nil { + t.Fatalf("logger: %v", err) + } + s := &Service{repo: repo, logger: log} + s.SetWorkflowEngineDispatcher(dispatcher) + return s +} + +// TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved +// is the AC-OFFICE-RUN-DEDUP-002.1 convergence test for the second blocker +// producer. office/scheduler.cascadeBlockersResolved (the other half of the +// same AC) already has a regression test +// (reactivity_blockers_resolved_test.go), but that test never exercises +// this package's resolveAndWakeIfUnblocked — it only proves +// dedupkeys.BlockerDigest is order-independent in isolation. Review round 2 +// found the AC's own explicit test requirement ("both producers derive a +// byte-identical digest, driven through the shared builder from both +// packages") unproven by any executing test. This test drives +// resolveAndWakeIfUnblocked directly and asserts its persisted operation id +// carries the exact digest office/scheduler's producer would derive for the +// identical blocker set. +func TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved(t *testing.T) { + repo := newBlockersConvergenceTestRepo(t) + ctx := context.Background() + + for _, row := range []struct{ id, state string }{ + {"blocked-1", "IN_PROGRESS"}, + {"blocker-conv-a", "COMPLETED"}, + {"blocker-conv-b", "COMPLETED"}, + } { + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + row.id, row.state, + ); err != nil { + t.Fatalf("insert task %s: %v", row.id, err) + } + } + for _, blockerID := range []string{"blocker-conv-a", "blocker-conv-b"} { + if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{ + TaskID: "blocked-1", BlockerTaskID: blockerID, + }); err != nil { + t.Fatalf("create blocker relationship for %s: %v", blockerID, err) + } + } + + dispatcher := &convergenceDispatcher{} + s := newConvergenceTestService(t, repo, dispatcher) + + if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-1", "blocker-conv-b"); err != nil { + t.Fatalf("resolveAndWakeIfUnblocked: %v", err) + } + + dispatcher.mu.Lock() + calls := append([]convergenceCall{}, dispatcher.calls...) + dispatcher.mu.Unlock() + if len(calls) != 1 { + t.Fatalf("HandleTrigger calls = %d, want 1: %#v", len(calls), calls) + } + call := calls[0] + if call.taskID != "blocked-1" { + t.Fatalf("taskID = %q, want blocked-1", call.taskID) + } + if call.trigger != engine.TriggerOnBlockerResolved { + t.Fatalf("trigger = %q, want %q", call.trigger, engine.TriggerOnBlockerResolved) + } + + // The digest office/scheduler.cascadeBlockersResolved would derive for + // the identical blocker set — proving the two independently-implemented + // producers converge on the same occurrence identity, per AC-002.1. + digest := dedupkeys.BlockerDigest([]string{"blocker-conv-a", "blocker-conv-b"}) + want := "blockers_resolved:blocked-1:" + digest + if call.opID != want { + t.Fatalf("operation id = %q, want %q", call.opID, want) + } +} + +// TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch proves the +// negative: a task with another outstanding blocker must not wake. +func TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch(t *testing.T) { + repo := newBlockersConvergenceTestRepo(t) + ctx := context.Background() + + for _, row := range []struct{ id, state string }{ + {"blocked-2", "IN_PROGRESS"}, + {"blocker-conv-c", "IN_PROGRESS"}, // never finishes + {"blocker-conv-d", "COMPLETED"}, + } { + if _, err := repo.ExecRaw(ctx, + `INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`, + row.id, row.state, + ); err != nil { + t.Fatalf("insert task %s: %v", row.id, err) + } + } + for _, blockerID := range []string{"blocker-conv-c", "blocker-conv-d"} { + if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{ + TaskID: "blocked-2", BlockerTaskID: blockerID, + }); err != nil { + t.Fatalf("create blocker relationship for %s: %v", blockerID, err) + } + } + + dispatcher := &convergenceDispatcher{} + s := newConvergenceTestService(t, repo, dispatcher) + + if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-2", "blocker-conv-d"); err != nil { + t.Fatalf("resolveAndWakeIfUnblocked: %v", err) + } + + dispatcher.mu.Lock() + defer dispatcher.mu.Unlock() + if len(dispatcher.calls) != 0 { + t.Fatalf("expected no HandleTrigger calls while blocker-conv-c is still outstanding, got %#v", dispatcher.calls) + } +} diff --git a/apps/backend/internal/office/service/budget_admission_test.go b/apps/backend/internal/office/service/budget_admission_test.go index 1a2fd5a33e..dd84b9fc02 100644 --- a/apps/backend/internal/office/service/budget_admission_test.go +++ b/apps/backend/internal/office/service/budget_admission_test.go @@ -101,7 +101,7 @@ func TestAdmitRun_NoEvaluatorWired_UnattendedCancels(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -128,7 +128,7 @@ func TestAdmitRun_NoEvaluatorWired_AttendedLaunches(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-no-evaluator-attended", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-no-evaluator-attended"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -163,7 +163,7 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T) t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-evaluator-fault", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-evaluator-fault"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -229,7 +229,7 @@ func TestAdmitRun_UnevaluatedPolicy_NamesPolicyInDeferral(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-unevaluated-policy", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-unevaluated-policy"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -279,7 +279,7 @@ func TestFinishPolicyBlock_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -308,7 +308,7 @@ func TestCancelBudgetRun_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -345,7 +345,7 @@ func TestFailRunNoEscalation_ClearsStaleAgentWorkingStatus(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -388,7 +388,7 @@ func TestProcessRun_MissingAgentReleasesPriorCheckout(t *testing.T) { INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-routed-missing', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) `) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routed-missing"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -445,7 +445,7 @@ func TestProcessRun_WorkspaceLookupErrorReleasesPriorCheckout(t *testing.T) { INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-routed-lookup-error', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) `) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routed-lookup-error"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -496,7 +496,7 @@ func TestAdmitRun_RepeatedDeferral_AttemptIncrementsPerActivity(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-attempt-increments", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-attempt-increments"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -536,7 +536,7 @@ func TestAdmitRun_DefaultCeiling_BlocksUnattendedRunWithZeroPolicies(t *testing. t.Fatalf("create agent: %v", err) } insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling", int64(600_000)) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -621,7 +621,7 @@ func TestAdmitRun_PricingDegradedBlock_UnmeasurableOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -652,7 +652,7 @@ func TestAdmitRun_DefaultCeiling_ExemptsAttendedRun(t *testing.T) { } insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling-attended", int64(600_000)) insertTestTask(t, svc, "task-default-ceiling-attended-run", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-default-ceiling-attended-run"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -711,7 +711,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -732,7 +732,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -752,7 +752,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -775,7 +775,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -794,7 +794,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -825,7 +825,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -847,7 +847,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -870,7 +870,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -891,7 +891,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -918,7 +918,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) diff --git a/apps/backend/internal/office/service/budget_metrics_test.go b/apps/backend/internal/office/service/budget_metrics_test.go index 1afa2a6262..21fd4e8e1d 100644 --- a/apps/backend/internal/office/service/budget_metrics_test.go +++ b/apps/backend/internal/office/service/budget_metrics_test.go @@ -47,7 +47,7 @@ func TestAdmitRun_MetricBlockedByLimit(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -78,7 +78,7 @@ func TestAdmitRun_MetricBlockedPricingDegraded(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -104,7 +104,7 @@ func TestAdmitRun_MetricDeferredEvaluatorFault(t *testing.T) { t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-metric-deferred-fault", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-metric-deferred-fault"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -126,7 +126,7 @@ func TestAdmitRun_MetricBlockedAbsentEvaluator(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -156,7 +156,7 @@ func TestAdmitRun_MetricDeferredWorkspaceLookup(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -192,7 +192,7 @@ func TestProcessRun_AgentRowDeleted_Cancels(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -239,7 +239,7 @@ func TestAdmitRun_MetricCancelledNoWorkspace(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -271,7 +271,7 @@ func TestAdmitRun_MetricCancelledStaleDeferral(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -302,7 +302,7 @@ func TestAdmitRun_MetricAdmittedDefault(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -335,7 +335,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaDefault(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -369,7 +369,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaStoredPolicyBypassingGate5(t * t.Fatalf("create agent: %v", err) } insertTestTask(t, svc, "task-metric-degraded-attended", "ws-1") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-metric-degraded-attended"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/channels.go b/apps/backend/internal/office/service/channels.go index c33297abcd..7f300d876c 100644 --- a/apps/backend/internal/office/service/channels.go +++ b/apps/backend/internal/office/service/channels.go @@ -54,7 +54,7 @@ func (s *Service) SetupChannel(ctx context.Context, channel *models.Channel) err // the runner row is keyed against an empty step_id; the office // repo's runner projection still resolves it because it joins // step_id = task.workflow_step_id (also empty). - if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { + if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil { _ = s.repo.DeleteChannel(ctx, channel.ID) return fmt.Errorf("set channel task assignee: %w", err) } diff --git a/apps/backend/internal/office/service/continuation_summary_reader_test.go b/apps/backend/internal/office/service/continuation_summary_reader_test.go index cd7329237c..8d28e07dae 100644 --- a/apps/backend/internal/office/service/continuation_summary_reader_test.go +++ b/apps/backend/internal/office/service/continuation_summary_reader_test.go @@ -26,7 +26,7 @@ func TestLoadContinuationSummary_AgentScope_RoundTripsThroughRealWriterAndReader createTestAgent(t, svc, "ws-1", "agent-scope-a") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "agent-scope-a", service.RunReasonTaskAssigned, "{}", "continuation-agent-scope", ); err != nil { t.Fatalf("queue run: %v", err) @@ -147,7 +147,7 @@ func TestLoadContinuationSummary_ScopeSurvivesCoalesceAfterClaim(t *testing.T) { createTestAgent(t, svc, "ws-1", "agent-scope-coalesce") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "agent-scope-coalesce", service.RunReasonTaskAssigned, "{}", "continuation-scope-coalesce", ); err != nil { t.Fatalf("queue run: %v", err) diff --git a/apps/backend/internal/office/service/event_subscribers.go b/apps/backend/internal/office/service/event_subscribers.go index 970ce12b36..6527ab92a5 100644 --- a/apps/backend/internal/office/service/event_subscribers.go +++ b/apps/backend/internal/office/service/event_subscribers.go @@ -21,6 +21,8 @@ import ( "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/shared" "github.com/kandev/kandev/internal/runs/commentkeys" + "github.com/kandev/kandev/internal/runs/dedupkeys" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/workflow/engine" ) @@ -107,12 +109,16 @@ type TaskMovedData struct { SessionID string `json:"session_id"` } -// TaskUpdatedData represents the payload of a task.updated event. +// TaskUpdatedData represents the payload of a task.created / task.updated +// event. AssignmentGeneration is carried on the extra map by +// publishTaskEventWithExtra's caller at task-creation time (never re-read); +// nil means the publishing event predates this field or is not a creation. type TaskUpdatedData struct { TaskID string `json:"task_id"` WorkspaceID string `json:"workspace_id"` AssigneeAgentProfileID string `json:"assignee_agent_profile_id"` Title string `json:"title"` + AssignmentGeneration *int64 `json:"assignment_generation"` } // CommentPostedData represents a comment event payload. @@ -994,22 +1000,39 @@ func (s *Service) handleTaskCreated(ctx context.Context, event *bus.Event) error if err != nil { return nil } - return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, true) + return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, true) } -// handleTaskUpdated fires a task_assigned run when an agent is assigned. +// handleTaskUpdated fires a task_assigned run when an agent is assigned. No +// production update path can put a new agent into the runner seat (see +// office/repository/sqlite/tasks.go's UpdateTaskAssignee vs the four inert +// syncRunnerInTx writers), so this stays subscribed as a redelivery and +// defensive path rather than a live assignment occurrence. func (s *Service) handleTaskUpdated(ctx context.Context, event *bus.Event) error { data, err := decodeEventData[TaskUpdatedData](event) if err != nil { return nil } - return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, false) + return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, false) } +// queueTaskAssignedRun fires a task_assigned run for the given occurrence. +// +// When fallbackToStoredRunner is set and the event carried no assignee, the +// agent is recovered through a fresh post-commit GetTaskExecutionFields read +// (task.created's payload can omit it) while the generation, if any, still +// describes the payload's own occurrence — not necessarily this freshly-read +// agent's. Task event publication is asynchronous (a per-task FIFO drainer), +// so a reassignment can commit between the event being built and this +// handler running, and the two halves would then name different occurrences. +// Forcing keyless whenever the fallback actually recovers an agent this way +// avoids mis-keying at the cost of a possible duplicate wake, which +// AC-003.2 already accepts. func (s *Service) queueTaskAssignedRun( ctx context.Context, taskID string, agentProfileID string, + assignmentGeneration *int64, fallbackToStoredRunner bool, ) error { if taskID == "" { @@ -1025,15 +1048,23 @@ func (s *Service) queueTaskAssignedRun( if fields == nil || !fields.IsFromOffice { return nil } + fellBackToStoredRunner := false if agentProfileID == "" && fallbackToStoredRunner { agentProfileID = fields.AssigneeAgentProfileID + fellBackToStoredRunner = agentProfileID != "" } if agentProfileID == "" { return nil } payload := mustJSON(map[string]string{"task_id": taskID}) - key := fmt.Sprintf("task_assigned:%s:%s", taskID, agentProfileID) - return s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key) + var key string + if fellBackToStoredRunner || assignmentGeneration == nil { + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "event_missing_generation") + } else { + key = dedupkeys.AssignmentKey(taskID, agentProfileID, *assignmentGeneration) + } + _, err = s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key) + return err } // handleTaskMoved keeps the legacy named-step activity fallback and queues @@ -1120,7 +1151,9 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID, if err != nil { return err } + blockerIDs := make([]string, 0, len(blockers)) for _, b := range blockers { + blockerIDs = append(blockerIDs, b.BlockerTaskID) if b.BlockerTaskID == resolvedBlockerID { continue } @@ -1129,7 +1162,10 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID, return err // still blocked } } - key := fmt.Sprintf("blockers_resolved:%s", blockedTaskID) + if len(blockerIDs) == 0 { + return nil + } + key := fmt.Sprintf("blockers_resolved:%s:%s", blockedTaskID, dedupkeys.BlockerDigest(blockerIDs)) return s.dispatchEngineTrigger(ctx, blockedTaskID, engine.TriggerOnBlockerResolved, engine.OnBlockerResolvedPayload{ ResolvedBlockerIDs: []string{resolvedBlockerID}, diff --git a/apps/backend/internal/office/service/event_subscribers_decision_test.go b/apps/backend/internal/office/service/event_subscribers_decision_test.go index d21c25d700..ecc4148237 100644 --- a/apps/backend/internal/office/service/event_subscribers_decision_test.go +++ b/apps/backend/internal/office/service/event_subscribers_decision_test.go @@ -39,7 +39,7 @@ func TestHandleAgentCompleted_WarnsWhenReviewDecisionMissing(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -91,7 +91,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyReviewStartedReason(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-1","agent_profile_id":"reviewer-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -139,7 +139,7 @@ func TestHandleAgentCompleted_WarnsOnStageIDKeyedPayload(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"approval","stage_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -179,7 +179,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyApprovalStartedReason(t *testing.T) { createTestAgent(t, svc, "ws-1", "approver-1") taskID := createOfficeTask(t, svc, "ws-1", "approver-1") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-approval","agent_profile_id":"approver-1"}` - if err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -219,7 +219,7 @@ func TestHandleAgentCompleted_WarnsOnTaskReviewRequested(t *testing.T) { svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-review-requested", "approval") payload := `{"task_id":"` + taskID + `","role":"approver"}` - if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -259,7 +259,7 @@ func TestHandleAgentCompleted_UsesAuthoritativeStageTypeWhenPayloadOmitsIt(t *te svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-authoritative-decision", "approval") payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-authoritative-decision"}` - if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -301,7 +301,7 @@ func TestHandleAgentCompleted_NoWarnWhenReviewDecisionRecorded(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1") payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil { + if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -347,7 +347,7 @@ func TestHandleAgentCompleted_NoWarnForWorkStage(t *testing.T) { taskID := createOfficeTask(t, svc, "ws-1", "builder-1") payload := `{"task_id":"` + taskID + `","stage_type":"work","workflow_step_id":"step-1"}` - if err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil { + if _, err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil { t.Fatalf("queue run: %v", err) } run, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/event_subscribers_engine_test.go b/apps/backend/internal/office/service/event_subscribers_engine_test.go index a63ce9d1cd..dae3946578 100644 --- a/apps/backend/internal/office/service/event_subscribers_engine_test.go +++ b/apps/backend/internal/office/service/event_subscribers_engine_test.go @@ -322,7 +322,7 @@ func queueTaskAssignedRunForAgentFailedTests( ) *models.Run { t.Helper() ctx := context.Background() - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, agentID, service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "", ); err != nil { t.Fatalf("queue run: %v", err) @@ -565,7 +565,7 @@ func TestEngineDispatcher_PathBEscalation_DoesNotFireAgentErrorTrigger(t *testin } createTestAgent(t, svc, "ws-1", "worker-pathb") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-pathb", service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "", ); err != nil { t.Fatalf("queue: %v", err) diff --git a/apps/backend/internal/office/service/event_subscribers_run_output_test.go b/apps/backend/internal/office/service/event_subscribers_run_output_test.go index 48b2a23180..de5bd17fb4 100644 --- a/apps/backend/internal/office/service/event_subscribers_run_output_test.go +++ b/apps/backend/internal/office/service/event_subscribers_run_output_test.go @@ -70,7 +70,7 @@ func TestHandleAgentCompleted_RecordsFinalAgentMessageAsOutputSummary(t *testing createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-init", ); err != nil { @@ -127,7 +127,7 @@ func TestHandleAgentCompleted_TruncatesOutputSummaryAt500Chars(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-truncate", ); err != nil { @@ -175,7 +175,7 @@ func TestHandleAgentCompleted_NoAgentMessageStaysBestEffort(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-none", ); err != nil { @@ -216,7 +216,7 @@ func TestHandleTasklessAgentCompleted_RecordsOutputSummaryAndKeepsContinuationSu createTestAgent(t, svc, "ws-1", "worker-taskless") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-taskless", service.RunReasonTaskAssigned, "{}", "run-output-taskless", ); err != nil { t.Fatalf("queue run: %v", err) @@ -268,7 +268,7 @@ func TestHandleAgentCompleted_LastAgentMessageWinsOverLaterUserMessage(t *testin createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-ordering", ); err != nil { @@ -318,7 +318,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) { // Run 1: queue, claim, produce an agent message, complete. The // session (sess-1) is reused for run 2 below, mirroring how office // task-bound sessions persist across turns. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-reuse-1", ); err != nil { @@ -359,7 +359,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) { // Run 2: reuses sess-1 but produces no new agent message before // completing (e.g. a turn that only made tool calls, or was cut // short). No new task_session_messages row is inserted here. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`"}`, "run-output-reuse-2", ); err != nil { @@ -390,7 +390,7 @@ func TestHandleAgentCompleted_DoesNotClearExistingSummaryWithoutMessage(t *testi ctx := context.Background() createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-preserve"); err != nil { t.Fatalf("queue run: %v", err) } @@ -423,7 +423,7 @@ func TestHandleAgentCompleted_UsesTurnScopedMessageForSharedSession(t *testing.T ctx := context.Background() createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "run-output-turn"); err != nil { t.Fatalf("queue run: %v", err) } diff --git a/apps/backend/internal/office/service/event_subscribers_test.go b/apps/backend/internal/office/service/event_subscribers_test.go index 43e1b3da5f..fa8980135b 100644 --- a/apps/backend/internal/office/service/event_subscribers_test.go +++ b/apps/backend/internal/office/service/event_subscribers_test.go @@ -18,6 +18,7 @@ import ( "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/service" "github.com/kandev/kandev/internal/office/shared" + "github.com/kandev/kandev/internal/runs/dedupkeys" "github.com/kandev/kandev/internal/workflow/engine" ) @@ -42,7 +43,8 @@ func (d *queueRunDispatcher) HandleTrigger( case engine.TriggerOnComment: p, _ := payload.(engine.OnCommentPayload) body, _ := json.Marshal(map[string]string{"task_id": taskID, "comment_id": p.CommentID}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID) + return err case engine.TriggerOnBlockerResolved: p, _ := payload.(engine.OnBlockerResolvedPayload) var resolved string @@ -50,10 +52,12 @@ func (d *queueRunDispatcher) HandleTrigger( resolved = p.ResolvedBlockerIDs[0] } body, _ := json.Marshal(map[string]string{"task_id": taskID, "resolved_blocker_id": resolved}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID) + return err case engine.TriggerOnChildrenCompleted: body, _ := json.Marshal(map[string]string{"task_id": taskID}) - return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID) + return err case engine.TriggerOnApprovalResolved: p, _ := payload.(engine.OnApprovalResolvedPayload) body, _ := json.Marshal(map[string]string{ @@ -61,7 +65,8 @@ func (d *queueRunDispatcher) HandleTrigger( "status": p.Status, "decision_note": p.Note, }) - return d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID) + _, err := d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID) + return err } return nil } @@ -289,6 +294,64 @@ func TestTaskCreated_WakesAssigneeFromStoredRunner(t *testing.T) { t.Fatalf("expected task_assigned run for worker-created, got %#v", runs) } +func TestTaskCreated_WithAssignmentIdentity_UsesGenerationKey(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-created-keyed") + insertTestTask(t, svc, "task-created-keyed", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project', assignment_generation = 1 WHERE id = ?`, "task-created-keyed") + setTestTaskAssignee(t, svc, "task-created-keyed", "worker-created-keyed") + + event := bus.NewEvent(events.TaskCreated, "task-service", map[string]any{ + "task_id": "task-created-keyed", + "assignee_agent_profile_id": "worker-created-keyed", + "assignment_generation": int64(1), + }) + if err := eb.Publish(ctx, events.TaskCreated, event); err != nil { + t.Fatalf("publish task created event: %v", err) + } + + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list creation runs: %v", err) + } + wantKey := dedupkeys.AssignmentKey("task-created-keyed", "worker-created-keyed", 1) + var creationRunCount int + for _, run := range runs { + if run.AgentProfileID != "worker-created-keyed" || run.Reason != service.RunReasonTaskAssigned { + continue + } + creationRunCount++ + if run.IdempotencyKey == nil || *run.IdempotencyKey != wantKey { + t.Fatalf("creation run key = %v, want %q", run.IdempotencyKey, wantKey) + } + } + if creationRunCount != 1 { + t.Fatalf("creation event queued %d runs, want 1", creationRunCount) + } + + // A replay after the first run is claimed must still use the same durable + // key and avoid inserting a second run. + svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE reason = ? AND idempotency_key = ?`, service.RunReasonTaskAssigned, wantKey) + if err := eb.Publish(ctx, events.TaskCreated, event); err != nil { + t.Fatalf("replay task created event: %v", err) + } + allRuns, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list replay runs: %v", err) + } + var totalTaskAssigned int + for _, run := range allRuns { + if run.AgentProfileID == "worker-created-keyed" && run.Reason == service.RunReasonTaskAssigned { + totalTaskAssigned++ + } + } + if totalTaskAssigned != 1 { + t.Fatalf("replayed creation event queued %d runs, want 1", totalTaskAssigned) + } +} + func TestTaskCreated_NoopWhenNoStoredRunner(t *testing.T) { svc, eb := newTestServiceWithBus(t) ctx := context.Background() diff --git a/apps/backend/internal/office/service/failure.go b/apps/backend/internal/office/service/failure.go index cd3e7cec11..9b3469d805 100644 --- a/apps/backend/internal/office/service/failure.go +++ b/apps/backend/internal/office/service/failure.go @@ -488,6 +488,9 @@ func (s *Service) autoPauseAgent( return nil } +// requeueRunForTask is a manual resume: the failed run's own id is the +// occurrence identity (falling back to the task id if it is somehow empty), +// so a duplicate "Mark fixed" click dedupes instead of double-queuing. func (s *Service) requeueRunForTask( ctx context.Context, agentID, taskID, failedRunID string, ) error { @@ -497,7 +500,8 @@ func (s *Service) requeueRunForTask( identity = taskID } key := fmt.Sprintf("%s:%s:%s", RunReasonManualResumeAfterFailure, agentID, identity) - return s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key) + _, err := s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key) + return err } func (s *Service) publishRunFailed( diff --git a/apps/backend/internal/office/service/failure_test.go b/apps/backend/internal/office/service/failure_test.go index 8d4077ccd4..fbf5c1cc94 100644 --- a/apps/backend/internal/office/service/failure_test.go +++ b/apps/backend/internal/office/service/failure_test.go @@ -26,7 +26,7 @@ func queueAndReadRun( ctx := context.Background() payload := mustMarshalJSON(map[string]string{"task_id": taskID}) idem := agentID + ":" + taskID - if err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil { + if _, err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil { t.Fatalf("queue run: %v", err) } rows, err := svc.ListRuns(ctx, "ws-1") @@ -255,7 +255,7 @@ func TestMarkAgentPausedFixed_RecoversAgent(t *testing.T) { // Proves the guardAgentStatus rejection is gone: QueueRun must // succeed now that the agent is actually idle. - if err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned, mustMarshalJSON(map[string]string{"task_id": "agent-recover-task-a"}), "agent-recover:post-fix"); err != nil { t.Fatalf("expected QueueRun to succeed after mark fixed, got: %v", err) diff --git a/apps/backend/internal/office/service/retry.go b/apps/backend/internal/office/service/retry.go index 7838c8c41c..2a588dca5a 100644 --- a/apps/backend/internal/office/service/retry.go +++ b/apps/backend/internal/office/service/retry.go @@ -227,7 +227,12 @@ func (s *Service) queueCEOAgentError( "run_id": run.ID, "error": errMsg, }) - _ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, "") + // The failed run's own id makes this occurrence identity: one escalation + // per failed run per CEO, but a later run by the same agent that also + // fails escalates again instead of being silently swallowed by a + // permanently-unique-per-pair key. + key := fmt.Sprintf("agent_error:%s:%s", run.ID, ceos[0].ID) + _, _ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, key) } // retryDelayWithJitter returns the base delay for a given retry index diff --git a/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go b/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go index ccbed8bfd4..a030d0f4e8 100644 --- a/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go +++ b/apps/backend/internal/office/service/retry_ceo_self_escalation_test.go @@ -36,7 +36,7 @@ func TestHandleRunFailure_CEOOwnFailure_DoesNotSelfEscalate(t *testing.T) { } for i := 0; i < 3; i++ { - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, ceo.ID, service.RunReasonAgentError, `{}`, "", ); err != nil { t.Fatalf("queue ceo run %d: %v", i, err) diff --git a/apps/backend/internal/office/service/retry_ratelimit_test.go b/apps/backend/internal/office/service/retry_ratelimit_test.go index 43b0d7c6d7..eba2e93583 100644 --- a/apps/backend/internal/office/service/retry_ratelimit_test.go +++ b/apps/backend/internal/office/service/retry_ratelimit_test.go @@ -226,7 +226,7 @@ func TestHandleRunFailure_RateLimitParsed_UsesResetTime(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -275,7 +275,7 @@ func TestHandleRunFailure_RateLimitNoParseable_UsesBackoff(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) @@ -325,7 +325,7 @@ func TestHandleRunFailure_NonRateLimit_UsesBackoff(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil { t.Fatalf("queue: %v", err) } run, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/run.go b/apps/backend/internal/office/service/run.go index 92808c1797..df76afdd23 100644 --- a/apps/backend/internal/office/service/run.go +++ b/apps/backend/internal/office/service/run.go @@ -85,18 +85,17 @@ const IdempotencyWindowHours = 24 func (s *Service) QueueRun( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if err := s.guardAgentStatus(ctx, agentInstanceID); err != nil { - return err + return runsservice.QueueOutcomeNone, err } if s.runsService != nil { - _, err := s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{ + return s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{ Reason: reason, IdempotencyKey: idempotencyKey, Payload: payloadWithAgent(payload, agentInstanceID), }) - return err } return s.queueRunInline(ctx, agentInstanceID, reason, payload, idempotencyKey) } @@ -107,28 +106,26 @@ func (s *Service) QueueRun( func (s *Service) queueRunInline( ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string, -) error { +) (runsservice.QueueOutcome, error) { if idempotencyKey != "" { dup, err := s.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours) if err != nil { - return fmt.Errorf("idempotency check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - s.logger.Debug("run skipped (idempotent)", - zap.String("key", idempotencyKey)) - return nil + return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil } } coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload) if err != nil { - return fmt.Errorf("coalesce check: %w", err) + return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { s.logger.Debug("run coalesced", zap.String("agent", agentInstanceID), zap.String("reason", reason)) - return nil + return runsservice.QueueOutcomeCoalesced, nil } var idemKeyPtr *string @@ -145,8 +142,13 @@ func (s *Service) queueRunInline( IdempotencyKey: idemKeyPtr, RequestedAt: time.Now().UTC(), } - if err := s.repo.CreateRun(ctx, req); err != nil { - return fmt.Errorf("enqueue run: %w", err) + insertErr := s.repo.CreateRun(ctx, req) + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr) + if err != nil { + return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err) + } + if outcome == runsservice.QueueOutcomeDeduped { + return outcome, nil } s.logger.Info("run queued", @@ -155,7 +157,7 @@ func (s *Service) queueRunInline( zap.String("reason", reason)) s.publishRunQueued(ctx, req, idempotencyKey) - return nil + return runsservice.QueueOutcomeQueued, nil } // payloadWithAgent decodes the JSON payload string and adds the diff --git a/apps/backend/internal/office/service/run_lifecycle_events_test.go b/apps/backend/internal/office/service/run_lifecycle_events_test.go index 61ec97f307..209509323c 100644 --- a/apps/backend/internal/office/service/run_lifecycle_events_test.go +++ b/apps/backend/internal/office/service/run_lifecycle_events_test.go @@ -30,7 +30,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) { // Queue + claim a run for the task so handleAgentTurnMessageSaved // resolves a runID via GetClaimedRunByTaskID. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "lifecycle-init", ); err != nil { @@ -102,7 +102,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) { // no longer find a claimed run. To exercise the persistence path // directly, we re-queue + claim once more and seed an activity // row tagged with the run id. - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`"}`, "lifecycle-second", ); err != nil { @@ -131,7 +131,7 @@ func TestRunLifecycle_ErrorEventEmitted(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "lifecycle-error", ); err != nil { @@ -249,7 +249,7 @@ func TestQueueRun_PublishesOfficeRunQueued(t *testing.T) { const idemKey = "task_comment:cm-123" payload := `{"task_id":"` + taskID + `","comment_id":"cm-123"}` - if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil { + if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil { t.Fatalf("queue run: %v", err) } @@ -293,7 +293,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`","comment_id":"cm-1"}`, "task_comment:cm-1", ); err != nil { @@ -353,7 +353,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T) sourceTaskID := "source-task" insertTestTask(t, svc, sourceTaskID, "ws-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+targetTaskID+`","source_task_id":"`+sourceTaskID+`","comment_id":"cm-source"}`, "task_comment:cm-source:target", @@ -406,7 +406,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-1") taskID := createOfficeTask(t, svc, "ws-1", "worker-1") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-1", service.RunReasonTaskComment, `{"task_id":"`+taskID+`","comment_id":"cm-2"}`, "task_comment:cm-2", ); err != nil { diff --git a/apps/backend/internal/office/service/run_test.go b/apps/backend/internal/office/service/run_test.go index 16da1241bb..e0e2257d93 100644 --- a/apps/backend/internal/office/service/run_test.go +++ b/apps/backend/internal/office/service/run_test.go @@ -2,12 +2,39 @@ package service_test import ( "context" + "expvar" "testing" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/service" + runsservice "github.com/kandev/kandev/internal/runs/service" ) +// windowedDedupCounterValue reads the current value of one +// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the +// label has never been reported. Callers snapshot before/after and assert +// the exact delta. +func windowedDedupCounterValue(t *testing.T, reason string) int64 { + t.Helper() + v := expvar.Get("office_run_dedup_total") + if v == nil { + t.Fatalf("expvar map office_run_dedup_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("office_run_dedup_total is not a *expvar.Map") + } + iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs") + if iv == nil { + return 0 + } + i, ok := iv.(*expvar.Int) + if !ok { + t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason) + } + return i.Value() +} + func TestQueueRun_Basic(t *testing.T) { svc := newTestService(t) ctx := context.Background() @@ -17,7 +44,7 @@ func TestQueueRun_Basic(t *testing.T) { t.Fatalf("create agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1") if err != nil { t.Fatalf("queue run: %v", err) } @@ -47,11 +74,11 @@ func TestQueueRun_Idempotency(t *testing.T) { } key := "idem-key-1" - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { t.Fatalf("first enqueue: %v", err) } // Second enqueue with same key should be silently dropped. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil { t.Fatalf("second enqueue: %v", err) } @@ -61,6 +88,46 @@ func TestQueueRun_Idempotency(t *testing.T) { } } +// TestQueueRun_Idempotency_ReportsWindowedDedupOutcome closes a phantom-green +// gap: TestQueueRun_Idempotency above only asserted the resulting row count, +// so deleting queueRunInline's ReportWindowedDedup call would leave it green +// while AC-OFFICE-RUN-DEDUP-004.1's observability contract silently went +// unmet on this queue implementation. Asserts the outcome value and the +// office_run_dedup_total counter directly. +func TestQueueRun_Idempotency_ReportsWindowedDedupOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-1", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + key := "idem-key-outcome" + first, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("first enqueue: %v", err) + } + if first != runsservice.QueueOutcomeQueued { + t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first) + } + + before := windowedDedupCounterValue(t, service.RunReasonTaskAssigned) + + second, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key) + if err != nil { + t.Fatalf("second enqueue: %v", err) + } + if second != runsservice.QueueOutcomeDeduped { + t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second) + } + + after := windowedDedupCounterValue(t, service.RunReasonTaskAssigned) + if after != before+1 { + t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", service.RunReasonTaskAssigned, after, before+1) + } +} + func TestQueueRun_SkipsPausedAgent(t *testing.T) { svc := newTestService(t) ctx := context.Background() @@ -74,7 +141,7 @@ func TestQueueRun_SkipsPausedAgent(t *testing.T) { t.Fatalf("pause agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") if err == nil { t.Fatal("expected error for paused agent") } @@ -97,7 +164,7 @@ func TestQueueRun_SkipsStoppedAgent(t *testing.T) { t.Fatalf("stop agent: %v", err) } - err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") + _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "") if err == nil { t.Fatal("expected error for stopped agent") } @@ -113,10 +180,10 @@ func TestQueueRun_Coalesce(t *testing.T) { } // Two runs with the same agent + reason within coalesce window should merge. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("first: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("second: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_contention_test.go b/apps/backend/internal/office/service/scheduler_checkout_contention_test.go index 488120052a..58a4b19761 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_contention_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_contention_test.go @@ -42,7 +42,7 @@ func TestSchedulerTick_ContendedCheckoutRequeuesRun(t *testing.T) { t.Fatalf("seed checkout: ok=%v err=%v", ok, err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-contended-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_error_test.go b/apps/backend/internal/office/service/scheduler_checkout_error_test.go index 8fd3cc590a..1cd259ec9e 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_error_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_error_test.go @@ -35,7 +35,7 @@ func TestSchedulerTick_CheckoutErrorRetriesInsteadOfFalseFinish(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-db-error-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go b/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go index 65259bc927..973a5b1084 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_inactive_agent_test.go @@ -55,7 +55,7 @@ func TestSchedulerTick_InactiveAgentRunDoesNotStealCheckout(t *testing.T) { // Queue while idle (QueueRun itself refuses to queue for a paused // agent), then go paused before the tick runs - reproducing the race // where an agent is deactivated between queueing and processing. - if err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned, `{"task_id":"task-inactive-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -125,7 +125,7 @@ func TestSchedulerTick_SameAgentPreCheckoutRunDoesNotStealOwnLiveCheckout(t *tes // queued while idle, then the agent goes paused before the tick picks // it up - reproducing the race where the holder is deactivated after // a second run was already queued behind its own live run. - if err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned, `{"task_id":"task-same-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_checkout_release_test.go b/apps/backend/internal/office/service/scheduler_checkout_release_test.go index f463d17607..2c6bd9ee29 100644 --- a/apps/backend/internal/office/service/scheduler_checkout_release_test.go +++ b/apps/backend/internal/office/service/scheduler_checkout_release_test.go @@ -63,7 +63,7 @@ func TestSchedulerTick_AgentCompletedReleasesTaskCheckout(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-checkout-release-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-release-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -151,7 +151,7 @@ func TestSchedulerTick_AgentFailedReleasesTaskCheckout(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-checkout-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-fail-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -228,7 +228,7 @@ func TestSchedulerTick_TasklessAgentCompletedStampsRuntime(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } claimed, err := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/scheduler_features_test.go b/apps/backend/internal/office/service/scheduler_features_test.go index c996f22905..e7fa5fbbfe 100644 --- a/apps/backend/internal/office/service/scheduler_features_test.go +++ b/apps/backend/internal/office/service/scheduler_features_test.go @@ -29,7 +29,7 @@ func TestCooldown_RecentFinish_GuardAllows(t *testing.T) { } // Queue a run. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -69,7 +69,7 @@ func TestCooldown_PastCooldown_ClaimedNormally(t *testing.T) { } // Queue a run. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -97,7 +97,7 @@ func TestRetry_FailedRun_RetriedWithBackoff(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -156,7 +156,7 @@ func TestRetry_FifthFailure_MarkedFailed(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -402,7 +402,7 @@ func TestIdleSkip_HeartbeatNoTasks_Skipped(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -457,7 +457,7 @@ func TestIdleSkip_HeartbeatWithActionableTasks_Proceeds(t *testing.T) { // Assign an IN_PROGRESS task to this agent. insertActionableTask(t, svc, "task-inprog-1", agent.ID, "IN_PROGRESS") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -499,7 +499,7 @@ func TestIdleSkip_HeartbeatSkipDisabled_Proceeds(t *testing.T) { `UPDATE agent_profiles SET skip_idle_runs = 0 WHERE id = ?`, agent.ID) // No tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -535,7 +535,7 @@ func TestIdleSkip_NonHeartbeatRun_NotSkipped(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-event-1', 'ws-1', 'Event Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -569,7 +569,7 @@ func TestIdleSkip_CEODefaultFalse_NotSkipped(t *testing.T) { } // CEO has no tasks, but skip_idle_runs defaults to false. - if err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration_routing_test.go b/apps/backend/internal/office/service/scheduler_integration_routing_test.go index cbdd70ad6e..9410984213 100644 --- a/apps/backend/internal/office/service/scheduler_integration_routing_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_routing_test.go @@ -109,7 +109,7 @@ func TestSchedulerIntegration_RoutingReceivesBuiltPromptAndEnv(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at) VALUES ('task-routing-1', 'ws-1', 'ROUTING_PROMPT_SENTINEL_TITLE', 'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routing-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -171,7 +171,7 @@ func TestSchedulerIntegration_SeatActionFlowsToPromptAndLaunch(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, "task-decision", "ws-1", "step-decision", "Decision task", "Review the change") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-decision"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -263,7 +263,7 @@ func TestSchedulerIntegration_NonSeatRunDoesNotReceiveDecisionSkill(t *testing.T svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, "task-non-seat", "ws-1", "step-non-seat", "Non-seat task", "Review the change") - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-non-seat"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -317,7 +317,7 @@ func TestSchedulerIntegration_RoutingFallThrough_FallsBackToLegacy(t *testing.T) } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-ft-1', 'ws-1', 'Fall-through Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-ft-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -361,7 +361,7 @@ func TestSchedulerIntegration_RoutingParked_LeavesAgentIdle(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-parked-1', 'ws-1', 'Parked Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-parked-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -402,7 +402,7 @@ func TestSchedulerIntegration_RoutingDispatchError_LeavesAgentIdle(t *testing.T) } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-routing-err-1', 'ws-1', 'Routing Error Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-routing-err-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration_test.go b/apps/backend/internal/office/service/scheduler_integration_test.go index 7e012f39d5..8f27d41af6 100644 --- a/apps/backend/internal/office/service/scheduler_integration_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_test.go @@ -20,7 +20,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -72,7 +72,7 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) { // The run was queued while the task was on step-old. A later workflow // move must prevent the scheduler from launching that stale step. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-moved-step","workflow_step_id":"step-old"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -150,7 +150,7 @@ func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) { VALUES ('task-project-exec', 'ws-1', ?, 'Project executor task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, project.ID) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-project-exec"}`, ""); err != nil { t.Fatalf("queue run: %v", err) } @@ -205,7 +205,7 @@ func TestSchedulerIntegration_PausedAgentSkipped(t *testing.T) { } // Queue while agent is active. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -244,10 +244,10 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) { } // Queue two runs. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil { t.Fatalf("queue first: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil { t.Fatalf("queue second: %v", err) } @@ -330,7 +330,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_recovery.go b/apps/backend/internal/office/service/scheduler_recovery.go index 3f8a73b24c..e464f567de 100644 --- a/apps/backend/internal/office/service/scheduler_recovery.go +++ b/apps/backend/internal/office/service/scheduler_recovery.go @@ -7,6 +7,7 @@ import ( "go.uber.org/zap" "github.com/kandev/kandev/internal/common/logger" + runsservice "github.com/kandev/kandev/internal/runs/service" ) // maxRecoveryPerTick caps the number of unstarted tasks recovered in one tick. @@ -69,7 +70,8 @@ func (si *SchedulerIntegration) recoverUnstartedTasks(ctx context.Context, log * zap.String("agent_profile_id", t.AssigneeAgentProfileID)) payload := mustJSON(map[string]string{"task_id": t.ID}) - if err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID, + runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseByDesign, "") + if _, err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID, RunReasonTaskAssigned, payload, ""); err != nil { if ctx.Err() != nil { return diff --git a/apps/backend/internal/office/service/scheduler_run_outcome_test.go b/apps/backend/internal/office/service/scheduler_run_outcome_test.go index 21d4a43059..944407f807 100644 --- a/apps/backend/internal/office/service/scheduler_run_outcome_test.go +++ b/apps/backend/internal/office/service/scheduler_run_outcome_test.go @@ -64,7 +64,7 @@ func TestSchedulerOutcome_AgentInactive_WritesOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -106,7 +106,7 @@ func TestSchedulerOutcome_IdleSkipped_WritesOutcome(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -134,7 +134,7 @@ func TestSchedulerOutcome_TaskTreeHeld_WritesOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-tree-outcome"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -184,7 +184,7 @@ func TestSchedulerOutcome_CheckoutError_RetriesInsteadOfFinishing(t *testing.T) if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-error"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -250,7 +250,7 @@ func TestSchedulerOutcome_CheckoutUnavailable_RetriesInsteadOfFinishing(t *testi if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-checkout-unavail"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -286,7 +286,7 @@ func TestSchedulerOutcome_BudgetBlocked_WritesOutcome(t *testing.T) { insertTestTask(t, svc, "task-budget-outcome", "ws-1") insertTestCostEvent(t, svc, agent.ID, "task-budget-outcome", int64(600)) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-budget-outcome"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -312,7 +312,7 @@ func TestSchedulerOutcome_NoTaskStarter_FailsRun(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-no-launch"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -353,7 +353,7 @@ func TestSchedulerOutcome_Processed_WritesOutcome(t *testing.T) { createTestAgent(t, svc, "ws-1", "worker-processed") taskID := createOfficeTask(t, svc, "ws-1", "worker-processed") - if err := svc.QueueRun( + if _, err := svc.QueueRun( ctx, "worker-processed", service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "processed-outcome", ); err != nil { diff --git a/apps/backend/internal/office/service/scheduler_runs_test.go b/apps/backend/internal/office/service/scheduler_runs_test.go index 20e7b883b6..4078bd08d5 100644 --- a/apps/backend/internal/office/service/scheduler_runs_test.go +++ b/apps/backend/internal/office/service/scheduler_runs_test.go @@ -30,7 +30,7 @@ func TestClaimNextRun_ClaimsQueued(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { t.Fatalf("queue: %v", err) } @@ -96,7 +96,7 @@ func TestFinishAndFailRun(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } @@ -132,7 +132,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } @@ -178,7 +178,7 @@ func TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome(t *testing.T) { if err := svc.CreateAgentInstance(ctx, agent); err != nil { t.Fatalf("create agent: %v", err) } - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { t.Fatalf("queue: %v", err) } req, _ := svc.ClaimNextRun(ctx) diff --git a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go index a921777287..49b54f6c3d 100644 --- a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go +++ b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go @@ -45,7 +45,7 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) { // Mirrors wakeup/dispatcher.go's createFreshRun: reason from the // routine trigger, payload literally "{}" (no task_id). - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -196,7 +196,7 @@ func TestSchedulerTick_TaskBoundRunStillLaunches(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-wo35-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -256,7 +256,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) { // mirrors 3 ticks of the pre-installed coordinator heartbeat routine. const firesAtThreshold = 3 for i := 0; i < firesAtThreshold; i++ { - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue taskless run %d: %v", i, err) } service.RunSchedulerTick(svc, ctx) @@ -281,7 +281,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) { // failures: a task-bound run queued afterwards must still launch. svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-wo35-pause-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil { t.Fatalf("queue task-bound run: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -330,7 +330,7 @@ func TestSchedulerTick_TasklessRunFailure_PublishesResolvableWorkspaceEvent(t *t } t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } service.RunSchedulerTick(svc, ctx) @@ -391,7 +391,7 @@ func TestSchedulerTick_UnlaunchableRun_PublishesResolvableWorkspaceEvent(t *test } t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-unlaunchable"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -446,7 +446,7 @@ func TestSchedulerTick_RepeatTasklessFailures_OnlyFirstStaysInInbox(t *testing.T const fires = 3 var firstRunID string for i := 0; i < fires; i++ { - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { t.Fatalf("queue taskless run %d: %v", i, err) } service.RunSchedulerTick(svc, ctx) @@ -512,7 +512,7 @@ func TestSchedulerTick_RepeatTasklessFailures_StayVisiblePerRoutineScope(t *test queueRoutineFailure := func(scope, idempotencyKey string) string { t.Helper() - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil { t.Fatalf("queue routine %s: %v", scope, err) } runs, err := svc.ListRuns(ctx, "ws-1") diff --git a/apps/backend/internal/office/service/task_assigned_generation_key_test.go b/apps/backend/internal/office/service/task_assigned_generation_key_test.go new file mode 100644 index 0000000000..13e0ac6c43 --- /dev/null +++ b/apps/backend/internal/office/service/task_assigned_generation_key_test.go @@ -0,0 +1,180 @@ +package service_test + +// This file proves the specific defect run-dedup-generation fixes: a +// task_assigned dedup key that used to be stable for the life of a +// (task, agent) pair (docs/specs/office/requirements/run-dedup-generation.md, +// AC-OFFICE-SCHEDULER-001.7). idx_run_idempotency has no time bound, so a +// stable key collided with the UNIQUE index forever, making a re-assignment +// of the same task to the same agent a silent, permanent no-op. These tests +// exercise the real bus -> handleTaskUpdated -> queueTaskAssignedRun path +// (not just the dedupkeys.AssignmentKey unit) and assert on the persisted +// idempotency_key, which none of the existing task_assigned tests do. + +import ( + "context" + "encoding/json" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/service" + "github.com/kandev/kandev/internal/runs/dedupkeys" +) + +func publishTaskAssigned( + t *testing.T, ctx context.Context, eb bus.EventBus, taskID, agentID string, generation *int64, +) { + t.Helper() + data := map[string]any{ + "task_id": taskID, + "assignee_agent_profile_id": agentID, + } + if generation != nil { + data["assignment_generation"] = *generation + } + event := bus.NewEvent(events.TaskUpdated, "test", data) + if err := eb.Publish(ctx, events.TaskUpdated, event); err != nil { + t.Fatalf("publish task updated event: %v", err) + } +} + +func gen(n int64) *int64 { return &n } + +// taskAssignedRunsFor returns every task_assigned run queued for agentID on +// the given task, in insertion order (ListRuns has no ordering guarantee +// this test relies on beyond "all rows present"). +func taskAssignedRunsFor(t *testing.T, svc *service.Service, wsID, taskID, agentID string) []string { + t.Helper() + runs, err := svc.ListRuns(context.Background(), wsID) + if err != nil { + t.Fatalf("list runs: %v", err) + } + var keys []string + for _, run := range runs { + if run.AgentProfileID != agentID || run.Reason != service.RunReasonTaskAssigned { + continue + } + var payload struct { + TaskID string `json:"task_id"` + } + if err := json.Unmarshal([]byte(run.Payload), &payload); err != nil { + t.Fatalf("decode run payload: %v", err) + } + if payload.TaskID != taskID { + continue + } + if run.IdempotencyKey == nil { + keys = append(keys, "") + continue + } + keys = append(keys, *run.IdempotencyKey) + } + return keys +} + +func TestTaskAssigned_SameGenerationRedelivery_DedupesForever(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-gen") + insertTestTask(t, svc, "task-gen-redeliver", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-gen-redeliver") + + publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0)) + publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0)) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-gen-redeliver", "worker-gen") + if len(keys) != 1 { + t.Fatalf("redelivering the same generation queued %d runs, want exactly 1 (deduped): keys=%#v", len(keys), keys) + } + want := dedupkeys.AssignmentKey("task-gen-redeliver", "worker-gen", 0) + if keys[0] != want { + t.Fatalf("idempotency key = %q, want %q", keys[0], want) + } +} + +// TestTaskAssigned_GenerationBump_AvoidsPermanentCollision is the direct +// regression test for the reported defect: reassigning the same task to the +// same agent must NOT be a permanent no-op once the durable unique index +// already holds a row for an earlier generation's key. Unlike +// TestTaskAssigned_ReassignmentUsesAgentScopedIdempotency (which reassigns to +// a *different* agent and never sets assignment_generation, so it never +// exercised the keyed path at all), this drives the same agent through two +// generations to prove the key itself changed. +func TestTaskAssigned_GenerationBump_AvoidsPermanentCollision(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-regen") + insertTestTask(t, svc, "task-regen", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-regen") + + // Occurrence 1: original assignment at generation 0. In production this + // row's idempotency_key (task_assigned:task-regen:worker-regen:0) is + // exactly the durable UNIQUE row that used to collide forever under the + // pre-fix key format (task_assigned::, no generation). + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(0)) + + // The first run is claimed and finishes (agent did the work) long before + // the reassignment — CoalesceRun only merges into a still-'queued' row, + // so this ensures the second occurrence exercises the durable + // idx_run_idempotency INSERT path, not the 5-second coalesce window. + // This is exactly the "more than 24 hours later" gap in the bug report. + svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`) + + // Simulate a real reassignment: UpdateTaskAssignee bumps the stored + // generation before the task.updated event carrying it is published. + svc.ExecSQL(t, `UPDATE tasks SET assignment_generation = 1 WHERE id = ?`, "task-regen") + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1)) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen") + if len(keys) != 2 { + t.Fatalf("reassigning the same agent at a new generation queued %d runs, want 2 (no false collision): keys=%#v", len(keys), keys) + } + wantGen0 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 0) + wantGen1 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 1) + matchesInOrder := keys[0] == wantGen0 && keys[1] == wantGen1 + matchesReversed := keys[0] == wantGen1 && keys[1] == wantGen0 + if !matchesInOrder && !matchesReversed { + t.Fatalf("keys = %#v, want %q and %q (one run per generation)", keys, wantGen0, wantGen1) + } + if keys[0] == keys[1] { + t.Fatalf("both runs share idempotency key %q — this is the reported defect: a reassignment collided with the durable unique index", keys[0]) + } + + // Redelivering generation 1 again — past the coalesce window and after + // the generation-1 run has itself been claimed — must still dedupe via + // the durable unique index: the fix must not turn every event into a + // fresh key, only a genuine generation change. + svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`) + publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1)) + keysAfterRedeliver := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen") + if len(keysAfterRedeliver) != 2 { + t.Fatalf("redelivering generation 1 queued a 3rd run: keys=%#v", keysAfterRedeliver) + } +} + +// TestTaskAssigned_MissingGeneration_EnqueuesKeyless proves SR-55's +// resolution: an event carrying no assignment_generation must never mint a +// key with a guessed or zero generation — it goes keyless (empty +// idempotency_key) so a possible duplicate wake is accepted (AC-003.2) +// instead of a mis-keyed row that could collide with an unrelated +// occurrence. +func TestTaskAssigned_MissingGeneration_EnqueuesKeyless(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-nogen") + insertTestTask(t, svc, "task-nogen", "ws-1") + svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-nogen") + + publishTaskAssigned(t, ctx, eb, "task-nogen", "worker-nogen", nil) + + keys := taskAssignedRunsFor(t, svc, "ws-1", "task-nogen", "worker-nogen") + if len(keys) != 1 { + t.Fatalf("got %d runs, want 1: keys=%#v", len(keys), keys) + } + if keys[0] != "" { + t.Fatalf("idempotency key = %q, want empty (keyless enqueue)", keys[0]) + } +} diff --git a/apps/backend/internal/office/service/task_assignee.go b/apps/backend/internal/office/service/task_assignee.go index 9c5e7a196f..94ef5d6a0b 100644 --- a/apps/backend/internal/office/service/task_assignee.go +++ b/apps/backend/internal/office/service/task_assignee.go @@ -38,7 +38,10 @@ const participantTypeAgent = "agent" // by tests and a couple of internal callers; the dashboard's // permissioned variant is SetTaskAssigneeAsAgent. func (s *Service) SetTaskAssignee(ctx context.Context, taskID, assigneeID string) error { - return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + // generation discarded — this caller does not carry it to the event bus; + // queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003). + _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + return err } // SetTaskAssigneeAsAgent checks can_assign_tasks for the given caller @@ -55,5 +58,8 @@ func (s *Service) SetTaskAssigneeAsAgent(ctx context.Context, callerAgentID, tas return shared.ErrForbidden } } - return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + // generation discarded — this caller does not carry it to the event bus; + // queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003). + _, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID) + return err } diff --git a/apps/backend/internal/office/service/task_starter_test.go b/apps/backend/internal/office/service/task_starter_test.go index 7b11a589b9..37cd086d79 100644 --- a/apps/backend/internal/office/service/task_starter_test.go +++ b/apps/backend/internal/office/service/task_starter_test.go @@ -99,7 +99,7 @@ func TestSchedulerTick_LaunchesAgent(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at) VALUES ('task-launch-1', 'ws-1', 'Build API', 'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -145,7 +145,7 @@ func TestSchedulerTick_LaunchIncludesRuntimeTokenEnv(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-token-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -192,7 +192,7 @@ func TestSchedulerTick_SnapshotsRunSkills(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-skill-1', 'ws-1', 'Review API', 'Review endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -239,7 +239,7 @@ func TestSchedulerTick_KeepsRunClaimedUntilAgentCompletes(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-life-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -301,7 +301,7 @@ func TestSchedulerTick_AgentStoppedFinishesRun(t *testing.T) { } svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) VALUES ('task-stop-1', 'ws-1', 'Stop handler test', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -386,7 +386,7 @@ func TestSchedulerTick_StartTaskError_TriggersRetry(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-fail-1', 'ws-1', 'Failing Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -439,7 +439,7 @@ func TestSchedulerTick_NoTaskStarter_FailsRunLoudly(t *testing.T) { svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('task-noop-1', 'ws-1', 'NoOp Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -494,7 +494,7 @@ func TestSchedulerTick_NoTaskStarter_UpdatesFailureAccounting(t *testing.T) { for i := 0; i < threshold; i++ { idempotencyKey := fmt.Sprintf("task-noop-accounting-%d", i) - if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-accounting"}`, idempotencyKey); err != nil { t.Fatalf("queue failure %d: %v", i, err) } diff --git a/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go b/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go index 2268840c0e..1fdf6b2050 100644 --- a/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go +++ b/apps/backend/internal/office/service/wo46_idle_skip_routine_dispatch_test.go @@ -32,7 +32,7 @@ func TestIdleSkip_RoutineDispatchNoTasks_Skipped(t *testing.T) { } // Worker defaults to skip_idle_runs=true, no tasks assigned. - if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } @@ -98,7 +98,7 @@ func TestIdleSkip_RoutineDispatch_CoordinatorNotSkipped(t *testing.T) { t.Fatalf("CEO role should default to SkipIdleRuns=false") } - if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { + if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil { t.Fatalf("queue: %v", err) } diff --git a/apps/backend/internal/office/shared/interfaces.go b/apps/backend/internal/office/shared/interfaces.go index f99aeb6cf1..40f2938fb1 100644 --- a/apps/backend/internal/office/shared/interfaces.go +++ b/apps/backend/internal/office/shared/interfaces.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kandev/kandev/internal/office/models" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/workflow/engine" ) @@ -39,8 +40,11 @@ type AgentWriter interface { // Implemented by the run feature (and transitionally by office/service.Service). type RunQueuer interface { // QueueRun enqueues a run for agentInstanceID with the given reason, payload, - // and optional idempotency key (empty string disables deduplication). - QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error + // and optional idempotency key (empty string disables deduplication). The + // returned QueueOutcome reports what actually happened (queued / deduped / + // coalesced / none-on-error) so callers that need to distinguish a fresh + // insert from a no-op don't have to infer it from side effects. + QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error) } // WorkflowEngineDispatcher routes typed office task events through the diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow.go b/apps/backend/internal/orchestrator/event_handlers_workflow.go index fda4968dcd..038ca4d752 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow.go @@ -22,6 +22,7 @@ import ( "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/orchestrator/messagequeue" "github.com/kandev/kandev/internal/orchestrator/watcher" + runsservice "github.com/kandev/kandev/internal/runs/service" "github.com/kandev/kandev/internal/steptelemetry" "github.com/kandev/kandev/internal/sysprompt" "github.com/kandev/kandev/internal/task/models" @@ -2179,13 +2180,16 @@ func (s *Service) queueOfficeAutoStartRun(ctx context.Context, task *models.Task } // officeAutoStartIdempotencyKey uses the immutable workflow-step transition -// row as the per-entry component. A legacy event without that field uses the -// task timestamp as a compatibility fallback until the event is republished. +// row as the per-entry component. A zero stepTransitionID means the +// per-occurrence identity is unavailable, so the enqueue goes keyless rather +// than falling back to a time-derived key that would never suppress a +// redelivery. func officeAutoStartIdempotencyKey(task *models.Task, agentProfileID, stepID string, stepTransitionID int64) string { - entryID := strconv.FormatInt(stepTransitionID, 10) if stepTransitionID == 0 { - entryID = "legacy:" + task.UpdatedAt.UTC().Format(time.RFC3339Nano) + runsservice.ReportKeylessEnqueue(officeAutoStartRunReason, runsservice.KeylessCauseUnresolved, "zero_step_transition") + return "" } + entryID := strconv.FormatInt(stepTransitionID, 10) return fmt.Sprintf("%s:%s:%s:%s:%s", officeAutoStartRunReason, task.ID, agentProfileID, stepID, entryID) } diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go index 5b75c2e466..5827a69637 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_office_autostart_test.go @@ -36,6 +36,20 @@ func TestOfficeAutoStartIdempotencyKey(t *testing.T) { } } +// TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless pins that a +// zero StepTransitionID (no per-occurrence identity available) enqueues with +// no key at all, rather than falling back to a time-derived key that would +// never suppress a genuine redelivery. +func TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless(t *testing.T) { + task := &models.Task{ID: "t1", UpdatedAt: time.Now().UTC(), WorkflowStepTransitionID: 0} + + key := officeAutoStartIdempotencyKey(task, "agent-1", "step2", task.WorkflowStepTransitionID) + + if key != "" { + t.Errorf("key = %q, want empty (keyless) for zero StepTransitionID", key) + } +} + // TestAutoStartOfficeTaskLogsQueuedOutcomeAtInfo pins that a real insert // (QueueOutcomeQueued) is logged as an info-level "queued" message, and the // log is only emitted after the queue attempt resolves — not asserted diff --git a/apps/backend/internal/runs/dedupkeys/dedupkeys.go b/apps/backend/internal/runs/dedupkeys/dedupkeys.go new file mode 100644 index 0000000000..95da27cf1d --- /dev/null +++ b/apps/backend/internal/runs/dedupkeys/dedupkeys.go @@ -0,0 +1,38 @@ +// Package dedupkeys centralizes the shared dedup-key builders that more than +// one office producer must derive identically for the same occurrence +// (docs/specs/office/system-design/run-dedup-generation-01.md#convergent-producers, +// #the-shared-key-builders). Placed beside internal/runs/commentkeys as a +// sibling package so the office service, the office scheduler and the +// orchestrator can all reach it without an import cycle. +package dedupkeys + +import ( + "crypto/sha256" + "fmt" + "sort" + "strings" +) + +// AssignmentKey builds the canonical task_assigned dedup key. Both +// task_assigned producers (office/service's queueTaskAssignedRun and +// office/scheduler's reactToAssigneeChange) call this rather than +// formatting the string themselves, which is what makes their convergence +// structural instead of coincidental (AC-OFFICE-RUN-DEDUP-002.1). +func AssignmentKey(taskID, agentProfileID string, generation int64) string { + return fmt.Sprintf("task_assigned:%s:%s:%d", taskID, agentProfileID, generation) +} + +// BlockerDigest computes the shared digest over a blocker-task-id set. Both +// blocker producers (office/service's resolveAndWakeIfUnblocked and +// office/scheduler's cascadeBlockersResolved) call this so a set observed in +// different read order still converges on the same digest. The encoding is +// binding: sort ascending by byte value (NOT ListTaskBlockers's created_at, +// which is not unique), join with a single comma, then SHA-256 the UTF-8 +// bytes and render the full digest as lowercase hex. +func BlockerDigest(blockerTaskIDs []string) string { + sorted := make([]string, len(blockerTaskIDs)) + copy(sorted, blockerTaskIDs) + sort.Strings(sorted) + sum := sha256.Sum256([]byte(strings.Join(sorted, ","))) + return fmt.Sprintf("%x", sum) +} diff --git a/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go b/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go new file mode 100644 index 0000000000..7e6867670c --- /dev/null +++ b/apps/backend/internal/runs/dedupkeys/dedupkeys_test.go @@ -0,0 +1,43 @@ +package dedupkeys + +import "testing" + +func TestAssignmentKey(t *testing.T) { + got := AssignmentKey("task-1", "agent-1", 3) + want := "task_assigned:task-1:agent-1:3" + if got != want { + t.Fatalf("AssignmentKey() = %q, want %q", got, want) + } +} + +func TestAssignmentKey_VariesByGeneration(t *testing.T) { + a := AssignmentKey("task-1", "agent-1", 1) + b := AssignmentKey("task-1", "agent-1", 2) + if a == b { + t.Fatalf("expected distinct keys for distinct generations, got %q for both", a) + } +} + +func TestBlockerDigest_OrderIndependent(t *testing.T) { + a := BlockerDigest([]string{"b1", "b2", "b3"}) + b := BlockerDigest([]string{"b3", "b1", "b2"}) + if a != b { + t.Fatalf("BlockerDigest should be order-independent: %q != %q", a, b) + } +} + +func TestBlockerDigest_VariesBySet(t *testing.T) { + a := BlockerDigest([]string{"b1", "b2"}) + b := BlockerDigest([]string{"b1", "b2", "b3"}) + if a == b { + t.Fatalf("expected distinct digests for distinct sets, got %q for both", a) + } +} + +func TestBlockerDigest_DoesNotMutateInput(t *testing.T) { + input := []string{"b3", "b1", "b2"} + _ = BlockerDigest(input) + if input[0] != "b3" || input[1] != "b1" || input[2] != "b2" { + t.Fatalf("BlockerDigest mutated its input slice: %v", input) + } +} diff --git a/apps/backend/internal/runs/service/dedup.go b/apps/backend/internal/runs/service/dedup.go new file mode 100644 index 0000000000..ce9d9f0f8e --- /dev/null +++ b/apps/backend/internal/runs/service/dedup.go @@ -0,0 +1,116 @@ +package service + +import ( + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" +) + +// QueueOutcomeNone means no enqueue was attempted, or the attempt returned +// an error. It is the zero value, so it is what a widened signature yields +// on any path that returns before deciding an outcome. +// +// It is declared identically in internal/workflow/engine/adapters.go — both +// declarations MUST match, the same invariant QueueOutcome itself already +// carries. +const QueueOutcomeNone QueueOutcome = "" + +// QueueSource distinguishes which queue implementation made a dedup +// decision, for the "queue" label on office_run_dedup_total. +type QueueSource string + +const ( + // QueueSourceRuns identifies a suppression on the runs table (the + // idx_run_idempotency windowed lookup or durable unique index). + QueueSourceRuns QueueSource = "runs" + // QueueSourceWakeup identifies a suppression on the wakeup-request + // table. That table has no windowed lookup, so its kind is always + // "durable". + QueueSourceWakeup QueueSource = "wakeup" +) + +// KeylessCause discriminates why a producer enqueued a wake with no dedup +// key: a generation that should have been available and was not +// (KeylessCauseUnresolved), or a producer that never had an occurrence to +// name (KeylessCauseByDesign). See +// docs/specs/office/system-design/run-dedup-generation-01.md#unresolvable-generation. +type KeylessCause string + +const ( + // KeylessCauseUnresolved means the occurrence has a generation that this + // producer simply was not handed or could not resolve. + KeylessCauseUnresolved KeylessCause = "unresolved" + // KeylessCauseByDesign means the producer never had an occurrence + // identity to name — a status transition with no redelivery path, an + // agent that expressed no dedup intent, and similar. + KeylessCauseByDesign KeylessCause = "by_design" +) + +func dedupLogger() *logger.Logger { + return logger.Default().WithFields(zap.String("component", "runs-dedup")) +} + +// ReportWindowedDedup records a suppression caught by the recent-duplicate +// lookup (a windowed dedup hit): counts kind="windowed", logs at Info, and +// returns QueueOutcomeDeduped. +func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome { + incRunDedup(q, reason, "windowed") + dedupLogger().Info("run deduplicated (windowed)", + zap.String("queue", string(q)), + zap.String("reason", reason), + zap.String("key", key)) + return QueueOutcomeDeduped +} + +// ReportInsertResult classifies the error from an insert attempt. A unique +// violation on idx_run_idempotency counts kind="durable", logs at Warn with +// key, reason and agent, and returns (QueueOutcomeDeduped, nil). Any other +// non-nil error is returned UNCHANGED with QueueOutcomeNone and moves no +// counter — a disk error is not a dedup decision. A nil error returns +// (QueueOutcomeQueued, nil). +func ReportInsertResult( + q QueueSource, reason, key, agentProfileID string, err error, +) (QueueOutcome, error) { + if err == nil { + return QueueOutcomeQueued, nil + } + if runssqlite.IsIdempotencyKeyUniqueViolation(err) { + return ReportDurableDedup(q, reason, key, agentProfileID), nil + } + return QueueOutcomeNone, err +} + +// ReportDurableDedup records a durable conflict the caller has already +// classified (the wakeup path's ErrWakeupIdempotencyConflict, or +// ReportInsertResult's own classification): counts kind="durable", logs at +// Warn, and returns QueueOutcomeDeduped. +func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome { + incRunDedup(q, reason, "durable") + dedupLogger().Warn("run deduplicated (durable index)", + zap.String("queue", string(q)), + zap.String("reason", reason), + zap.String("key", key), + zap.String("agent_profile_id", agentProfileID)) + return QueueOutcomeDeduped +} + +// ReportKeylessEnqueue records a producer's decision to enqueue a wake with +// no dedup key, attributed to reason and discriminated by cause so a +// resolution failure (unresolved) is countable separately from expected, +// by-design keyless traffic. Called by the producer at its own decision +// site, immediately before enqueuing — the queue itself never infers a +// cause from an empty key. detail names what failed to resolve (a fixed +// lowercase snake_case constant, never a formatted message); it is a log +// field only, never a counter label, and is empty for cause=by_design, +// which is counted but never logged. +func ReportKeylessEnqueue(reason string, cause KeylessCause, detail string) { + incRunDedupKeyless(reason, cause) + if cause != KeylessCauseUnresolved { + return + } + dedupLogger().Info("run enqueued with no dedup key", + zap.String("reason", reason), + zap.String("cause", string(cause)), + zap.String("detail", detail)) +} diff --git a/apps/backend/internal/runs/service/dedup_test.go b/apps/backend/internal/runs/service/dedup_test.go new file mode 100644 index 0000000000..5bb38b05f0 --- /dev/null +++ b/apps/backend/internal/runs/service/dedup_test.go @@ -0,0 +1,140 @@ +package service_test + +import ( + "errors" + "expvar" + "strings" + "testing" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// counterHasLabel reports whether the named expvar.Map has an entry whose +// key contains every given substring. Counters are process-global expvar +// state (no reset hook), so tests assert presence/growth rather than exact +// values that could collide with other tests in the same binary run. +func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a *expvar.Map", mapName) + } + found := false + m.Do(func(kv expvar.KeyValue) { + matches := true + for _, s := range substrs { + if !strings.Contains(kv.Key, s) { + matches = false + break + } + } + if matches { + found = true + } + }) + return found +} + +func TestQueueOutcomeNone_IsZeroValue(t *testing.T) { + if runsservice.QueueOutcomeNone != "" { + t.Fatalf("QueueOutcomeNone = %q, want empty string", runsservice.QueueOutcomeNone) + } + var zero runsservice.QueueOutcome + if zero != runsservice.QueueOutcomeNone { + t.Fatalf("zero QueueOutcome %q != QueueOutcomeNone", zero) + } +} + +func TestReportWindowedDedup(t *testing.T) { + reason := "agent-supplied-windowed-" + t.Name() + outcome := runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, "some-key") + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=windowed", "queue=runs") { + t.Fatal("expected custom reasons to share the bounded windowed/runs entry") + } + if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) { + t.Fatal("agent-supplied reason must not create its own metric series") + } +} + +func TestReportInsertResult_NilError(t *testing.T) { + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, "r", "k", "agent", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if outcome != runsservice.QueueOutcomeQueued { + t.Fatalf("outcome = %q, want queued", outcome) + } +} + +func TestReportInsertResult_UniqueViolation(t *testing.T) { + reason := "agent-supplied-durable-" + t.Name() + violation := errors.New("UNIQUE constraint failed: runs.idempotency_key") + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", violation) + if err != nil { + t.Fatalf("expected nil error on classified conflict, got %v", err) + } + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=runs") { + t.Fatal("expected custom reasons to share the bounded durable/runs entry") + } + if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) { + t.Fatal("agent-supplied reason must not create its own metric series") + } +} + +func TestReportInsertResult_OtherError_PassesThrough(t *testing.T) { + reason := "test_other_" + t.Name() + diskErr := errors.New("disk I/O error") + outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", diskErr) + if !errors.Is(err, diskErr) { + t.Fatalf("expected the original error to pass through unchanged, got %v", err) + } + if outcome != runsservice.QueueOutcomeNone { + t.Fatalf("outcome = %q, want none", outcome) + } + if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) { + t.Fatal("a non-conflict error must not move the dedup counter") + } +} + +func TestReportDurableDedup_Wakeup(t *testing.T) { + reason := "agent-supplied-wakeup-" + t.Name() + outcome := runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, reason, "k", "agent-1") + if outcome != runsservice.QueueOutcomeDeduped { + t.Fatalf("outcome = %q, want deduped", outcome) + } + if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=wakeup") { + t.Fatal("expected custom reasons to share the bounded durable/wakeup entry") + } + if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) { + t.Fatal("agent-supplied reason must not create its own metric series") + } +} + +func TestReportKeylessEnqueue_CountsBothCauses(t *testing.T) { + reasonUnresolved := "agent-supplied-keyless-unresolved-" + t.Name() + reasonByDesign := "agent-supplied-keyless-by-design-" + t.Name() + + runsservice.ReportKeylessEnqueue(reasonUnresolved, runsservice.KeylessCauseUnresolved, "some_detail") + runsservice.ReportKeylessEnqueue(reasonByDesign, runsservice.KeylessCauseByDesign, "") + + if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason=custom", "cause=unresolved") { + t.Fatal("expected custom reasons to share the bounded unresolved entry") + } + if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason=custom", "cause=by_design") { + t.Fatal("expected custom reasons to share the bounded by_design entry") + } + if counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonUnresolved) || + counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonByDesign) { + t.Fatal("agent-supplied reasons must not create their own keyless metric series") + } +} diff --git a/apps/backend/internal/runs/service/metrics_vars.go b/apps/backend/internal/runs/service/metrics_vars.go new file mode 100644 index 0000000000..614d7362a0 --- /dev/null +++ b/apps/backend/internal/runs/service/metrics_vars.go @@ -0,0 +1,75 @@ +package service + +import ( + "expvar" + "strings" +) + +const metricReasonCustom = "custom" + +// metricReasons is the bounded set of run reasons that receive their own +// metric series. Agent-supplied or future reasons use metricReasonCustom so +// process-global expvar maps cannot grow once per distinct input. +var metricReasons = map[string]struct{}{ + "task_assigned": {}, + "task_comment": {}, + "task_blockers_resolved": {}, + "task_children_completed": {}, + "approval_resolved": {}, + "task_review_requested": {}, + "task_changes_requested": {}, + "routine_trigger": {}, + "heartbeat": {}, + "budget_alert": {}, + "agent_error": {}, + "task_unblocked": {}, + "task_reopened": {}, + "task_reopened_via_comment": {}, + "task_mentioned": {}, + "stage_pending": {}, + "stage_changes_requested": {}, + "task_ready_to_close": {}, + "routine_dispatch": {}, + "routine_dispatch_cron": {}, + "routine_dispatch_event": {}, + "manual_resume_after_failure": {}, + "review_started": {}, + "approval_started": {}, + "blockers_resolved": {}, + "children_completed": {}, +} + +// expvar maps published at package init, exposed via stdlib's /debug/vars +// handler. Mirrors internal/office/scheduler/metrics_vars.go's "k=v;k=v" +// label model and counters-only rule. +var ( + runDedupTotal = expvar.NewMap("office_run_dedup_total") + runDedupKeylessTotal = expvar.NewMap("office_run_dedup_keyless_total") +) + +// metricLabel builds a "k1=v1;k2=v2;..." label string for an expvar map key. +func metricLabel(pairs ...string) string { + if len(pairs)%2 != 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2) + for i := 0; i < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + return strings.Join(parts, ";") +} + +func incRunDedup(q QueueSource, reason, kind string) { + runDedupTotal.Add(metricLabel("reason", metricReason(reason), "kind", kind, "queue", string(q)), 1) +} + +func incRunDedupKeyless(reason string, cause KeylessCause) { + runDedupKeylessTotal.Add(metricLabel("reason", metricReason(reason), "cause", string(cause)), 1) +} + +func metricReason(reason string) string { + if _, ok := metricReasons[reason]; ok { + return reason + } + return metricReasonCustom +} diff --git a/apps/backend/internal/runs/service/queue_outcome_none_test.go b/apps/backend/internal/runs/service/queue_outcome_none_test.go new file mode 100644 index 0000000000..d1edb0eb40 --- /dev/null +++ b/apps/backend/internal/runs/service/queue_outcome_none_test.go @@ -0,0 +1,22 @@ +package service_test + +import ( + "testing" + + "github.com/kandev/kandev/internal/workflow/engine" + + runsservice "github.com/kandev/kandev/internal/runs/service" +) + +// TestQueueOutcomeNone_MatchesEngineDeclaration pins the "both MUST match" +// invariant both QueueOutcomeNone doc comments carry: runs/service and +// workflow/engine declare the same string type and zero-value constant +// independently, and a change to one without the other would silently break +// callers comparing an outcome received from one package against a constant +// imported from the other. +func TestQueueOutcomeNone_MatchesEngineDeclaration(t *testing.T) { + if string(runsservice.QueueOutcomeNone) != string(engine.QueueOutcomeNone) { + t.Fatalf("runsservice.QueueOutcomeNone = %q, engine.QueueOutcomeNone = %q; both MUST match", + runsservice.QueueOutcomeNone, engine.QueueOutcomeNone) + } +} diff --git a/apps/backend/internal/runs/service/service.go b/apps/backend/internal/runs/service/service.go index ea0c42e449..43d9349c53 100644 --- a/apps/backend/internal/runs/service/service.go +++ b/apps/backend/internal/runs/service/service.go @@ -12,7 +12,6 @@ package service import ( "context" "encoding/json" - "errors" "fmt" "time" @@ -27,18 +26,6 @@ import ( runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" ) -// errIdempotencyKeyConflict signals that insertRun's CreateRun failed -// because idx_run_idempotency rejected a duplicate idempotency_key — -// distinct from the earlier CheckIdempotencyKey miss, which only looks -// within IdempotencyWindowHours. Two independent producers deriving the -// same operation id for the same event can both pass that fast-path check -// before either commits (or the colliding row can simply be older than the -// window); the unique index is what actually stops the second insert. -// QueueRun treats this as a durable dedupe hit: QueueOutcomeDeduped, not an -// error, so the losing producer's caller does not abort or log a spurious -// failure for what is really a no-op. -var errIdempotencyKeyConflict = errors.New("idempotency key conflict") - // RunQueueAdapter is the interface the workflow engine uses to enqueue // runs from queue_run actions. Phase 2 final's parallel agent // declares the same shape inside internal/workflow/engine; both @@ -180,34 +167,32 @@ func (s *Service) SubscribeSignal() <-chan struct{} { return s.signalCh } func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutcome, error) { agentInstanceID, err := s.resolveAgentInstance(ctx, req) if err != nil { - return "", err + return QueueOutcomeNone, err } if agentInstanceID == "" { - return "", fmt.Errorf("queue run: agent_profile_id is required") + return QueueOutcomeNone, fmt.Errorf("queue run: agent_profile_id is required") } if req.IdempotencyKey != "" { dup, err := s.repo.CheckIdempotencyKey(ctx, req.IdempotencyKey, IdempotencyWindowHours) if err != nil { - return "", fmt.Errorf("idempotency check: %w", err) + return QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err) } if dup { - s.log.Debug("run skipped (idempotent)", - zap.String("key", req.IdempotencyKey)) - return QueueOutcomeDeduped, nil + return ReportWindowedDedup(QueueSourceRuns, req.Reason, req.IdempotencyKey), nil } } payloadMap := runPayload(req, agentInstanceID) payload, err := encodePayload(payloadMap) if err != nil { - return "", fmt.Errorf("encode payload: %w", err) + return QueueOutcomeNone, fmt.Errorf("encode payload: %w", err) } if shouldCoalesceRun(req) { coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, req.Reason, CoalesceWindowSeconds, payload) if err != nil { - return "", fmt.Errorf("coalesce check: %w", err) + return QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err) } if coalesced { s.log.Debug("run coalesced", @@ -220,20 +205,17 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco } } - row, err := s.insertRun(ctx, agentInstanceID, req, payload) + row, insertErr := s.insertRun(ctx, agentInstanceID, req, payload) + // idx_run_idempotency has no time bound, so a conflict here can come + // from a row older than IdempotencyWindowHours, not just the windowed + // race CheckIdempotencyKey guards against above. ReportInsertResult + // classifies that as a no-op dedupe rather than a hard error. + outcome, err := ReportInsertResult(QueueSourceRuns, req.Reason, req.IdempotencyKey, agentInstanceID, insertErr) if err != nil { - // idx_run_idempotency has no time bound, so a conflict here can - // come from a row older than IdempotencyWindowHours, not just the - // windowed race CheckIdempotencyKey guards against above. Either - // way the existing row is definitionally the same operation this - // key identifies, so treat it as a no-op dedupe rather than a hard - // error (see errIdempotencyKeyConflict's doc comment). - if errors.Is(err, errIdempotencyKeyConflict) { - s.log.Debug("run skipped (idempotency index race)", - zap.String("key", req.IdempotencyKey)) - return QueueOutcomeDeduped, nil - } - return "", err + return QueueOutcomeNone, err + } + if outcome == QueueOutcomeDeduped { + return outcome, nil } s.log.Info("run queued", @@ -247,7 +229,10 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco } // insertRun creates the runs row and returns it. Pulled out of -// QueueRun to keep the latter under the funlen budget. +// QueueRun to keep the latter under the funlen budget. The returned error is +// the raw CreateRun error (wrapped for context, never reclassified) — the +// caller runs it through ReportInsertResult, which does its own unique-index +// classification against the original error chain. func (s *Service) insertRun( ctx context.Context, agentInstanceID string, req QueueRunRequest, payload string, ) (*models.Run, error) { @@ -267,9 +252,6 @@ func (s *Service) insertRun( RequestedAt: time.Now().UTC(), } if err := s.repo.CreateRun(ctx, row); err != nil { - if runssqlite.IsIdempotencyKeyUniqueViolation(err) { - return nil, errIdempotencyKeyConflict - } return nil, fmt.Errorf("enqueue run: %w", err) } return row, nil diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 095d3b5747..4f942f4240 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -116,6 +116,12 @@ func (r *Repository) runMigrations() error { if err := r.migrateTasksRemoveWorkflowFK(); err != nil { return err } + // Must run AFTER migrateTasksRemoveWorkflowFK: that migration recreates + // tasks from an explicit column list. Adding this column beforehand would + // have it silently dropped by the recreate on any database still carrying + // the legacy FK, leaving it absent for the remainder of that boot (the + // same hazard class as the task_sessions.name comment above). + _ = r.migrate.Apply("tasks.assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`) if err := r.dropRetiredSlackIntegration(); err != nil { return err } diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index 21c861fd4c..ddca362aa8 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -390,6 +390,14 @@ func (r *Repository) insertTaskTx(ctx context.Context, tx *sql.Tx, task *models. if err := upsertRunnerInTx(ctx, tx, r.db.Rebind, task.WorkflowStepID, task.ID, task.AssigneeAgentProfileID); err != nil { return "", err } + // A task created already assigned starts at generation 1, the same + // value UpdateTaskAssignee would commit for a first assignment - a + // creation and a following first assignment must not both mint + // generation 1. + if _, err := tx.ExecContext(ctx, r.db.Rebind( + `UPDATE tasks SET assignment_generation = 1 WHERE id = ?`), task.ID); err != nil { + return "", err + } } return entryID, nil } diff --git a/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go b/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go new file mode 100644 index 0000000000..8df37ce5fa --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/tasks_workflow_fk_removal_assignment_generation_test.go @@ -0,0 +1,117 @@ +package sqlite + +import ( + "strings" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" +) + +// TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild is the +// replay regression test flagged (but not written) in the run-dedup-generation +// build: migrateTasksRemoveWorkflowFK (base_migrations.go) recreates `tasks` +// from an explicit 19-column list that predates assignment_generation. The +// tasks.assignment_generation ADD COLUMN is sequenced strictly AFTER that +// rebuild in runMigrations() precisely so a database still carrying the +// legacy `FOREIGN KEY (workflow_id)` DDL does not have the new column +// silently dropped for the remainder of that boot (the same hazard class as +// the task_sessions.name comment in the same file). This had zero coverage +// before this test: nothing exercised migrateTasksRemoveWorkflowFK's rebuild +// path at all. +func TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild(t *testing.T) { + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + // Seed a pre-migration `tasks` table: the legacy FK clause that triggers + // the rebuild, the 19 columns migrateTasksRemoveWorkflowFK's SELECT list + // expects, and no assignment_generation column at all — matching a real + // database that predates this feature. + if _, err := db.Exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + workflow_id TEXT NOT NULL DEFAULT '', + workflow_step_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + description TEXT DEFAULT '', + state TEXT DEFAULT 'TODO', + priority INTEGER DEFAULT 0, + position INTEGER DEFAULT 0, + wip_admitted INTEGER NOT NULL DEFAULT 1, + queued_for_step_id TEXT NOT NULL DEFAULT '', + queued_at TIMESTAMP, + metadata TEXT DEFAULT '{}', + is_ephemeral INTEGER NOT NULL DEFAULT 0, + parent_id TEXT DEFAULT '', + autopilot_enabled INTEGER NOT NULL DEFAULT 0, + archived_at TIMESTAMP, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + FOREIGN KEY (workflow_id) REFERENCES workflows(id) + ); + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES ('task-legacy-fk', 'ws-legacy', 'Legacy FK task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); + `); err != nil { + t.Fatalf("seed legacy schema: %v", err) + } + + if _, err := NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo (run migrations): %v", err) + } + + // The rebuild must have fired: the legacy FK clause is gone. + var tableSQL string + if err := db.QueryRow( + `SELECT sql FROM sqlite_master WHERE type='table' AND name='tasks'`, + ).Scan(&tableSQL); err != nil { + t.Fatalf("read tasks schema: %v", err) + } + if strings.Contains(tableSQL, "FOREIGN KEY (workflow_id)") { + t.Fatalf("tasks table still carries the legacy workflow_id FK after migration: %s", tableSQL) + } + + // assignment_generation must be present post-rebuild, not silently + // dropped by the rebuild's pre-feature explicit column list. + rows, err := db.Queryx(`PRAGMA table_info(tasks)`) + if err != nil { + t.Fatalf("pragma table_info: %v", err) + } + found := false + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dflt *string + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + _ = rows.Close() + t.Fatalf("scan pragma: %v", err) + } + if name == "assignment_generation" { + found = true + } + } + _ = rows.Close() + if !found { + t.Fatalf("assignment_generation column missing after migrating a legacy FK-bearing tasks table") + } + + // The pre-existing row must survive the rebuild, with the new column + // defaulted to 0. + var gotTitle string + var gotGeneration int64 + if err := db.QueryRow( + `SELECT title, assignment_generation FROM tasks WHERE id = ?`, "task-legacy-fk", + ).Scan(&gotTitle, &gotGeneration); err != nil { + t.Fatalf("query migrated row: %v", err) + } + if gotTitle != "Legacy FK task" { + t.Fatalf("title = %q, want %q (row lost during rebuild)", gotTitle, "Legacy FK task") + } + if gotGeneration != 0 { + t.Fatalf("assignment_generation = %d, want 0 for a pre-existing row", gotGeneration) + } +} diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index b89a9d56cf..8c85b0f092 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -440,7 +440,11 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas task.Repositories = repos } - s.publishTaskEvent(ctx, events.TaskCreated, task, nil) + s.publishTaskEventWithExtra(ctx, events.TaskCreated, task, nil, + map[string]interface{}{ + "assignee_agent_profile_id": task.AssigneeAgentProfileID, + "assignment_generation": assignmentGenerationForCreate(task), + }) s.pullTasksFromNewFeederWork(ctx, task.WorkflowID, task.WorkflowStepID) if refreshed, err := s.tasks.GetTask(ctx, task.ID); err != nil { s.logger.Warn("failed to refresh task after feeder pull", zap.String("task_id", task.ID), zap.Error(err)) @@ -453,6 +457,17 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas return CreateTaskResult{Task: task, Outcome: CreateTaskOutcomeCreated}, nil } +// assignmentGenerationForCreate mirrors insertTaskTx's runner-row guard in +// memory rather than re-reading the row it just wrote: a task created +// already assigned starts at generation 1 (matching the value insertTaskTx +// committed), everything else starts at 0 (never assigned). +func assignmentGenerationForCreate(task *models.Task) int64 { + if task.AssigneeAgentProfileID != "" && task.WorkflowStepID != "" { + return 1 + } + return 0 +} + func (s *Service) prepareWorkspacePolicyForCreation(ctx context.Context, req *CreateTaskRequest) error { policy := WorkspacePolicy{} if req.WorkspacePolicy != nil { diff --git a/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go b/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go new file mode 100644 index 0000000000..749c34d767 --- /dev/null +++ b/apps/backend/internal/task/service/service_tasks_assignment_generation_test.go @@ -0,0 +1,125 @@ +package service + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// setupOfficeTestWithBus mirrors setupOfficeTest but keeps the MockEventBus +// so a test can inspect the payload of a published task.created event. +func setupOfficeTestWithBus(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repository) { + t.Helper() + svc, bus, repo := createTestService(t) + ctx := context.Background() + + _, err := repo.DB().Exec(` + CREATE TABLE IF NOT EXISTS workflow_steps ( + id TEXT PRIMARY KEY, + workflow_id TEXT NOT NULL, + name TEXT NOT NULL, + position INTEGER NOT NULL, + color TEXT DEFAULT '', + prompt TEXT DEFAULT '', + events TEXT DEFAULT '{}', + allow_manual_move INTEGER DEFAULT 1, + is_start_step INTEGER DEFAULT 0, + show_in_command_panel INTEGER DEFAULT 1, + auto_archive_after_hours INTEGER DEFAULT 0, + agent_profile_id TEXT DEFAULT '', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + FOREIGN KEY (workflow_id) REFERENCES workflows(id) ON DELETE CASCADE + )`) + if err != nil { + t.Fatalf("create workflow_steps: %v", err) + } + + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("create workspace: %v", err) + } + if _, err := repo.EnsureOfficeWorkflow(ctx, "ws-1"); err != nil { + t.Fatalf("EnsureOfficeWorkflow: %v", err) + } + svc.SetStartStepResolver(&dbStepResolver{repo: repo}) + svc.SetWorkspacePolicyAttacher(noOpWorkspacePolicyAttacher{}) + return svc, bus, repo +} + +// AC-OFFICE-RUN-DEDUP-001.9: a task created already assigned publishes +// assignment_generation 1 on its task.created event, matching the value +// insertTaskTx committed - a creation and the first following assignment +// must not both mint generation 1. +func TestCreateTask_Assigned_PublishesAssignmentGenerationOne(t *testing.T) { + svc, bus, repo := setupOfficeTestWithBus(t) + ctx := context.Background() + + result, err := svc.CreateTask(ctx, &CreateTaskRequest{ + WorkspaceID: "ws-1", + Title: "Agent Task", + Origin: models.TaskOriginAgentCreated, + AssigneeAgentProfileID: "agent-1", + }) + if err != nil { + t.Fatalf("CreateTask: %v", err) + } + + data := singlePublishedEventData(t, bus) + gen, ok := data["assignment_generation"].(int64) + if !ok || gen != 1 { + t.Fatalf("assignment_generation = %#v, want int64(1)", data["assignment_generation"]) + } + if got, ok := data["assignee_agent_profile_id"].(string); !ok || got != "agent-1" { + t.Fatalf("assignee_agent_profile_id = %#v, want agent-1", data["assignee_agent_profile_id"]) + } + + stored, err := repo.GetTask(ctx, result.Task.ID) + if err != nil { + t.Fatalf("GetTask: %v", err) + } + var persistedGen int64 + if err := repo.DB().QueryRowContext(ctx, + `SELECT assignment_generation FROM tasks WHERE id = ?`, stored.ID).Scan(&persistedGen); err != nil { + t.Fatalf("query assignment_generation: %v", err) + } + if persistedGen != 1 { + t.Fatalf("persisted assignment_generation = %d, want 1", persistedGen) + } +} + +// AC-OFFICE-RUN-DEDUP-001.9 boundary: a task created UNASSIGNED publishes +// assignment_generation 0, never 1 - the boundary that keeps a creation and a +// first assignment from both minting generation 1. +func TestCreateTask_Unassigned_PublishesAssignmentGenerationZero(t *testing.T) { + svc, bus, repo := setupOfficeTestWithBus(t) + ctx := context.Background() + + result, err := svc.CreateTask(ctx, &CreateTaskRequest{ + WorkspaceID: "ws-1", + Title: "Unassigned Task", + ProjectID: "proj-1", + }) + if err != nil { + t.Fatalf("CreateTask: %v", err) + } + + data := singlePublishedEventData(t, bus) + gen, ok := data["assignment_generation"].(int64) + if !ok || gen != 0 { + t.Fatalf("assignment_generation = %#v, want int64(0)", data["assignment_generation"]) + } + if got, ok := data["assignee_agent_profile_id"].(string); !ok || got != "" { + t.Fatalf("assignee_agent_profile_id = %#v, want empty", data["assignee_agent_profile_id"]) + } + + var persistedGen int64 + if err := repo.DB().QueryRowContext(ctx, + `SELECT assignment_generation FROM tasks WHERE id = ?`, result.Task.ID).Scan(&persistedGen); err != nil { + t.Fatalf("query assignment_generation: %v", err) + } + if persistedGen != 0 { + t.Fatalf("persisted assignment_generation = %d, want 0", persistedGen) + } +} diff --git a/apps/backend/internal/workflow/engine/adapters.go b/apps/backend/internal/workflow/engine/adapters.go index dc5533374d..66634dd558 100644 --- a/apps/backend/internal/workflow/engine/adapters.go +++ b/apps/backend/internal/workflow/engine/adapters.go @@ -34,6 +34,12 @@ const ( // queued row for the same agent + reason within the coalescing // window, so nothing new was inserted. QueueOutcomeCoalesced QueueOutcome = "coalesced" + // QueueOutcomeNone means no enqueue was attempted, or the attempt + // returned an error. It is the zero value, so it is what a widened + // signature yields on any path that returns before deciding an + // outcome. Declared identically in internal/runs/service — both + // declarations MUST match. + QueueOutcomeNone QueueOutcome = "" ) // QueueRunRequest is the typed payload the engine hands to RunQueueAdapter. diff --git a/docs/plans/office-run-dedup-generation/plan.md b/docs/plans/office-run-dedup-generation/plan.md new file mode 100644 index 0000000000..ff6da7af4e --- /dev/null +++ b/docs/plans/office-run-dedup-generation/plan.md @@ -0,0 +1,47 @@ +--- +created: 2026-09-13 +status: completed +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +system_design: + - ../../specs/office/system-design/run-dedup-generation-01.md + - ../../specs/office/system-design/run-dedup-generation-02.md + - ../../specs/office/system-design/run-dedup-generation-03.md +--- + +# Implementation Plan: Office Run Deduplication Generation Identity + +## Overview + +Give Office wake requests an occurrence identity that survives retries and +allows later occurrences to run. Persist the task assignment generation, carry +it through task-created events, keep producer keys convergent, and report +deduplication outcomes with bounded metric labels. + +## Scope + +- Persist and increment `tasks.assignment_generation` for assignment writes. +- Include the assignee profile and generation in task-created event data. +- Use generation-aware keys for assignment and other durable wake occurrences. +- Use a keyless wake when a generation cannot be resolved. +- Keep agent-supplied metric reasons bounded to known labels or `custom`. +- Preserve the existing queue lookup, unique indexes, coalescing window, and run + lifecycle. + +## Implementation Wave + +- [x] [task-01-run-dedup-generation](task-01-run-dedup-generation.md) + +## Verification + +```bash +cd apps/backend && go test ./internal/runs/service ./internal/office/service \ + ./internal/office/dashboard ./internal/task/service +python3 scripts/lint-spec-files.py --all +``` + +The focused backend tests cover assignment event replay, same-agent +reassignment, bounded metric labels, and the existing deduplication paths. diff --git a/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md b/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md new file mode 100644 index 0000000000..15438035ae --- /dev/null +++ b/docs/plans/office-run-dedup-generation/task-01-run-dedup-generation.md @@ -0,0 +1,74 @@ +--- +id: "01-run-dedup-generation" +title: "Implement Office run deduplication generation identity" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +acceptance_criteria: + - AC-OFFICE-RUN-DEDUP-001.1 + - AC-OFFICE-RUN-DEDUP-001.2 + - AC-OFFICE-RUN-DEDUP-001.3 + - AC-OFFICE-RUN-DEDUP-001.4 + - AC-OFFICE-RUN-DEDUP-001.5 + - AC-OFFICE-RUN-DEDUP-001.6 + - AC-OFFICE-RUN-DEDUP-001.7 + - AC-OFFICE-RUN-DEDUP-001.8 + - AC-OFFICE-RUN-DEDUP-001.9 + - AC-OFFICE-RUN-DEDUP-002.1 + - AC-OFFICE-RUN-DEDUP-002.2 + - AC-OFFICE-RUN-DEDUP-002.3 + - AC-OFFICE-RUN-DEDUP-002.4 + - AC-OFFICE-RUN-DEDUP-003.1 + - AC-OFFICE-RUN-DEDUP-003.2 + - AC-OFFICE-RUN-DEDUP-003.3 + - AC-OFFICE-RUN-DEDUP-003.4 + - AC-OFFICE-RUN-DEDUP-004.1 + - AC-OFFICE-RUN-DEDUP-004.2 + - AC-OFFICE-RUN-DEDUP-004.3 + - AC-OFFICE-RUN-DEDUP-004.4 + - AC-OFFICE-RUN-DEDUP-004.5 + - AC-OFFICE-RUN-DEDUP-004.6 +system_design: + - ../../specs/office/system-design/run-dedup-generation-01.md + - ../../specs/office/system-design/run-dedup-generation-02.md + - ../../specs/office/system-design/run-dedup-generation-03.md +--- + +# Task 01: Implement Office run deduplication generation identity + +## Scope + +- Add and preserve the task assignment generation across schema migrations. +- Carry the immutable assignee profile ID and generation in task-created events. +- Derive the same generation-aware assignment key in each producer. +- Keep unresolved generations keyless and observable. +- Bound metric label cardinality for agent-supplied reasons. +- Avoid changes to queue coalescing and other deferred deduplication work. + +## Acceptance + +An assignment replay uses one durable generation key. A later assignment to the +same agent gets a new generation and can enqueue work. A task-created event +replayed after its run is claimed remains deduplicated. An unknown metric reason +does not create a new permanent expvar label. + +## Verification + +```bash +cd apps/backend && go test ./internal/runs/service ./internal/office/service \ + ./internal/office/dashboard ./internal/task/service +python3 scripts/lint-spec-files.py --all +``` + +## Results + +Implemented in the contributor change plus fixup commit +`44a5ccc94d8d1fc212b9e839a240d1db2fe16e5f`. The fixup also carries the current +`main` branch and resolves the PR merge conflict. Focused backend tests and +specification lint pass. diff --git a/docs/specs/office/requirements/run-dedup-generation.md b/docs/specs/office/requirements/run-dedup-generation.md new file mode 100644 index 0000000000..e61c23ea06 --- /dev/null +++ b/docs/specs/office/requirements/run-dedup-generation.md @@ -0,0 +1,336 @@ +--- +status: draft +system: office +created: 2026-09-07 +owners: + - kandev +--- + +# Office Run Deduplication — Generation Identity + +## Overview + +Every autonomous wake in Office is enqueued through the shared runs queue with an +`idempotency_key`. The key exists so that a *redelivery* of one occurrence (a +replayed event, a retried handler, two producers reacting to the same fact) does +not launch the same agent twice. + +Several producers instead mint a key that is **permanently unique per (reason, +task, agent)**. Such a key cannot distinguish a redelivery from a genuine repeat +of the same kind of work, so the second legitimate occurrence is suppressed. The +suppression is total and leaves no trace: no run row, no inbox item, no wire +event, and a `Debug`-level log line. + +The canonical case is assignment. `task_assigned::` is stable for +the life of that pair, so both of these are permanent, invisible no-ops: + +- Reassigning a task back to an agent that previously held it (A -> B -> A), + suppressed by the 24-hour lookup. +- Reassigning the same task to the same agent more than 24 hours later, + suppressed by the unbounded `UNIQUE(idempotency_key)` index after the lookup + has already passed. + +The second case is the more damaging of the two: the fast lookup reports no +duplicate, the producer proceeds, and the durable index rejects the insert. The +queue reports that rejection as a successful no-op. + +Two producers already avoid the trap by varying their key with something that +changes per occurrence: the workflow auto-start key carries the step-transition +row id, and the step-entry key carries the entry id and position. Re-running an +epic by moving it through a step therefore works, while re-running it by +re-assignment does not. This capability generalises what those two do into a +contract that every producer must satisfy, and makes the queue's dedup decisions +observable. + +This capability owns the *contract on the key*, and the *observability of the +outcome*. It does not change the 24-hour lookup, the durable unique index, the +coalescing window, or the run lifecycle. + +## Terminology + +- **Occurrence:** one real-world fact that justifies waking an agent — this + assignment, this comment, this decision, this cron fire. Two occurrences of + the same kind on the same (task, agent) are distinct work. Two occurrences are + **distinguishable** when some durable row the producer can read differs between + them; where nothing does, they collapse to one, and every case that collapses is + named under `## Out of scope`. +- **Redelivery:** a second attempt to enqueue a wake for an occurrence that has + already been enqueued — an event replayed, a handler retried, or a second + producer reacting to the same fact. +- **Dedup key:** the value persisted in `runs.idempotency_key` or + `agent_wakeup_requests.idempotency_key`. +- **Generation component:** the part of a dedup key that changes between two + occurrences and stays equal across redeliveries of one occurrence. +- **Windowed dedup hit:** the queue's recent-duplicate lookup matched an + existing row inside its lookback window. +- **Durable dedup hit:** the recent-duplicate lookup found nothing, and the + unbounded unique index then rejected the insert. After this capability ships + this means one of three things: a genuine race between two producers; a producer + that minted a colliding key; or a **legitimate late redelivery** — one occurrence + re-enqueued more than the lookback window after the original, which a + level-triggered reconciler that re-derives the same key on every tick produces + as normal operation rather than as a fault. Only the first two are anomalies. + +## Prior art + +### Our own recorded reasoning (wiki) + +**Searched:** resolved a configured personal wiki vault path and collection +name from the local `obsidian-wiki` config. The leg then **did not run**: +neither `obsidian-wiki` nor `qmd` is on this executor's PATH, no qmd MCP server +is exposed to this session, and the vault directory itself is unreadable here +(a macOS privacy restriction on this process, not a permission-gate denial). +This is a skipped step, not an empty result: the vault is configured and may +well hold relevant prior positions that this specification therefore did not +consult. + +### What other products shipped (saas-kb) + +**Searched:** nothing. The `saas-kb` MCP server and its `search_fsm_docs` tool +are not exposed to this session; the only MCP servers available are `codex` and +`kandev`. Skipped step, not an empty result. + +### Prior art inside this repository + +This leg did run, and it is the one that shaped the design. + +- **Two producers already solve this problem**, which is why the defect is + visible as an asymmetry rather than a total outage. + `officeAutoStartIdempotencyKey` carries the step-transition row id, and the + step-entry key carries the entry id and position. Re-running an epic by + moving it through a step works; re-running it by re-assignment does not. This + capability generalises their shape rather than inventing one. +- **`childrenCompletedIdempotencyKey`** already argues, in a written comment, + the exact principle this specification makes general: a key that is + "permanently unique per (parent, agent)" would wake a parent only for its + first delegation wave, so the key digests the child set instead. It also + already made the judgment that the digest covers *which* children exist + rather than each child's state, to avoid spurious wakes. That reasoning is + adopted here, not re-derived. +- **`task_sessions.route_generation`** is a monotonic counter already in the + tree, bumped on a routing decision and compared for staleness. The assignment + generation this capability needs is the same pattern, so it follows that + precedent rather than introducing a new one. + +**What we are doing differently:** the existing solutions are per-producer, each +one discovered after its own bug report. This capability makes the generation +component a property of the key contract itself (AC-OFFICE-RUN-DEDUP-001.1), +enforced by a test over every producer, so the next producer added cannot +reintroduce a permanent key silently. It also inverts the failure direction: +where `officeAutoStartIdempotencyKey` falls back to a time-derived legacy key +when its generation is missing, this contract falls back to *no key at all*, +because a duplicate wake is recoverable and a suppressed one is not. + +## Requirements + +### REQ-OFFICE-RUN-DEDUP-001: Generation identity in every persisted dedup key + +**Intent:** A dedup key must suppress a redelivery and must not suppress a +repeat. A key built only from durable identifiers that never change (reason, +task, agent) cannot do both, and today it silently chooses the wrong one. + +**User story:** As an Office operator, I want re-assigning a task to an agent +that already held it to actually wake that agent, so that a re-run is not a +silent no-op. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-001.1:** When a producer enqueues a wake with a + non-empty dedup key, that key shall contain a generation component that is + equal for two enqueue attempts describing the same occurrence, and different + for two enqueue attempts describing occurrences that differ in some durable row + the producer can read. Occurrences that no durable row distinguishes collapse + to one key; each such case shall be named under `## Out of scope`. +- **AC-OFFICE-RUN-DEDUP-001.2:** When a task is assigned to an agent, then to a + different agent, then back to the first agent, the system shall wake the first + agent for the third assignment — by inserting a run row, or, when that + assignment lands within the coalescing window of a still-queued run for the + same agent and task, by merging into that row. It shall not report a + deduplicated outcome, which is the only outcome that produces no run at all. +- **AC-OFFICE-RUN-DEDUP-001.3:** When a task is assigned to the same agent a + second time and the earlier assignment's run is older than the queue's + recent-duplicate lookback window, the system shall insert a run row. That + earlier run is necessarily outside the far shorter coalescing window, so the + outcome here is queued and never coalesced. +- **AC-OFFICE-RUN-DEDUP-001.4:** When the same assignment occurrence is + delivered to a producer more than once and the first delivery persisted a run + row bearing its key, the system shall queue exactly one run for it. A delivery + that coalesced persisted no such row, so a redelivery after the coalescing + window queues a second run; see `## Out of scope`. +- **AC-OFFICE-RUN-DEDUP-001.5:** When a wake is triggered by an occurrence that a + durable row identifies, its generation component shall be derived from that row + — its primary key, or a persisted column whose value is fixed for that + occurrence — and shall not be read from the clock at the moment the wake is + produced. A persisted scheduled time is a stored column, not a clock read, and + satisfies this criterion; the processing time at which a producer happens to run + does not. +- **AC-OFFICE-RUN-DEDUP-001.6:** When a wake is triggered by a clock tick that no + durable row identifies, its generation component can be derived from the tick's + scheduled time. A tick whose due time is persisted and claimed on a durable row + before dispatch is identified by that row, and is governed by + AC-OFFICE-RUN-DEDUP-001.5 instead of by this criterion. +- **AC-OFFICE-RUN-DEDUP-001.9:** When a durable row identifies an occurrence but + the producer would have to re-read that row after the occurrence has advanced, + the identifying value shall be captured at the point the occurrence is committed + or claimed and carried to the producer. A producer shall not recover a + generation component by re-reading a row that a later occurrence has since + moved. +- **AC-OFFICE-RUN-DEDUP-001.7:** When a dedup key is supplied by an agent + through a runtime action, the system shall scope that key to the supplying + run so a value the agent reuses across two of its own runs does not suppress + the second. +- **AC-OFFICE-RUN-DEDUP-001.8:** When a run row persisted before this capability + shipped carries a key in the previous format, the system shall leave that row + unchanged and shall not treat it as a duplicate of a key in the new format. + +### REQ-OFFICE-RUN-DEDUP-002: One occurrence yields one key across producers + +**Intent:** Several occurrences are observed by more than one producer. Assignment +is observed by both the task-event subscriber and the reactivity pipeline, and +those two producers converge on one key today. That convergence is what makes a +reassignment wake an agent once rather than twice. Adding a generation component +independently in each producer would break it. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-002.1:** When two producers that each derive a dedup key + enqueue a wake for the same occurrence with the same reason, task, and agent, + they shall derive an identical key. A producer enqueueing that occurrence with + no key is governed by REQ-OFFICE-RUN-DEDUP-003 and does not violate this + criterion. +- **AC-OFFICE-RUN-DEDUP-002.2:** When an assignment is observed by both the + task-event subscriber and the reactivity pipeline, the system shall queue + exactly one run. +- **AC-OFFICE-RUN-DEDUP-002.3:** When two producers attempt to enqueue the same + key concurrently, each shall either persist a run or observe a suppression + outcome — deduplicated, or coalesced when a coalescible run was queued for that + agent, reason and task at its coalesce check — without returning an error to + its caller. At most one may persist, and where such a run was queued for both, + neither does. No outcome adds a second row bearing the key; none is a failure. +- **AC-OFFICE-RUN-DEDUP-002.4:** When two producers attempt to enqueue the same + key concurrently, at most one row shall ever bear that key — never two — and + neither producer shall abort. Whether it is one row or none depends on whether + a coalescible run was queued at each producer's coalesce check, not on which + insert committed first; nothing may branch on which outcome a producer saw. + +### REQ-OFFICE-RUN-DEDUP-003: Never suppress silently when the generation is unknown + +**Intent:** A generation component can be unresolvable — a legacy event without +the field, an occurrence row not yet committed, a read that failed. Falling back +to a permanently-unique key in that case reintroduces exactly the defect this +capability removes, and does so in the paths that are hardest to observe. A +duplicate wake costs one redundant agent turn; a suppressed wake costs the work. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-003.1:** When a producer cannot resolve a generation + component for an occurrence, it shall not enqueue the wake under a key whose + only components are the reason, task, and agent. +- **AC-OFFICE-RUN-DEDUP-003.2:** When a producer cannot resolve a generation + component, it shall enqueue the wake with no dedup key, accepting a possible + duplicate run rather than a possible suppression. +- **AC-OFFICE-RUN-DEDUP-003.3:** When a producer enqueues a wake with no dedup + key, the system shall record that event in telemetry, attributed to the run + reason and discriminated by whether the key was omitted because a generation + was unresolvable or because that reason is keyless by design, so that a + resolution failure is countable on its own. +- **AC-OFFICE-RUN-DEDUP-003.4:** When a wake is enqueued with an empty dedup + key, the system shall not apply idempotency suppression to it. + +### REQ-OFFICE-RUN-DEDUP-004: Deduplication is observable + +**Intent:** A dedup is a decision not to do requested work. Today it is reported +at `Debug`, is not counted, and is reported to the office producer path as an +undifferentiated success. An operator asking "why did nothing happen when I +reassigned this" has nothing to read. + +**User story:** As an Office operator, I want to see that a wake was suppressed +and why, so that a no-op is diagnosable without attaching a debugger. + +#### Acceptance criteria + +- **AC-OFFICE-RUN-DEDUP-004.1:** When the queue suppresses a wake, it shall + increment a counter exposed through the process metrics endpoint, labelled by + run reason and by whether the suppression was a windowed or a durable dedup + hit. +- **AC-OFFICE-RUN-DEDUP-004.2:** When the queue suppresses a wake through the + durable unique index after the recent-duplicate lookup reported no duplicate, + it shall emit a structured log record at a level an operator sees by default, + carrying the dedup key, the run reason, and the resolved agent. +- **AC-OFFICE-RUN-DEDUP-004.3:** When the queue suppresses a wake through the + recent-duplicate lookup, it shall emit a structured log record carrying the + dedup key and the run reason. +- **AC-OFFICE-RUN-DEDUP-004.4:** When a caller enqueues a wake through the + **run queue**, the run queue shall report which of queued, deduplicated, or + coalesced occurred, and shall report it through every run-queue enqueue + interface Office producers use. This criterion does not reach the + wakeup-request table, which has neither a windowed lookup nor coalescing and so + has no third outcome to report; AC-OFFICE-RUN-DEDUP-004.6 covers it in full. +- **AC-OFFICE-RUN-DEDUP-004.5:** When the **run queue** reports a deduplicated or + coalesced outcome, it shall not return an error, and the caller shall not treat + the outcome as a failure. The wakeup-request table is exempt from the first + clause only: it reports its one suppression as a sentinel error, and a caller + receiving that sentinel shall still not treat it as a failure. +- **AC-OFFICE-RUN-DEDUP-004.6:** When a wakeup request is rejected by the wakeup + table's unique index, the system shall increment the same counter family with + the wakeup queue identified as the source. + +## Out of scope + +- **Rewriting persisted keys on existing rows.** A run row is a historical + record of a wake that happened. Backfilling it to the new key format would + restate history and could itself collide with a key a live producer is about + to mint. AC-OFFICE-RUN-DEDUP-001.8 makes the old rows inert instead. +- **Changing the 24-hour recent-duplicate lookback window.** The window is a + performance affordance on the fast path; the unbounded unique index is what + actually enforces identity. Neither is the defect, and both are unchanged. +- **Changing the 5-second coalescing window or which reasons coalesce.** + Coalescing merges wakes that are already agreed to be redundant within seconds; + it is a separate mechanism from identity-based dedup and is out of scope here. + It is deliberately *not* a suppression: a coalesced wake merges into a run row + that is still queued for that agent and task, carrying the newer payload, so the + agent is woken. Only a deduplicated outcome produces no run at all. That is why + AC-OFFICE-RUN-DEDUP-001.2 accepts a coalesced outcome and forbids only a + deduplicated one, and why the coalescing predicate is left exactly as it stands. + Two distinct assignment generations landing inside the same five seconds + therefore yield one wake, not two: the operator asked twice in five seconds and + gets one launch carrying the second request's payload. A wake that coalesces + also leaves no row bearing its own key, so a redelivery of that occurrence + after the window queues a second run. AC-OFFICE-RUN-DEDUP-003.2 elects a + duplicate over a suppression, so that is the accepted direction; closing it + would mean keying the merged row, which changes the coalescing mechanism. + +- **Distinguishing a blocker set that resolves, un-resolves and re-resolves + identically.** `task_blockers_resolved` identifies its occurrence by digesting + the blocker id set, and blocker edges survive resolution — the readiness check + reads each blocker task's state, not the edge's existence. So if every blocker + resolves, one is reopened, and it then completes again with no blocker added or + removed, the second wave digests to the same key as the first and is suppressed + once past the lookback window. This is the one case + AC-OFFICE-RUN-DEDUP-001.1 requires be named here: no durable row distinguishes + the two waves. Both candidates were rejected — the resolving task's id is + identical in both waves, because the readiness check fires only on the last + blocker to complete; and a blocker task's `updated_at` is exactly the + time-derived shape AC-OFFICE-RUN-DEDUP-001.5 disallows, besides re-introducing + the spurious wakes `childrenCompletedIdempotencyKey` digests the child *set* to + avoid. A durable per-wave row would close this; creating one is separate work. +- **A user-facing surface for a suppressed wake.** Deduplication is a + high-frequency, expected outcome on the engine redelivery paths; an inbox + item or a task-timeline entry per occurrence would be noise that trains + operators to ignore the surface. REQ-OFFICE-RUN-DEDUP-004 places the + observability floor at operator telemetry and logs. If a user-visible surface + is wanted later, the counter added here is the evidence needed to size it. +- **Retiring the duplicate `RunReason*` constant blocks** in + `internal/office/scheduler` and `internal/office/service`. The audit crosses + them, but consolidating them is an unrelated refactor. +- **The order in which several wakes from one mutation are enqueued.** A single + task mutation can fan out to an assignee and several reviewers. This + capability changes each wake's key, not the sequence the reactivity pipeline + emits them in, and defines no ordering or tiebreak over that fan-out. +- **Changing the `heartbeat` wake source's key.** `fireHeartbeat` still mints a + live engine operation id (`heartbeat:::`) that the + engine expands into a persisted key, so heartbeat *is* a producer and is + covered by the audit. It is already generational on its tick time, which + AC-OFFICE-RUN-DEDUP-001.6 permits, so no change is required to it. Retiring + the source itself, and the duplicate `heartbeat` constants left behind, are + separate work. diff --git a/docs/specs/office/system-design/run-dedup-generation-01.md b/docs/specs/office/system-design/run-dedup-generation-01.md new file mode 100644 index 0000000000..dde2341fd3 --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-01.md @@ -0,0 +1,568 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 +--- + +# Office Run Deduplication — Generation Identity System Design Part 1: the key contract + +Part 1 covers what goes *into* a dedup key: the producer audit, the assignment +generation and how it is carried, the per-producer generation sources, producer +convergence, and the keyless path. Observability, failure and recovery, and the +test plan are in +[Part 2](run-dedup-generation-02.md). The full producer inventory and the exact Go +signatures this capability adds or widens are in +[Part 3](run-dedup-generation-03.md). + +## Purpose and boundaries + +This design changes what goes *into* a dedup key and what the queue *reports* +when it acts on one. It changes none of the queue's suppression mechanisms: +`CheckIdempotencyKey` (the 24-hour recent-duplicate lookup), `idx_run_idempotency` +and `idx_wakeup_idempotency` (the unbounded partial unique indexes), and +`CoalesceRun` / `shouldCoalesceRun` (the 5-second merge) are all unchanged. The +last is deliberate; see [Concurrency](#concurrency). + +Adjacent contracts read and constrained but not owned: + +- `internal/runs/service` — `QueueRun`, `QueueOutcome`, + `errIdempotencyKeyConflict`. +- `internal/workflow/engine` — `idempotencyKey` and its `EntryID` / + `OperationID` precedence; already generational, unchanged. +- `internal/orchestrator/event_handlers_workflow.go` — + `officeAutoStartIdempotencyKey`; generational, unchanged apart from its legacy + fallback. +- `internal/task/repository/sqlite` — the `tasks` table and the + `workflow_step_participants` runner seat. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-OFFICE-RUN-DEDUP-001` | [Generation sources per producer](#generation-sources-per-producer), [Assignment generation](#assignment-generation) | +| `REQ-OFFICE-RUN-DEDUP-002` | [Convergent producers](#convergent-producers) | +| `REQ-OFFICE-RUN-DEDUP-003` | [Unresolvable generation](#unresolvable-generation), [Keyless causes per producer](run-dedup-generation-03.md#keyless-causes-per-producer) (Part 3) | +| `REQ-OFFICE-RUN-DEDUP-004` | [Observability](run-dedup-generation-02.md#observability) (Part 2) | + +## Producer audit + +The full inventory — every producer that reaches a run queue, including the four +that pass no key at all — moved to +[Part 3](run-dedup-generation-03.md#producer-audit) when it grew a fourth table. +Two entries are load-bearing here and are repeated rather than chased: +`queueTaskAssignedRun` and `QueueRunCtx`'s default are the non-generational +producers this design replaces, and `QueueRunCtx`'s default reaches every +reactivity wake except children-completed. + +## Assignment generation + +Assignment is the only occurrence in the audit with no durable identity to +borrow. `UpdateTaskAssignee` updates the runner seat in place, so +`workflow_step_participants.id` does not change across a reassignment, and a +plain reassignment writes no `task_step_transitions` row. + +Add a monotonic counter to the task: + +```text +tasks.assignment_generation INTEGER NOT NULL DEFAULT 0 +``` + +Applied through the existing additive migration mechanism +(`internal/task/repository/sqlite/base_migrations.go`), mirroring +`task_sessions.route_generation`, the same pattern already in the tree. + +### The two bump sites + +The runner seat has **two** writers, not one, and both bump: + +| Writer | Reached from | Bump | +| --- | --- | --- | +| `office/repository/sqlite/tasks.go` `UpdateTaskAssignee` | every reassignment and unassignment | increment, inside its existing transaction | +| `task/repository/sqlite/task.go` `insertTaskTx` -> `upsertRunnerInTx` | `CreateTask` | insert `assignment_generation = 1` **under exactly the guard that writes the runner row** (`AssigneeAgentProfileID != "" && WorkflowStepID != ""`); any other create inserts `0` | + +Both already run in a transaction. Each bumps on **every committed assignment +write**, including one setting the agent that already held the seat. That +unconditional rule is load-bearing: a guard of "only when the resolved runner +changes" would leave a repeat assignment minting an identical key, the permanent +no-op AC-001.3 forbids. Re-assigning a task to the agent that already holds it is +a real occurrence — the operator is asking for the work again. + +An unassignment (`assigneeID == ""`, deleting the runner row) is an assignment +write and bumps too, so A -> unassigned -> A yields three distinct generations +and the final assignment is not suppressed by the first. + +The occurrence's generation is stamped into: + +```text +task_assigned::: +``` + +A different shape from the pre-existing `task_assigned::` rows, so +AC-001.8 holds without a backfill: an old row can never equal a new key. The +counter is task-level, not per-agent — A -> B -> A produces generations 1, 2, 3, +so the third assignment's key differs from the first's. + +### Four writers of the same row that must NOT bump + +`task/repository/sqlite/task.go` also calls `syncRunnerInTx` from `updateTaskTx`, +`UpdateTaskIfWorkflowStepHasCapacity`, +`PromoteQueuedTaskIfWorkflowStepHasCapacity` and +`RestoreTaskMessageRollbackIfSessionState`, each passing +`task.AssigneeAgentProfileID`. They write the same `role='runner'` row and are +provably inert as assignments: the field is **not a stored column** — `tasks` has +had none since ADR 0005 Wave F, every read derives it through `runnerProjection` +from the seat — so each of these paths writes back the value it just read. + +No production caller can put a *new* agent into the seat through these four. +`task/service.UpdateTaskRequest` and `task/dto.UpdateTaskRequest` have no +`AssigneeAgentProfileID`, only `AssigneeUserID`, which its own doc comment +records as independent of the agent assignee. **Package-qualify that claim +wherever it is cited:** `office/dashboard.UpdateTaskRequest` is a different type +that *does* carry the field, but it routes to +`DashboardService.SetTaskAssigneeAsAgent` -> `UpdateTaskAssignee` — bump site one +— and never reaches `syncRunnerInTx`. Every production site supplying a new value +reaches either that path or `CreateTaskRequest` -> `insertTaskTx`, the second +bump site (`task/service/service_tasks.go`, `service_child_task.go` inheriting +the parent's, `backendapp/adapters_office.go`, `mcp/handlers/handlers.go`). Every +other non-test mention of the field is a read, a comparison, or a DTO projection. + +Two of the four are step moves, which re-key the seat to a new `step_id`. That is +a seat migration, not an assignment; see +[What the counter does not cover](#what-the-counter-does-not-cover). + +### Carrying the generation + +**It is captured inside the assigning transaction and carried to the producers. +A producer never re-reads it.** The `UpdateTaskAssignee` bump site reads the new +value back inside its existing transaction, before `Commit`, and returns it; the +caller passes it to each producer explicitly. The create-path bump site returns +nothing and changes no signature: its generation is a constant the runner guard +already fixes (Part 3). On the dashboard path that caller is +`DashboardService.SetTaskAssigneeAsAgent` (`office/dashboard/service_tasks.go`). +Note that `office/service` declares a same-named `SetTaskAssigneeAsAgent` and a +`SetTaskAssignee` which also reach `UpdateTaskAssignee` but publish nothing and +run no reactivity; they are not producers, may discard the value, and neither +signature changes. Two channel-setup sites reach it as well and are likewise not +producers. + +`UpdateTaskAssignee`'s widened signature, all five of its non-test call sites and +what each does with the value, and the behaviour when the read-back itself fails, +are specified in +[Part 3](run-dedup-generation-03.md#updatetaskassignee-and-the-bump-sites). The +wire encoding of the event field is specified in +[Part 3](run-dedup-generation-03.md#the-event-payload-field); `0` is a legal +generation and is never a sentinel for absent. + +Two carriers, both **new fields this design adds** (neither exists today): + +- `TaskReactivityChange.AssignmentGeneration` (`office/dashboard/service.go`), + copied by `convertChangeToMutation` (`office/scheduler/dashboard_adapter.go`) + onto a matching `TaskMutation.AssignmentGeneration` + (`office/scheduler/reactivity.go`). Both `*int64`; nil means "not supplied". No + struct is needed — the mutation already carries the assignee as the bare + `NewAssigneeID *string` and the generation follows that shape. +- `assignment_generation` on the `task.created` / `task.updated` payload, decoded + by `TaskUpdatedData` (`office/service/event_subscribers.go`) and published by + `task/service/service_events.go`. + +The field exists because the read is unsound: a producer reading +`assignment_generation` after the commit reads *the task's current* generation, +not *its occurrence's*. Given A -> B (1), B -> A (2), A -> B (3), a producer +still on occurrence 1 reads 3 and mints occurrence 3's key byte for byte; one is +then suppressed by the unique index, the direction REQ-OFFICE-RUN-DEDUP-003 +forbids. A producer handed no generation takes the keyless path of +[Unresolvable generation](#unresolvable-generation) and does **not** read the +task row to recover one. + +**The one producer that already re-reads stays keyless.** `handleTaskCreated` +calls `queueTaskAssignedRun` with `fallbackToStoredRunner=true`, so when the +`task.created` payload omits the assignee that producer recovers it through +`GetTaskExecutionFields` — a fresh read after the commit. Do **not** extend +`TaskExecutionFields` with the generation; that would make the unsound read above +a supported path. When the event carries no `assignment_generation` the producer +goes keyless with `cause=unresolved` even though it did recover an agent id: it +has an agent to wake and no occurrence identity to name. + +### The reactivity gate + +A new key alone is not sufficient. `ApplyTaskMutation` +(`office/scheduler/reactivity.go`) reaches `reactToAssigneeChange` only when the +agent differs — `change.NewAssigneeID != nil && *change.NewAssigneeID != +task.AssigneeAgentProfileID` — and `DashboardReactivityAdapter` substitutes the +*pre-update* assignee into the snapshot. On a repeat assignment to the agent +already holding the seat both sides are equal, so **no `task_assigned` wake is +produced at all** and AC-001.3 fails before any key is built. + +The gate drops the equality test and fires whenever `change.NewAssigneeID` is +non-nil. That is safe by the same call-site argument the bump sites use: of the +three `TaskReactivityChange` constructors in `office/dashboard/service_tasks.go`, +only `runReactivityForAssigneeChange` sets `NewAssigneeID`; the status and +comment constructors leave it nil, so a non-nil value already means "this +mutation is an assignment". An unassignment (non-nil but `""`) needs no new +guard: the pipeline's `queue` closure already returns early on an empty agent id. + +**The interrupt needs a guard of its own, and does not have one today.** +`reactToAssigneeChange` also sets `res.InterruptSessionID`, hard-cancelling the +seat's previous session — and it does so on `task.AssigneeAgentProfileID != ""` +alone, never comparing `newAssigneeID`. That was safe only because the caller's +equality gate never delivered it a same-agent repeat. Removing that gate exposes +it, and because the adapter substitutes the pre-update assignee, on an A -> A +repeat the previous assignee is non-empty and the interrupt would cancel the +agent's own in-flight run. Move the comparison inside: set `InterruptSessionID` +only when `task.AssigneeAgentProfileID != ""` **and** `newAssigneeID != +task.AssigneeAgentProfileID`. Only the wake becomes unconditional — this +capability changes dedup identity, not session lifecycle. + +### What the counter does not cover + +Three cases a builder would otherwise guess at. Ordering note: +`assignment_generation` is compared only for equality, never `>`/`<`. Nothing +reads it as a sequence, so no ordering or tiebreak over generations is defined. + +**Two concurrent reassignments of one task.** Each bump site increments inside +its own transaction and writes are serialised, so the commits take distinct +generations N and N+1 — two keys, two runs, one per assignment. No extra locking. + +**A producer handed no generation.** Because the value is carried, not read, the +only "stale value" case left is *absent* value — an event minted before this +shipped, or a nil `AssignmentGeneration`. It goes keyless. + +**A step move.** It re-keys the seat row to the new `step_id`, and because +`runnerProjection` falls back to `workflow_steps.agent_profile_id` it can both +change the resolved runner with no participant write and materialise a per-task +row where none existed. Neither is an assignment: the occurrence is already +covered by `officeAutoStartIdempotencyKey`'s `stepTransitionID`. Do not bump to +compensate — that would re-wake an agent auto-start has already woken. + +## Generation sources per producer + +The replacement for each non-generational producer. + +| Reason | New key | Generation source | +| --- | --- | --- | +| `task_assigned` | `task_assigned:::` | `tasks.assignment_generation` | +| `task_comment` (reactivity, assignee) | `task_comment::` | comment row id, plus the recipient | +| `task_mentioned` | `task_mentioned::` | comment row id, plus the recipient | +| `task_blockers_resolved` | `task_blockers_resolved:::` | blocker-set digest | +| `blockers_resolved` operation id | `blockers_resolved::` | same digest, same sort rule | +| `task_unblocked` | none (keyless) | no durable occurrence row | +| `task_reopened` | none (keyless) | no durable occurrence row | +| `task_reopened_via_comment` | `task_reopened_via_comment::` | comment row id | +| `task_review_requested` | none (keyless) | no durable occurrence row | + +`task_mentioned` must carry the recipient: the mention wake and the assignee +comment wake describe the same comment but address different agents, and +including the agent keeps the fan-out independent when a comment mentions several +agents and one is later re-resolved. `task_comment` carries it for the same +reason — see [Convergent producers](#convergent-producers). + +**AC-001.8 for the reshaped comment keys.** Only `task_assigned` gains a segment. +`task_comment`, `task_mentioned` and `task_reopened_via_comment` keep three +segments and merely swap the task id for a comment id, so the "different shape" +argument above does not carry over; all three hold instead because comment ids +and task ids come from disjoint uuid-keyed tables, so +`::` can never equal `::` +for a real pair. `task_comment`'s superseded form is the +`{reason}:{taskID}:{agentID}` default this design removes from `QueueRunCtx`. + +`task_blockers_resolved` uses a set digest rather than the resolving blocker's +id, matching `childrenCompletedIdempotencyKey`'s already-argued shape: a wave is +identified by which blockers were in play, so a second wave that adds blockers +gets a distinct key. A wave re-resolving the *same* set digests identically and +is suppressed — the one collapse the requirements name under `## Out of scope`. +No producer may improvise around it. + +Both blocker producers must sort the ids **lexicographically by +`blocker_task_id`** before digesting, and must not inherit `ListTaskBlockers`'s +ordering. That query is `ORDER BY created_at`, which is not unique: two blockers +added in one timestamp tick have undefined relative order, so two reads of one +unchanged set can digest differently and mint two keys for one occurrence. +`childrenCompletedIdempotencyKey` is safe only because `ListChildStates` orders +by `id`; the blocker path must impose its own. + +Sorting is necessary but not sufficient: the two blocker producers live in +different packages and AC-002.1 requires them to derive an identical key, so they +call **one shared builder** rather than each implementing the same convention. +The delimiter, the digest encoding and the empty-set rule are binding and are +specified in +[Part 3](run-dedup-generation-03.md#the-shared-key-builders). + +Three status-driven reasons are keyless. `ApplyTaskMutation` is called +synchronously after the mutation commits, never from the event bus, so a status +transition has no redelivery path and the key guarded nothing. Two guards +suffice: the pipeline's per-mutation `seen` map (`{agentID}:{taskID}:{reason}`) +stops a double-queue inside one mutation, and the 5-second coalescing window +absorbs a rapid repeat across mutations. A status-transition ledger is not +justified by the defect, and AC-003.2 elects a duplicate over a suppression. + +`QueueRunCtx`'s `idempotencyKey == ""` default is **removed**, not replaced. Its +existence is what silently gave every future caller a permanent key. After this +change a `RunContext` with no `IdempotencyKey` is enqueued keyless, the safe +direction. Producers that want dedup set the field explicitly, and the +`RunContext.IdempotencyKey` doc comment records that empty now means "no dedup" +rather than "derive one for me". + +### Agent-supplied keys + +`SpawnAgentRun` prefixes a **non-empty** agent value with the calling run's id: + +```text +agent:: +``` + +A retry of the same run reuses the run id and still dedupes, which is the case +the agent is protecting against; a later run gets a different prefix and is not +suppressed. With no caller run id the request is enqueued keyless. + +An **empty** `input.IdempotencyKey` is not prefixed — it stays empty and goes +keyless. Prefixing it would collapse every no-dedup-intent `SpawnAgentRun` inside +one run onto the single key `agent::`, suppressing the second such +call: the run-scoped form of the exact defect this capability removes. + +### Routine keys + +Every routine fire commits a `RoutineRun` row (`dispatchRoutineRun` -> +`CreateRoutineRun`) before `materialiseLightweightRoutineRun` builds the key, so a +durable row is always available. But **that row identifies the dispatch attempt, +not always the occurrence**, and which is which depends on the source. The +discriminator is `RoutineRun.Source` (`shared.RoutineSourceCron` = `"cron"`, +`"manual"`, `"webhook"`), which `buildRoutineIdempotencyKey`'s current +`(routineID, triggerID string, startedAt *time.Time)` signature does not receive +and must gain. + +| `run.Source` | New key | Generation source | +| --- | --- | --- | +| `cron` | `routine:::tick:` | the scheduled tick claimed off `office_routine_triggers.next_run_at` | +| `manual`, `webhook` | `routine::run:` | `RoutineRun.ID` | + +**Why cron does not use `RoutineRun.ID`.** `processCronTrigger` opens with +`ClaimTrigger`, a compare-and-swap (`SET next_run_at = NULL WHERE id = ? AND +next_run_at = ?`). Exactly one caller wins a given scheduled tick and the loser +returns before any row is written, so there is one `RoutineRun` per *slot*: the +row is the attempt, the slot is the occurrence. Keying on `run.ID` would give a +redelivery of one slot a fresh key, the direction AC-001.4 forbids. Catch-up does +not change this - `computeRoutineMissed` collapses N missed ticks into **one** +fire carrying `missedTicks`, so a catch-up is still one occurrence. + +**Why manual and webhook do use `RoutineRun.ID`.** Neither claims a slot. Two +manual fires are two distinct intents, an operator asking twice, and +`RoutineRun.ID` is their only durable distinguishing identity. This is also the +collision the audit table names: both carry `triggerID == ""`, so today they +collapse onto one `routine::` key. + +The consequence, stated rather than left to be inferred: a **retried** manual or +webhook request commits a second `RoutineRun` and therefore wakes twice, because +nothing upstream of `CreateRoutineRun` deduplicates the request itself. That is +the direction AC-003.2 elects, a duplicate wake over a suppressed one, and +narrowing it would need a request-level identity this capability does not +introduce. Two concurrent manual fires behave the same way: two rows, two keys, +two wakes, no locking. Cron is the only source where the CAS makes redelivery +converge. + +**The claimed tick must be CARRIED, not re-read (AC-001.9).** `processCronTrigger` +calls `UpdateTriggerNextRun(advanceTo)` *before* dispatching, so by the time the +key is built the trigger row's `next_run_at` already names the **next** slot; +only the in-memory `*trigger.NextRunAt` still holds the claimed value. Thread it +from `processCronTrigger` through `DispatchRoutineRunWithMissed` -> +`dispatchRoutineRun` -> `materialiseRoutineRun` -> +`materialiseLightweightRoutineRun`. A producer that re-reads the trigger row +mints the next occurrence's key, the same unsound-read failure +[Carrying the generation](#carrying-the-generation) rules out for assignment. + +`trigger` reaches `dispatchRoutineRun` and stops there, so the claimed tick +travels the last two hops as its own `*time.Time` parameter, exactly as +`missedTicks` already does. `materialiseRoutineRun` branches on `tmpl.Title`; +`materialiseHeavyRoutineRun` ignores the parameter, the treatment `missedTicks` +already receives on that branch, because the heavy path creates a real task and +builds no wakeup key. + +**Nil and boundary.** `processCronTrigger` returns early when +`trigger.NextRunAt == nil`, so a live cron dispatch always has a claimed tick. The +exported `DispatchRoutineRunWithMissed` can still be called with +`source == "cron"` and a nil trigger or nil claimed tick; that enqueues +**keyless** with `cause=unresolved` (see +[Unresolvable generation](#unresolvable-generation)) rather than falling back to a +minute. Unix seconds suffice because cron granularity is one minute at finest, so +two distinct slots of one trigger can never share a value. + +A `RoutineRun.Source` that is neither `cron` nor `manual`/`webhook` matches no +branch of the table above. It enqueues **keyless** with `cause=unresolved`, the +same direction as a cron dispatch with no claimed tick: an unrecognised source is +a source whose occurrence this design cannot name, and inventing a key for it +would be guessing at an identity rather than reading one. + +`unix_minute` is removed from both branches; no fire keeps a processing-clock +key. + +## Convergent producers + +Two producers observe an assignment on the paths this section governs: +`queueTaskAssignedRun` +(`office/service/event_subscribers.go`, from `task.created` / `task.updated`) and +`reactToAssigneeChange` (`office/scheduler/reactivity.go`, through +`ApplyTaskMutation`). They converge on one key today by accident of format — +`fmt.Sprintf("task_assigned:%s:%s", …)` and `QueueRunCtx`'s +`fmt.Sprintf("%s:%s:%s", reason, taskID, agentID)` produce the same string — and +that convergence must survive (AC-002.2). + +**They are driven by different entry paths, and no decision here may assume one +call sequence reaches both.** Verified: + +| Entry path | Publishes / calls | Producer that fires | +| --- | --- | --- | +| Office dashboard `SetTaskAssigneeAsAgent` | `publishTaskUpdated` -> `events.OfficeTaskUpdated`, then `runReactivityForAssigneeChange` | reactivity **only** | +| Task creation with an assignee | `events.TaskCreated` | `queueTaskAssignedRun` **only** | +| Task service update | `events.TaskUpdated` | `queueTaskAssignedRun`, but see below | + +`handleTaskUpdated` subscribes to `events.TaskUpdated`; `publishTaskUpdated` +emits `events.OfficeTaskUpdated`. Different subjects, so the dashboard assignment +path does **not** reach `queueTaskAssignedRun`. + +**The `task.updated` row cannot carry an assignment *change*.** As +[Four writers](#four-writers-of-the-same-row-that-must-not-bump) establishes, no +production update path reaching `syncRunnerInTx` can put a new agent into the +seat. So `queueTaskAssignedRun`'s live assignment occurrence is task +**creation**; `handleTaskUpdated` stays subscribed as a redelivery and defensive +path, and if it fires without a carried generation it goes keyless with +`cause=unresolved`. Nothing here may assume a task-service update produces an +assignment occurrence. + +**A third producer exists on the onboarding path and does not converge.** +`office/onboarding` creates a task with an assignee and enqueues its own +`task_assigned` run, while the same creation's `task.created` event drives +`queueTaskAssignedRun`. The onboarding producer is never handed the generation, +so it goes keyless rather than minting a divergent key. AC-002.1 binds only +producers that each derive a key, so a keyless third producer owes no +convergence; see +[Part 3](run-dedup-generation-03.md#onboarding-is-a-third-task_assigned-producer). + +AC-002.2 therefore requires something narrower than "both always fire": *when one +assignment occurrence is observed by both producers — a redelivery, or a future +path that drives both — they must derive the same key.* Carrying the generation +delivers that: both stamp the value the assigning transaction committed rather +than each re-deriving one. Part 2's test case drives both producers directly +rather than through a dashboard call, because on that path only one fires. + +To make the convergence structural rather than coincidental, both producers call +one exported builder instead of formatting the string themselves. Place it beside +`commentkeys` as a sibling package so the office service, the office scheduler +and the orchestrator all reach it without an import cycle; +`officeAutoStartIdempotencyKey` keeps its own shape, a step-driven auto-start +being a different occurrence from a reassignment. + +**The comment wakes do not converge, and do not need to.** `queueCommentRun` +passes `commentkeys.TaskComment(commentID)` to `dispatchEngineTrigger` as an +*operation id*, which the engine expands to +`::::` before persisting, so the shapes can +never be equal. They are already mutually exclusive by construction: +`runReactivityForComment` (`office/dashboard/service_tasks.go` — **not** +`handleCommentCreated`, which never touches the field) sets +`SkipAssigneeCommentWake` from `dispatchCommentEngineTrigger`'s return, and the +adapter carries it across as `SkipAssigneeWake`, so when the engine handled the +comment the reactivity assignee wake is skipped. One comment, one assignee wake, +enforced by a branch rather than a key collision. The engine is unchanged. + +Consequently `task_comment` **does** carry the recipient. The occurrence is +"(this comment, this recipient)", not "this comment": one comment fans out to an +assignee and every mentioned agent, and a bare `task_comment:` would +collide two recipients' wakes and suppress one. The `task_comment:` prefix is +preserved, so `shouldCoalesceRun`'s prefix test still excludes these from +coalescing, and `commentkeys` already parses salted keys (`CommentIDFromKey` cuts +at the first colon), so the extra segment needs no parser change. + +### Concurrency + +Two producers racing on one key resolve without locking, and this design adds +none. Both call `CheckIdempotencyKey` and may see no duplicate; both proceed; +`idx_run_idempotency` admits one; the loser's `CreateRun` returns a unique +violation, mapped to `errIdempotencyKeyConflict` and reported as +`QueueOutcomeDeduped` with a nil error. + +That last step is today true **only** in `runs/service`. `queueRunInline` and +`SchedulerService.QueueRun` return the raw violation as an error, so a race on +the reactivity path surfaces as a failure rather than a dedupe. Both gain the +mapping via the shared helper (see +[Counters](run-dedup-generation-02.md#counters)); without it AC-002.3 does not +hold on the path this capability fixes. + +**The loser can observe either suppression outcome, and AC-002.3 accepts both.** +The queue checks the key, *then* coalesces, *then* inserts. A loser whose lookup +ran after the winner's insert sees `Deduped`; one whose lookup ran before that +insert and whose coalesce check ran after it sees `Coalesced`, because the +winner's row is queued for the same agent, reason and task inside the 5-second +window. Both leave exactly one run row and neither returns an error. A test for +AC-002.3 must accept either and must not pin `Deduped`. Note also that +`SchedulerService.QueueRun` coalesces unconditionally where `runs/service` first +consults `shouldCoalesceRun`; neither is changed, and this outcome contract is +written to hold for both. + +Neither caller aborts, and the observable result is the same whichever insert +committed first (AC-002.4): at most one row ever bears that key, and none does +when a coalescible run was queued for that agent, reason and task at each +producer's coalesce check, because a coalesced enqueue never reaches +`insertRun`. See +[Key durability](run-dedup-generation-02.md#key-durability), which owns that +boundary. No ordering or tiebreak is defined: the losing path has no output to +order. The run `id` is a fresh UUID per attempt, so nothing may depend on which +uuid won. + +## Unresolvable generation + +A producer that cannot resolve its generation component enqueues with an empty +key. It does **not** fall back to `::` (AC-003.1), because +that is the defect. + +Concretely: an event carrying no `assignment_generation`, or a nil +`AssignmentGeneration` on a mutation; `officeAutoStartIdempotencyKey`'s zero +`stepTransitionID`; `SpawnAgentRun` with no caller run id; a reactivity comment +wake whose `MutationComment` is nil; and a `source == "cron"` routine dispatch +reaching [Routine keys](#routine-keys) with no claimed tick carried. + +`SpawnAgentRun` with an **empty agent-supplied key** is deliberately absent from +that list: nothing failed to resolve, the agent expressed no dedup intent, so it +is `by_design`. Part 3's +[keyless causes](run-dedup-generation-03.md#keyless-causes-per-producer) assigns +every producer its side and agrees. + +An empty key already means "no dedup" throughout the queue: `QueueRun` skips +`CheckIdempotencyKey`, `insertRun` writes a NULL `idempotency_key`, and both +unique indexes are partial and ignore NULL (AC-003.4). Coalescing still applies — +`shouldCoalesceRun` only excludes the `task_comment:` prefix. + +**Where the keyless counter is incremented, and how `cause` reaches it.** The +cause is a property of the producer's *decision*, not of the request: by the time +an enqueue reaches a queue, "I never needed a key" and "I tried to build one and +failed" are both an empty string, and `RunContext` carries nothing separating +them. So the queue does **not** report keyless enqueues and nothing infers a cause +from an empty key. Instead `internal/runs/service` exports a reporter beside the +suppression helper of [Counters](run-dedup-generation-02.md#counters): + +```text +runsservice.ReportKeylessEnqueue(reason string, cause KeylessCause, detail string) +``` + +with `KeylessCauseUnresolved` and `KeylessCauseByDesign`. Each producer calls it +where it decides to go keyless, naming its own cause, immediately before +enqueuing. `detail` names **what failed to resolve** — the short constant +[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns each +producer — so the `Info` record [Part 2](run-dedup-generation-02.md#logs) +requires for `cause=unresolved` separates two producers that go keyless under one +reason. It is `""` for `cause=by_design`, which is not logged. Counters stay declared once in `internal/runs/service` — no parallel +map — and the import direction already holds: `runs/service` imports +`office/models` only, so `office/service`, `office/scheduler` and +`office/routines` can all call it without a cycle. + +`cause=unresolved` is every site enumerated in *this* section. `cause=by_design` +is the three status reasons and any other producer that never had an occurrence to +name. **The earlier rule — that any `RunContext` setting no key is `by_design` — +is withdrawn**: with `QueueRunCtx`'s default removed a producer that failed to +resolve also arrives with no key, so that rule counted precisely the failures +AC-003.3 exists to isolate as normal traffic. A producer reaching a queue keyless +without having called the reporter is a bug; the key-format table test in +[Part 2](run-dedup-generation-02.md#testing) is where it is caught. + +The counters themselves are defined in +[Part 2](run-dedup-generation-02.md#counters). diff --git a/docs/specs/office/system-design/run-dedup-generation-02.md b/docs/specs/office/system-design/run-dedup-generation-02.md new file mode 100644 index 0000000000..8b77e636de --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-02.md @@ -0,0 +1,398 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +--- + +# Office Run Deduplication — Generation Identity System Design Part 2: observability and verification + +Part 2 covers what the queue *reports* when it acts on a dedup key +(`REQ-OFFICE-RUN-DEDUP-004`), the upgrade and failure behaviour, and the test +plan for all four requirements. The key contract itself — the producer audit, +the assignment generation, the per-producer sources, convergence and the keyless +path — is in [Part 1](run-dedup-generation-01.md). The full producer inventory +and the exact Go signatures named below are in +[Part 3](run-dedup-generation-03.md). + +## Observability + +### Counters + +New expvar maps, following `internal/office/scheduler/metrics_vars.go`'s +`k=v;k=v` label model and the counters-only rule stated there. + +**Three queue implementations make a dedup decision, not one, and all three are +in scope:** + +| Queue | Reached from | Today | +| --- | --- | --- | +| `runs/service.Service.QueueRun` | `office/service.Service.QueueRun` when a runs service is wired | checks the key; maps the unique violation to `Deduped`; logs `Debug` | +| `office/service.Service.queueRunInline` | same, when no runs service is wired (older tests, transitional deployments) | checks the key; returns the raw `CreateRun` error; logs `Debug` | +| `office/scheduler.SchedulerService.QueueRun` | `QueueRunCtx`, and so every reactivity wake | checks the key; returns the raw `CreateRun` error; logs `Debug` | + +The third most needs instrumenting: `ApplyTaskMutation` -> `QueueRunCtx` -> +`SchedulerService.QueueRun` is the path a reassignment travels and is the *least* +observable today. Instrumenting only `internal/runs/service` would leave +AC-004.1 / .2 / .3 unmet on the flow the card reports. + +To keep one behaviour rather than three, the *reporting* is one exported helper +in `internal/runs/service`: it classifies a suppression as `windowed` or +`durable`, increments the counter, emits the log, and returns the `QueueOutcome`. +All three queues call it. Counters are declared once there; nothing declares a +parallel map. Its name, its two entry points, and what it does with an error that +is *not* a unique violation are specified in +[Part 3](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome). + +The keyless counter is **not** driven from the queues. It has its own exported +reporter in the same package, `ReportKeylessEnqueue(reason, cause, detail)`, called by the +producer at its own decision site, because only the producer knows whether the key +was never needed or could not be resolved — see +[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation). + +| Name | Labels | Incremented when | +| --- | --- | --- | +| `office_run_dedup_total` | `reason`, `kind` (`windowed` or `durable`), `queue` (`runs` or `wakeup`) | the queue suppresses a wake | +| `office_run_dedup_keyless_total` | `reason`, `cause` (`unresolved` or `by_design`) | a producer calls `ReportKeylessEnqueue` before enqueuing with no dedup key | + +`kind` earns the metric: `windowed` is the expected, high-frequency outcome on +the engine redelivery paths, while `durable` means the 24-hour lookup found +nothing and the unbounded index still rejected the insert. After this ships that +is one of three things — a genuine race, a producer minting a colliding key, or a +**legitimate late redelivery** of one occurrence past the lookback window. The +third is not a fault and is not rare: `ParentWakeReconciler` is a level-triggered +backstop that re-derives the identical `wakeOperationID` from current task state +on every tick, with no age bound, so a parent stuck for more than a day produces a +durable hit as normal operation. `Warn` is still the right level — a durable hit +always means the fast path missed something an operator may want to see — but +neither the log text nor any alert built on this counter may call a durable hit an +anomaly on its own. One undifferentiated counter would bury the first two. +`cause` earns its place the same way: `by_design` is high-frequency and expected, +`unresolved` is the generation that should have been carried and was not +(AC-003.3), and by `reason` alone the two are indistinguishable. Metric reasons +use the finite set of known run reasons. Any agent-supplied or future reason is +reported as `custom`, so process-global expvar maps cannot grow once per input. +Cardinality is therefore the known reason set plus one custom bucket times a +small constant, the same order as `routing_*`. + +`office_run_dedup_total{queue="wakeup"}` increments where `CreateWakeupRequest` +returns `ErrWakeupIdempotencyConflict` (AC-004.6); that path has no windowed +lookup, so its `kind` is always `durable`. Its `reason` label is +`WakeupRequest.Reason`, the field already on the row being inserted, not the +coarser `Source`, which would collapse every wakeup onto one series. + +### Logs + +- Windowed hit: `Info`, carrying key and reason. Currently `Debug`. +- Durable hit: `Warn`, carrying key, reason, and resolved agent. Currently + `Debug`, on both the `runs` and wakeup paths. +- Keyless enqueue with `cause=unresolved`: `Info`, carrying `reason` and the + `detail` constant naming what failed to resolve — `ReportKeylessEnqueue`'s + third argument, whose per-producer values + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) fixes. Without + it the four producers that go keyless under `reason=task_assigned` are + indistinguishable in the log, which is what AC-003.3 exists to prevent. + `detail` is a log field only, never a counter label. A `cause=by_design` + keyless enqueue is counted but **not** logged — it is the normal path for + several reasons and would be pure noise. + +### Outcome propagation + +`runs/service.QueueRun` already returns `QueueOutcome`. Office's own enqueue +interfaces discard it: `office/shared.RunQueuer.QueueRun`, +`office/service.Service.QueueRun`, `office/scheduler.SchedulerService.QueueRun` +and `QueueRunCtx`, plus the duplicate declarations in `office/runtime/actions.go` +and `office/approvals/service.go`, all return only `error`. Widen them to +`(runsservice.QueueOutcome, error)` (AC-004.4). + +The three queues under [Counters](#counters) produce that outcome, and +`office/service.Service.QueueRun` forwards whichever of its two it used. +`queueRunInline` and `SchedulerService.QueueRun` return no `QueueOutcome` today +and map no unique violation; both gain the mapping from the shared helper, so a +durable conflict on the reactivity path becomes `Deduped` with a nil error +instead of today's `"reactivity run failed"` `Error` log (AC-002.3). + +**The wakeup enqueue is outside this widening**, and AC-004.4 now says so in its +own text rather than leaving it to be inferred. `CreateWakeupRequest` +(`office/repository/sqlite/wakeup_requests.go`, interface in +`office/routines/service.go`) keeps its `error` signature: no coalescing and no +windowed lookup, so there is no third outcome to report, and its one suppression +is the sentinel `ErrWakeupIdempotencyConflict`, which AC-004.5 exempts by name. +The exemption covers the signature only — a caller receiving that sentinel must +still not treat it as a failure, which the one live caller already does by +logging and continuing. + +**Sizing.** This widening is the highest-diff item here — `.QueueRun(` appears +~112 times across 33 files in `internal/office`, ~9 of them production call +sites. Size it as its own task rather than folding it into "observability". + +**A path that enqueues nothing reports `QueueOutcomeNone`.** `QueueOutcome`'s +zero value is the empty string, which is none of `queued`, `deduped` or +`coalesced`. Part 3 gives it a name and a meaning +([the no-op outcome](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome)). +This is additive and changes no acceptance criterion: AC-004.4 obliges the queue +to report one of three outcomes *when a caller enqueues a wake*, and a widened +signature returning a non-nil error never meets that condition. The reactivity +pipeline's empty-agent-id guard is **not** an instance of it: that guard lives in +the `queue` closure and returns *before* calling `QueueRunCtx`, so it yields no +outcome at all rather than `QueueOutcomeNone`. `QueueRunCtx` itself has no such +early return. + +Callers that have nothing to decide assign the outcome to `_`. No caller may +treat `Deduped` or `Coalesced` as an error (AC-004.5); the reactivity pipeline's +`queue` closure in particular must keep appending to `res.Runs` only for +`QueueOutcomeQueued`, so `ApplyTaskMutationResult.Runs` stops reporting +suppressed wakes as queued ones. + +## Key durability + +A dedup key is durable only for an enqueue that reached `insertRun`. All three +queues run `CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, and +`CoalesceRun` (`internal/runs/repository/sqlite/runs.go`) selects its neighbour +on `agent_profile_id`, `reason`, `status='queued'`, `requested_at` inside the +window, — for `task_assigned` only — the payload's `task_id`, **and +`(idempotency_key IS NULL OR idempotency_key NOT LIKE 'task_comment:%')`**. + +That last predicate is the one to be exact about, because two different keys are +in play and only one of them is read: + +- **The incoming enqueue's key is neither read nor written here.** The `UPDATE` + sets `coalesced_count` and `payload` only, so the merged row keeps its own key + and the incoming key is persisted nowhere. That is what makes a coalesced + enqueue **persist no row bearing its own key**; neither unique index ever sees + it. Whether the incoming enqueue may coalesce at all is decided earlier and + elsewhere, by `shouldCoalesceRun` (`internal/runs/service`), which excludes + only a `task_comment:`-prefixed key. +- **The predicate reads the candidate *neighbour's* key**, keeping a + `task_comment:`-keyed row from absorbing another reason's wake. A neighbour is + therefore matchable when its key is NULL **or** any non-`task_comment:` key, + including a `task_assigned:::` row. A test that needs + a matchable neighbour must not construct a `task_comment:`-keyed one — it would + never match, and the test would pass while asserting nothing. + +Two consequences the contract states rather than leaving to be discovered: + +- Two producers racing on one key may **both** coalesce into a queued neighbour, + leaving **zero** rows bearing that key. If the neighbour is claimed between + their two coalesce checks, the later one inserts and exactly one row bears it. + AC-002.4 is therefore written as *at most one, never two* rather than *exactly + one*: that is the only invariant true in every interleaving, and a test must + not pin the count to one. +- An occurrence that coalesced and is then redelivered *after* the coalescing + window has no durable key to match, and queues a second run (AC-001.4). That + duplicate is accepted, not closed: AC-003.2 elects a duplicate over a + suppression, and keying the merged row would change `CoalesceRun`, which + [Part 1](run-dedup-generation-01.md#purpose-and-boundaries) holds unchanged. + +This is a property of the pre-existing coalescing mechanism, not of the +generation component — but the generation component makes it *reachable* for +`task_assigned`, which is why it is stated here. Before this capability two +assignments inside one window shared a permanent key, so the second was stopped +by `CheckIdempotencyKey` and never reached `CoalesceRun`. With distinct +generations the windowed lookup misses and the coalesce arm is live. + +## Failure and recovery + +- A producer handed no `assignment_generation` goes keyless, never permanent: + the wake happens, dedup for that one enqueue does not. The only degraded case. +- A counter increment is best-effort and never fails an enqueue. +- The migration is additive with a `0` default and runs at startup, so both bump + sites always have the column. Every task starts at `0` and the first assignment + after upgrade commits `1`, so no task's first new-format key is `...:0`. +- **A second migration rebuilds `tasks` and must carry the column through it.** + `taskPriorityMigrationStatements` + (`office/repository/sqlite/base_migrations.go`) recreates `tasks` from an + **explicit column list** to change `priority` from INTEGER to TEXT. That list + would silently drop `assignment_generation`, so the column joins it the same + way eight other columns already do in that file — **its established idiom for + this exact hazard, followed rather than replaced**: + 1. a defensive `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT + NULL DEFAULT 0` before the recreate, with the error swallowed like its + neighbours (`archived_by_cascade_id`, `wip_admitted`, `external_id` and the + rest), so the `SELECT` can reference the column on a legacy fixture that + predates it; + 2. the column declared in the `tasks_priority_new` definition; + 3. `COALESCE(assignment_generation,0)` in the recreate `SELECT`, so values are + **preserved**, not reset. + Copying rather than defaulting is what makes this order-independent: it holds + whether or not the task repository's own migration ran first, so nothing rests + on an argument about which repository initialises when. + Left undone the defect is not a hard failure, which is why it is easy to miss. + The task repository initialises first (`backendapp/storage.go`), so the column + is added, then dropped by the rebuild, absent for the remainder of that boot, + then re-added at `0` on the next one — `db.MigrateLogger.Apply` + (`internal/db/migratelog.go`) is not ledger-backed, so the ALTER re-runs. A + task previously at generation 3 re-mints `...:1` and collides with its own + historical key on the unbounded index: this capability's own defect, + resurrected. +- Events in flight across the upgrade carry no `assignment_generation`. Their + producer goes keyless with `cause=unresolved` rather than minting a + legacy-shaped key: one event's lifetime, in the direction AC-003.2 elects. +- No runtime feature flag. A key-format change plus telemetry has no + partially-migrated state to guard — a mixed fleet writing old and new formats + does not collide, the property AC-001.8 relies on. + +## Testing + +Backend `*_test.go` beside each source. Behaviours with no equivalent test today: + +- A -> B -> A reassignment wakes the first agent again on the third assignment + (AC-001.2): three distinct keys, and the third enqueue reports queued or + coalesced, never deduped. Assert the **outcome**, not a row count — a test + driving all three inside the 5-second window legitimately gets two rows. +- A repeat assignment whose prior run's `requested_at` predates the window queues + a run (AC-001.3), driven by writing an aged row. That row is also outside the + coalescing window, so this case asserts `Queued` exactly. +- Both producers, driven directly with the same task, agent and carried + generation, derive one key and queue one run (AC-002.1 / .2). At the producers, + not through a dashboard call — only one fires there. +- A repeat assignment to the **same** agent passes the relaxed reactivity gate, + bumps, and wakes the agent (AC-001.3), including when the prior run is aged + past the window; and it does **not** set `InterruptSessionID`. A reassignment + to a *different* agent still does — both branches of the guard added in + [The reactivity gate](run-dedup-generation-01.md#the-reactivity-gate). +- A non-assignment task update (title, priority) reaches `syncRunnerInTx` but + does not bump and queues no `task_assigned` run — the inert-writer claim in + [Four writers](run-dedup-generation-01.md#four-writers-of-the-same-row-that-must-not-bump). +- A task created with an assignee whose `task.created` event carries + `assignment_generation` commits generation `1` and queues one run; the same + creation with the field absent goes keyless with `cause=unresolved` and still + queues, exercising `fallbackToStoredRunner`. +- **A task created UNASSIGNED publishes `assignment_generation` `0`**, carried on + `publishTaskEventWithExtra`'s `extra` map, and queues no `task_assigned` run. + Assigning it afterwards commits `1` and queues one. This pins the boundary that + keeps a creation and a first assignment from both minting `...:1`, and it is + the only assertion that catches a create path that hardcodes `1`. +- A producer handed a generation does not read the task row for one: drive + occurrence 1's producer after occurrence 3 has committed and assert it still + mints occurrence 1's key. +- A durable conflict on the **reactivity** path (`SchedulerService.QueueRun`, and + the same for `queueRunInline`) returns `Deduped` with a nil error and does not + log `"reactivity run failed"` (AC-002.3). +- A key-format table test over **every row of the + [producer audit](run-dedup-generation-03.md#producer-audit)**, with the + assertion chosen by what the row produces. **Two assertions, not one:** + - A row that mints a key asserts its output has a segment that varies across + two constructed occurrences (AC-001.1), so a new producer reintroducing a + permanent key fails a test rather than shipping. + - A row that is **keyless by contract** has no key and therefore no segment, + so the varying-segment assertion is unsatisfiable and is not applied to it. + It asserts instead that the enqueue carries an **empty** key and that the + producer reported the `cause` and `detail` + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns it. + These are the three permanently-keyless rows of + [Part 3](run-dedup-generation-03.md#keyless-today--must-report-a-cause) — + the fourth, `agent_error`, becomes generational here and takes the first + assertion — plus the three status-driven reactivity reasons. + The same-set blocker re-resolution named in the requirements' `## Out of scope` + is the one row that mints a key and is asserted *stable* rather than varying, + pinning the collapse rather than letting it drift. +- Durable-index conflict increments `office_run_dedup_total` with `kind=durable` + and returns `Deduped` with a nil error. +- An unresolvable generation enqueues keyless with + `office_run_dedup_keyless_total{cause=unresolved}`; a status-driven reason + enqueues keyless with `cause=by_design` and emits no log record. +- `SpawnAgentRun` with an empty agent key and a live caller run id enqueues + keyless, and two such calls in one run are **not deduplicated**. Assert the + outcome, not a row count: an empty key passes `shouldCoalesceRun`, and the + first call's own row — which persisted a NULL key — satisfies the + `idempotency_key IS NULL` arm of the neighbour predicate in + [Key durability](#key-durability), whose task-scoping clause applies only to + `task_assigned`, so two calls to one agent with one reason inside the 5-second + window legitimately coalesce. Drive them with distinct reasons, or + outside the window, to assert `Queued` twice. +- Two recipients' wakes for one comment (assignee plus a mentioned agent) mint + distinct keys and both queue. +- **AC-001.7's run-id prefix, on the non-empty agent key** — the case the + empty-key bullet above does not reach. Two `SpawnAgentRun` calls passing the + *same* literal key inside one caller run dedupe; the same literal key from a + *second* caller run queues, because the prefix differs. A test that drives only + the empty-key path leaves the prefix logic unpinned. +- **The wakeup queue increments its own counter (AC-004.6).** A + `CreateWakeupRequest` rejected with `ErrWakeupIdempotencyConflict` moves + `office_run_dedup_total{queue="wakeup",kind="durable"}` **through + `ReportDurableDedup`, not `ReportInsertResult`** — the caller does its own + `errors.Is` on the sentinel, so assert the counter moves without + `runs/service` ever being handed that error. Labelled by + `WakeupRequest.Reason` and not by `Source`, and the caller does not treat the + sentinel as a failure. Nothing else in this plan exercises that file, so + without this the label can be unwired or wired to the wrong `queue` value with + the suite green. +- **The three direct keyless producers each report their assigned cause** — the + onboarding wake and `handleTaskCreated`'s fallback as `unresolved`, the + recovery sweep as `by_design` — per + [Part 3](run-dedup-generation-03.md#keyless-causes-per-producer). The recovery + sweep additionally asserts it does **not** mint the assignment key: drive a + sweep over a task whose assignment run already persisted + `task_assigned:::` and assert the sweep still queues, + rather than being suppressed by the durable index. +- **`agent_error` and `manual_resume_after_failure` are generational** on the + failed run id (`manual_resume_after_failure` falls back to the task id when + the failed run id is empty). Two distinct run failures for one agent mint + different keys and both queue; a redelivery of one failure escalation is + suppressed. There is no two-CEO case to assert for `agent_error`: a + workspace admits at most one CEO (`ErrAgentCEOAlreadyExists`) and + `queueCEOAgentError` escalates to `ceos[0]` alone. +- **Both blocker producers derive a byte-identical digest** for one blocker set, + driven through the shared builder from both packages, including a set whose ids + were returned in a different order. An empty blocker set enqueues nothing and + moves no keyless counter. +- **A path that enqueues nothing returns `QueueOutcomeNone`**, not + `QueueOutcomeQueued`: a widened signature returning an error. Assert + `ApplyTaskMutationResult.Runs` gains no entry. The `queue` closure's + empty-agent-id guard is a **separate** case with a different assertion — it + reaches no queue at all, so assert `res.Runs` gains no entry and that + `QueueRunCtx` was never called, not that it returned an outcome. +- **`QueueOutcomeNone` is declared identically in both packages.** A compile-time + or table assertion that `runsservice.QueueOutcomeNone` and + `engine.QueueOutcomeNone` hold the same value, pinning the "both MUST match" + invariant those two declarations already carry. +- **`ReportInsertResult` passes a non-conflict error through unchanged**, with + `QueueOutcomeNone` and no counter movement — a disk error is not a dedup + decision and must not be counted or swallowed as one. +- **The `tasks` rebuild preserves the column and its values.** Drive + `taskPriorityMigrationStatements` over a legacy fixture whose `tasks.priority` + is still INTEGER, seeded with one row at `assignment_generation` 3 and one at + `0`, and assert both survive the recreate unchanged. Also drive it over a + fixture predating the column entirely and assert the defensive ALTER makes the + `SELECT` succeed at `0`. Without this the loss is invisible: the column is + re-added on the next boot, so nothing fails and only the resurrected key + collision would ever show it. +- Routine keys, per source: two cron fires of one trigger on **different claimed + ticks** mint different keys; one slot re-dispatched with the same claimed tick + mints the identical key; two **manual** fires of one routine inside one minute + mint different keys (today they collide); and a `source == "cron"` dispatch with + no claimed tick carried enqueues keyless with `cause=unresolved`. The cron case + is driven by supplying the claimed tick directly, not by advancing a clock. +- A cron producer does not recover the tick by re-reading the trigger row + (AC-001.9): drive a dispatch after `UpdateTriggerNextRun` has advanced + `next_run_at` and assert the key still names the claimed slot, not the next one. +- `ReportKeylessEnqueue` is what moves `office_run_dedup_keyless_total`: a + producer that goes keyless increments its own `cause` and logs its own + `detail` constant — assert two producers going keyless under + `reason=task_assigned` are distinguishable in the log record, which is the + whole reason the argument exists; and an enqueue with an + empty key that did **not** call the reporter moves no counter — the assertion + that keeps the withdrawn "empty key means `by_design`" inference from returning. +- An occurrence that coalesced into a queued neighbour and is then redelivered + after the coalescing window queues a **second** run (AC-001.4, see + [Key durability](#key-durability)): enqueue generation N, enqueue generation + N+1 inside the window and assert `Coalesced`, age the merged row past the + window, then re-enqueue generation N+1's key and assert `Queued`. This pins the + accepted duplicate — a test asserting one run here is asserting the wrong + contract. +- Two producers enqueuing one key concurrently **with** a coalescible run already + queued for that agent, reason and task: both observe a suppression, neither + errors, and no row bears the new key (AC-002.3, AC-002.4). The no-neighbour + arrangement is the separate case the same ACs bound, and needs its own test. +- A durable-index conflict produced by re-deriving one occurrence's key past the + lookback window (the `ParentWakeReconciler` shape) increments + `office_run_dedup_total{kind=durable}`, returns `Deduped` with a nil error, and + is not reported as a failure anywhere. + +No Playwright coverage: the observable surfaces are `/debug/vars` and backend +logs, and the one user-visible consequence (a run row appearing for a repeat +assignment) is asserted at the queue in Go. diff --git a/docs/specs/office/system-design/run-dedup-generation-03.md b/docs/specs/office/system-design/run-dedup-generation-03.md new file mode 100644 index 0000000000..48d792a470 --- /dev/null +++ b/docs/specs/office/system-design/run-dedup-generation-03.md @@ -0,0 +1,504 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-RUN-DEDUP-001 + - REQ-OFFICE-RUN-DEDUP-002 + - REQ-OFFICE-RUN-DEDUP-003 + - REQ-OFFICE-RUN-DEDUP-004 +created: 2026-09-07 +owners: + - kandev +--- + +# Office Run Deduplication — Generation Identity System Design Part 3: producer inventory and API contracts + +Part 3 owns two things Parts 1 and 2 refer to but do not spell out: the complete +inventory of every producer that reaches a run queue, and the exact Go contracts +this capability adds or widens. The conceptual key contract is in +[Part 1](run-dedup-generation-01.md); observability, key durability and the test +plan are in [Part 2](run-dedup-generation-02.md). + +Everything here is a contract a builder would otherwise have to invent. Where a +name or a byte encoding is given, it is binding — two producers that must agree +(AC-OFFICE-RUN-DEDUP-002.1) cannot agree on a convention nobody wrote down. + +## Producer audit + +Every site that reaches a run queue. "Generational" means the key already varies +with the occurrence. The fourth table is the one Parts 1 and 2 previously +omitted: producers that pass no key at all, which +AC-OFFICE-RUN-DEDUP-003.3 nonetheless obliges to report a cause. + +### Already generational — no change + +| Producer | Key shape | Generation component | +| --- | --- | --- | +| `office/service` `queueCommentRun` | op id `task_comment:`, engine-expanded on persist | comment row id | +| `office/service` `handleApprovalResolved` | `approval_resolved:` | approval row id | +| `office/approvals/service.go` `queueApprovalRun` | `approval:` | approval row id | +| `office/dashboard/decisions.go` `decisionRunIdempotencyKey` | `decision:` | decision row id | +| `office/scheduler` `childrenCompletedIdempotencyKey` | `task_children_completed:::` | child-set digest | +| `workflow/engine/phase2_callbacks.go` `idempotencyKey` | `::::` | step-entry / operation id | +| `orchestrator` `officeAutoStartIdempotencyKey` | `task_assigned::::` | step-transition row id | +| `orchestrator` step-entry operation id | `step_entry::` | entry id and position | +| `scheduler/cron/heartbeat.go` operation id | `heartbeat:::` | tick time (AC-001.6) | +| `office/service/scheduler_wake_reconciler.go` `wakeOperationID` | `task_children_completed::` op id, shared with `queueChildrenCompletedRun` | child-set digest | + +`officeAutoStartIdempotencyKey`'s `legacy:` branch fires when +`stepTransitionID` is zero. It is time-derived, which AC-001.5 disallows for an +occurrence that has a durable row. Replace it with the keyless path of +[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation); do +not otherwise touch this producer. + +### Not generational — must change + +| Producer | Key today | Failure | +| --- | --- | --- | +| `office/service` `queueTaskAssignedRun` | `task_assigned::` | permanent per pair | +| `office/scheduler/run.go` `QueueRunCtx` default | `::` | permanent per (reason, task, agent) | +| `office/service` blocker-resolved dispatch | op id `blockers_resolved:` | permanent per blocked task | + +`QueueRunCtx`'s default is the larger of the two: every `RunContext` that does +not set `IdempotencyKey` inherits it — every reactivity wake except +children-completed. + +### Unconstrained — must be scoped + +| Producer | Key today | Failure | +| --- | --- | --- | +| `office/runtime/actions.go` `SpawnAgentRun` | verbatim `input.IdempotencyKey` | an agent reusing a literal permanently suppresses its own later wakes | +| `office/routines/service.go` `buildRoutineIdempotencyKey` | `routine:[:]:` | the minute comes from the **processing** clock (`run.StartedAt`, set to `time.Now()` inside `dispatchRoutineRun`), not from the occurrence. A cron fire's occurrence is its claimed scheduled tick; a manual or webhook fire's is `RoutineRun.ID`. Both are durable and neither is what the key uses. The reachable collision today is two manual fires of one routine inside one minute — they share `triggerID == ""` and the same minute | + +### Keyless today — must report a cause + +These four call a run queue **directly** with an empty key. None of them passes +through `QueueRunCtx`, so the blanket entry in the second table does not reach +them, and none was previously enumerated anywhere in this design. Each is +covered by [Keyless causes per producer](#keyless-causes-per-producer) below, and +each is a row the key-format table test of +[Part 2](run-dedup-generation-02.md#testing) must iterate — under that test's +keyless assertion (empty key, plus the reported `cause` and `detail`), never its +varying-segment one, which a producer with no key cannot satisfy. + +| Producer | Reason | Key today | Disposition | +| --- | --- | --- | --- | +| `office/onboarding/service.go` `maybeCreateOnboardingTask` | `task_assigned` | `""` | stays keyless, `cause=unresolved` | +| `office/service/scheduler_recovery.go` recovery sweep | `task_assigned` | `""` | stays keyless, `cause=by_design` | +| `office/service/retry.go` CEO error escalation | `agent_error` | `""` | **becomes generational** on the failed run id | +| `office/service/failure.go` `requeueRunForTask` | `manual_resume_after_failure` | `""` | **becomes generational** on the failed run id (task id if empty) | + +`office/onboarding` also declares its own `runReasonTaskAssigned` constant, a +third copy of the same string. Consolidating it is out of scope for the same +reason the requirements already give for the `office/scheduler` and +`office/service` blocks. + +### Writers that reach no queue at all + +`UpdateTaskAssignee` has five non-test call sites and only one of them is a +producer. The other four build no key and wake nobody, so they never appear in +the tables above; they are enumerated in +[UpdateTaskAssignee and the bump sites](#updatetaskassignee-and-the-bump-sites) +because they still take the widened signature. + +## Keyless causes per producer + +AC-OFFICE-RUN-DEDUP-003.3 splits keyless enqueues into `unresolved` — a +generation that should have been available and was not — and `by_design` — a +producer that never had an occurrence to name. The split only pays for itself if +each producer is assigned a side deliberately, so each is assigned one here. + +| Producer | Reason | `cause` | Why | +| --- | --- | --- | --- | +| reactivity status wakes | `task_unblocked`, `task_reopened`, `task_review_requested` | `by_design` | a status transition has no redelivery path; see Part 1 | +| `office/service/scheduler_recovery.go` | `task_assigned` | `by_design` | see [The recovery sweep](#the-recovery-sweep-must-not-borrow-the-assignment-generation) | +| `office/onboarding/service.go` | `task_assigned` | `unresolved` | the occurrence **does** have a generation; this producer simply is not handed it | +| `handleTaskCreated` with `fallbackToStoredRunner` | `task_assigned` | `unresolved` | Part 1 | +| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `task_assigned` | `unresolved` | Part 1 | +| `SpawnAgentRun`, no caller run id | agent-supplied | `unresolved` | Part 1 | +| `SpawnAgentRun`, empty agent key | agent-supplied | `by_design` | the agent expressed no dedup intent; nothing failed to resolve | +| routine dispatch, `cron` with no claimed tick | routine reasons | `unresolved` | Part 1 | +| routine dispatch, unrecognised `Source` | routine reasons | `unresolved` | Part 1 | +| reactivity comment wake, nil `MutationComment` | `task_comment` | `unresolved` | Part 1 | + +**The `detail` argument, per producer.** Part 1's +[`ReportKeylessEnqueue`](run-dedup-generation-01.md#unresolvable-generation) +takes `detail` so the `cause=unresolved` log record names *what* failed to +resolve, which `reason` alone cannot: four distinct producers above go keyless +under `reason=task_assigned`. Each `unresolved` row reports a fixed lowercase +snake_case constant naming the producer's own failure, not a formatted message: + +| Producer | `detail` | +| --- | --- | +| `office/onboarding/service.go` | `onboarding_no_generation` | +| `handleTaskCreated` with `fallbackToStoredRunner` | `event_missing_generation` | +| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `zero_step_transition` | +| `SpawnAgentRun`, no caller run id | `no_caller_run` | +| routine dispatch, `cron` with no claimed tick | `cron_no_claimed_tick` | +| routine dispatch, unrecognised `Source` | `unrecognised_routine_source` | +| reactivity comment wake, nil `MutationComment` | `nil_mutation_comment` | + +Every `by_design` row passes `""`: that cause is counted and never logged, so it +has nothing to distinguish. `detail` is a log field only and **not** a counter +label — these seven values would multiply the keyless counter's cardinality for +no operational question that `cause` and `reason` do not already answer. + +### The recovery sweep must not borrow the assignment generation + +`office/service/scheduler_recovery.go` re-queues tasks it finds unstarted. It is +level-triggered from current task state, the same shape as `ParentWakeReconciler`, +and by the time it runs, `tasks.assignment_generation` for that task is readable. +Reading it is nonetheless **forbidden**, and not only because AC-001.9 rules out +recovering a generation by re-reading a row. + +The decisive argument is that it would defeat the sweep. The sweep exists to +recover a task whose assignment wake was lost or whose run never started. If it +minted `task_assigned:::` from the current row, that is +byte-for-byte the key the original assignment already persisted on the run that +failed. `idx_run_idempotency` is unbounded, so the recovery enqueue would be +rejected as a durable dedup hit and the sweep would recover nothing — it would be +a permanent no-op with a counter, which is the defect this whole capability +exists to remove, reintroduced in the one path meant to be the backstop. + +So the sweep enqueues keyless with `cause=by_design`. Its occurrence is "this +task was still unstarted at this sweep", which no durable row records. The cost +is a possible duplicate wake if the sweep runs twice before a run is claimed; +AC-003.2 elects exactly that trade. The 5-second coalescing window absorbs the +common case. + +### Onboarding is a third `task_assigned` producer + +`maybeCreateOnboardingTask` calls `CreateOfficeTask` -> `taskservice.CreateTask` +with `AssigneeAgentProfileID` set, then enqueues a `task_assigned` run itself. +Because that creation publishes `events.TaskCreated`, `queueTaskAssignedRun` also +observes the same occurrence and — after this capability — mints the keyed +`task_assigned:::1`. + +Part 1's [Convergent producers](run-dedup-generation-01.md#convergent-producers) +says two producers observe an assignment. For a creation occurrence reached +through onboarding there are **three**, and the third cannot converge with the +other two: `CreateOfficeTask` returns only the task id, so onboarding never +receives the generation the creating transaction committed. Widening that +adapter is not undertaken here. + +Onboarding therefore enqueues keyless with `cause=unresolved` — `unresolved` +rather than `by_design` precisely because the occurrence *does* have a +generation and this producer merely is not handed it, which is the distinction +AC-003.3 exists to make countable. This does not violate AC-002.1: that criterion +binds two producers that **each derive a key**, and a producer enqueueing with no +key is governed by REQ-003 instead. The scoping is in the criterion's own text, +so the onboarding pair needs no exemption of its own. The consequence is stated rather than left to +be discovered: onboarding's wake and `queueTaskAssignedRun`'s wake are two +enqueues for one occurrence, and only one carries a key. Both fire on one call +path within milliseconds, for one agent, reason and task, so `CoalesceRun` merges +them into a single queued run in the ordinary case. If they are ever separated by +more than the coalescing window, the operator gets a duplicate onboarding wake. +That is AC-003.2's elected direction, and the counter makes it visible. + +### `agent_error` becomes generational + +`office/service/retry.go` escalates a run failure to the workspace CEO. Its +payload already carries `run_id`, so the occurrence has a durable identity and +the audit's own rule applies: a producer with an occurrence to name must name it. + +```text +agent_error:: +``` + +The CEO agent id is included to keep the key's shape uniform with the other +recipient-addressed keys (`task_comment`, `task_mentioned`), which is the whole +of its justification. **It is not doing collision-avoidance work**, and a builder +must not go looking for the fan-out that would make it necessary: a workspace has +at most one CEO — `office/agents/service.go` and `office/service/agents.go` both +reject a second with `ErrAgentCEOAlreadyExists` — and `queueCEOAgentError` +(`office/service/retry.go`) escalates to `ceos[0]` alone, with no fan-out. +`agent_error:` would be equally correct today; the salt is retained +because it costs nothing and survives a CEO seat being replaced. + +A redelivery of one failure escalation is suppressed; a second, distinct failure +of the same agent produces a different `run_id` and wakes again. + +This producer moves out of the keyless path entirely and takes no `cause`. AC-001.8 +needs no argument here: every `agent_error` row persisted before this capability +carries a NULL `idempotency_key`, and both unique indexes are partial and ignore +NULL, so an old row can never collide with the new key. + +## Go API contracts + +### `UpdateTaskAssignee` and the bump sites + +`office/repository/sqlite/tasks.go` and the interface declaration in +`office/dashboard/service.go` both widen to return the generation the +transaction committed: + +```go +UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) +``` + +The returned value is read back **inside the existing transaction, after the +UPDATE and before `Commit`**. It is the generation of *this* assignment +occurrence, which is the only value a producer may key on. + +All five non-test call sites, and what each does with the value: + +| Call site | Role | Disposition | +| --- | --- | --- | +| `office/dashboard/service_tasks.go` (reached from `DashboardService.SetTaskAssigneeAsAgent`) | **producer** | carries the value into `TaskReactivityChange.AssignmentGeneration` | +| `office/service/task_assignee.go` `SetTaskAssignee` | non-producer | discards it (`_`); its own signature is unchanged | +| `office/service/task_assignee.go` `SetTaskAssigneeAsAgent` | non-producer | discards it (`_`); its own signature is unchanged | +| `office/service/channels.go` channel setup | non-producer | discards it (`_`) | +| `office/channels/service.go` channel setup | non-producer | discards it (`_`) | + +**The channel-task flow is explicitly out of scope as a producer, and in scope as +a bump.** Both channel sites assign a long-lived channel task's runner seat +through `UpdateTaskAssignee`, so both bump the counter — that is correct, a +channel task's runner genuinely is being assigned. Neither publishes an +assignment event nor runs the reactivity pipeline, so neither builds a key and +neither wakes anybody. They take the widened signature and discard the value. No +`ReportKeylessEnqueue` call is made, because no enqueue happens. + +**A read-back failure fails the assignment.** The read is a `SELECT` of one +column on the row just written, in the same transaction, so it can only fail if +that transaction is already unusable. When it does fail, the transaction rolls +back and `UpdateTaskAssignee` returns the error with a zero generation: the +assignment does **not** commit. Committing an assignment whose generation could +not be read would produce an occurrence that no producer can ever name, which is +strictly worse than a failed call the caller can retry. + +The create-path bump site, `task/repository/sqlite/task.go` `insertTaskTx` -> +`upsertRunnerInTx`, inserts the literal `1` under the guard that writes the +runner row — `task.AssigneeAgentProfileID != "" && task.WorkflowStepID != ""` — +so it needs no read-back and neither signature changes. + +**What carries the generation onto `task.created`.** Not a post-commit re-read; +AC-001.9 forbids it and it is not needed, because the value is a constant the +guard above already determines: + +- the guard fired -> a runner row committed at generation `1`; +- the guard did not fire -> no runner row, and the task keeps the column default, + generation `0`. + +`CreateTask` (`task/service/service_tasks.go`) holds the same two fields the +guard reads, on the very `*models.Task` it handed the transaction, so it +re-evaluates that condition without a query and publishes the result through +`publishTaskEventWithExtra` +(`task/service/service_events.go`), whose `extra map[string]interface{}` is the +carrier. The create site calls `publishTaskEvent` today and moves to the +`WithExtra` form; `publishTaskEvent` already delegates to it, so no new plumbing +is introduced. + +**A task created unassigned publishes generation `0`, not `1`.** `0` means "never +assigned" and can never name an assignment occurrence, so no `task_assigned` +producer keys on it — a creation that assigned nobody produces no assignment +occurrence to wake for, and the task's first real assignment goes through +`UpdateTaskAssignee` and commits `1`. Publishing `1` for an unassigned creation +would let one task mint `task_assigned:::1` twice — once at a +creation that assigned nobody, once at its genuine first assignment — recreating +the durable collision this capability exists to remove. + +This satisfies AC-001.9 rather than bending it. The prohibition is on recovering +a generation by re-reading a row a later occurrence has since moved; nothing is +re-read here. The value is derived from in-memory fields of the struct the +creating transaction was given, so it cannot observe a later occurrence, and for +a creation it is a constant either way. + +### The event payload field + +`assignment_generation` on the `task.created` / `task.updated` payload is a +**nullable** JSON number, decoded into the `*int64` that Part 1 specifies: + +- absent, or JSON `null` -> `nil` -> the producer goes keyless with + `cause=unresolved`; +- a JSON number -> that generation, used verbatim. + +`0` on the wire is a literal generation zero and **is not** a sentinel for +"absent". A task at generation `0` has never been assigned, so it can never be +the subject of a `task_assigned` occurrence; but a decoder that mapped `0` to +"absent" would silently convert a malformed payload into a keyless enqueue, and +one that mapped "absent" to `0` would mint `...:0` keys that collide across every +unassigned task. Neither is permitted: the field is nullable on the wire and +optional in the decoder, and the two states stay distinct. + +### The shared key builders + +Two exported builders live beside `internal/runs/commentkeys`, reachable from +`office/service`, `office/scheduler` and `orchestrator` without an import cycle. + +**The assignment key.** Both `task_assigned` producers call it rather than +formatting the string themselves, which is what makes their convergence +structural instead of coincidental (AC-002.1). + +**The blocker-set digest.** Both blocker producers — `task_blockers_resolved` in +`office/scheduler` and the `blockers_resolved` operation id in `office/service` — +call **one shared builder**. They live in different packages, and +AC-002.1 requires them to derive an identical key for one occurrence, so a +convention each implements separately is not sufficient: two packages choosing +`,` and `:` as a delimiter produce different digests for the same blocker set and +fail invisibly, presenting as a durable dedup miss rather than as an error. + +The encoding is binding, and is the encoding +`childrenCompletedIdempotencyKey` already uses, so the two agree by construction: + +1. take the `blocker_task_id` of each blocker in the wave; +2. sort ascending by byte value (**not** by `ListTaskBlockers`'s `created_at`, + which is not unique); +3. join with a single `,` (U+002C); +4. SHA-256 over the UTF-8 bytes of that string; +5. render the full 32-byte digest as lowercase hex (Go `%x`), never truncated. + +No de-duplication step is needed before sorting: `task_blockers` is +`PRIMARY KEY (task_id, blocker_task_id)`, so one task cannot list the same +blocker twice and two reads of an unchanged set always digest the same input. + +**An empty blocker set is not an occurrence and is not enqueued.** A wave with no +blockers has nothing to have resolved, and the alternative is worse than a +missing wake: the digest of the empty string is a constant, so an empty-set key +would be `task_blockers_resolved:::` — permanently +unique per (task, agent), which is precisely the defect this capability removes. +A producer that finds an empty set returns without enqueuing and without calling +`ReportKeylessEnqueue`, because no enqueue was attempted. + +**The set is digested at the readiness read, and is never re-read to build the +key.** Each blocker producer already reads the blocker rows to decide whether the +task is ready; that read is the snapshot the digest uses. This is the +capture-and-carry AC-001.9 imposes on the assignment generation and the cron +claimed tick, applied to a set rather than a scalar, and it is what stops a +producer digesting a set that a later wave has already moved. + +- `office/service` `resolveAndWakeIfUnblocked` (`event_subscribers.go`) already + holds the slice it read from `ListTaskBlockers`, and digests that slice. +- `office/scheduler` `allBlockersResolvedExcept` (`reactivity.go`) reads the same + rows and currently **discards** them, returning only a boolean. It returns the + slice alongside that boolean, and `cascadeBlockersResolved` passes it to the + shared builder rather than issuing a second read. + +Two producers observing one occurrence therefore digest what each saw at its own +readiness decision, and the consequence is stated rather than left to be +inferred. A blocker edge **added** between the two reads makes the later producer +find the task not ready at all, so it enqueues nothing. An edge **removed** +between them makes it digest a smaller set and mint a second key, which is a +duplicate wake — the direction AC-003.2 elects. A set that differs between the +two reads is a different wave, so AC-002.1's "same occurrence" premise does not +hold across it and no convergence is owed. Nothing here may lock the blocker +table or re-read to reconcile the two. + +**A failed blocker read is not an empty set, and the two must not collapse.** An +empty set returns without enqueuing because there was no occurrence; a failed +read means the producer does not know whether there was one. On a read error the +producer propagates the error and enqueues nothing — it does **not** fall through +to the empty-set arm, does not enqueue keyless, and does not call +`ReportKeylessEnqueue`, because no enqueue was attempted in either case. + +It must nonetheless be visible, and on one of the two paths it is not today. +`resolveAndWakeIfUnblocked` returns the error and `queueBlockersResolvedRuns` +logs it at `Error` with the blocked task id. `cascadeBlockersResolved` instead +drops it with a bare `continue` on `err != nil || !ready`, recording nothing, so +a wake lost to a read failure is indistinguishable from a task that was +legitimately still blocked. That arm gains an `Error` log naming the blocked task +and the error. This is the only behaviour change this section makes to the +readiness check itself; the check's own logic is unchanged. + +### The suppression reporter and the no-op outcome + +Part 2 requires one shared helper so three queue implementations report one +behaviour. It is exported from `internal/runs/service`, beside +`ReportKeylessEnqueue`, as three functions rather than one — the suppressions +are reached at different points and carry different evidence, and one of them is +classified by its caller rather than here: + +```go +type QueueSource string + +const ( + QueueSourceRuns QueueSource = "runs" + QueueSourceWakeup QueueSource = "wakeup" +) + +// The recent-duplicate lookup matched. Counts kind="windowed", logs at Info, +// and returns QueueOutcomeDeduped. +func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome + +// Classifies the error from an insert. A unique violation on idempotency_key +// counts kind="durable", logs at Warn with key, reason and agent, and returns +// (QueueOutcomeDeduped, nil). Any other non-nil error is returned UNCHANGED +// with QueueOutcomeNone and moves no counter. A nil error returns +// (QueueOutcomeQueued, nil). +func ReportInsertResult( + q QueueSource, reason, key, agentProfileID string, err error, +) (QueueOutcome, error) + +// A durable conflict the CALLER has already classified. Counts kind="durable", +// logs at Warn, and returns QueueOutcomeDeduped. ReportInsertResult delegates +// here once its own classification succeeds. +func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome +``` + +`ReportInsertResult` is the single place the *runs* unique violation is +recognised and mapped, which is what makes AC-002.3 hold on `queueRunInline` and +`SchedulerService.QueueRun` as well as in `runs/service`. Neither function ever +returns `QueueOutcomeCoalesced`: all three queues order +`CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, so an enqueue that +coalesced returns from the coalesce arm and never reaches an insert to classify. A non-conflict error is +deliberately passed through untouched: a disk error is not a dedup decision and +must not be counted as one, nor swallowed into a successful-looking outcome. + +**The wakeup path classifies its own conflict and calls `ReportDurableDedup`**, +not `ReportInsertResult` (AC-004.6). `ReportInsertResult` recognises a conflict +with `runssqlite.IsIdempotencyKeyUniqueViolation` +(`internal/runs/repository/sqlite`), which matches `idx_run_idempotency` on the +`runs` table and is already the classifier `runs/service` uses. It does **not** +recognise `ErrWakeupIdempotencyConflict` +(`office/repository/sqlite/wakeup_requests.go`) — a wrapped sentinel raised by a +different table's index, in the office tree — and it must not be taught to: +importing that sentinel would reverse the import direction this design pins +(`runs/service` imports only `office/models` from the office tree). + +So the classification stays where the sentinel already is. The wakeup caller +lives in the office tree, already receives the error, does its own +`errors.Is(err, ErrWakeupIdempotencyConflict)`, and calls `ReportDurableDedup` +with `QueueSourceWakeup`. Its `kind` is always `durable`: that table has no +windowed lookup. This is reporting only; per AC-004.4 and AC-004.5 the wakeup +signature itself is not widened and keeps returning that sentinel, and the caller +keeps treating it as a suppression rather than a failure. + +**The no-op outcome.** `QueueOutcome` is a string type whose zero value is `""`, +which is not one of `queued`, `deduped` or `coalesced`. That zero value is given +a name and a meaning rather than being left for a builder to rediscover: + +```go +// QueueOutcomeNone means no enqueue was attempted, or the attempt returned an +// error. It is the zero value, so it is what a widened signature yields on any +// path that returns before deciding an outcome. +const QueueOutcomeNone QueueOutcome = "" +``` + +**It is declared in both copies of the type.** `QueueOutcome` exists twice — +`internal/runs/service/service.go` and `internal/workflow/engine/adapters.go` — +and each declaration carries the doc invariant "both MUST match". +`QueueOutcomeNone` is therefore added to **both**, with the same value and the +same doc comment. Part 1 lists `internal/workflow/engine` as already generational +and unchanged; that is a statement about its *keys*, and adding this constant is +the only edit this capability makes to that package. Adding it to one side alone +would break an invariant the code asserts in its own comments. + +This is additive and changes no acceptance criterion. AC-004.4's obligation to +report queued, deduplicated or coalesced is conditioned on a caller enqueuing a +wake; a path that enqueues nothing never meets that condition. The paths that +return `QueueOutcomeNone` are the widened signatures returning a non-nil error, +including `ReportInsertResult`'s non-conflict arm. + +**`QueueRunCtx` is not one of them.** It has no early return on an empty resolved +agent id. That guard sits one level up, in the reactivity pipeline's `queue` +closure (`office/scheduler/reactivity.go`), which returns *before* calling +`QueueRunCtx` at all — so an empty agent id yields no outcome to report rather +than `QueueOutcomeNone`, and `ApplyTaskMutationResult.Runs` gains nothing because +no enqueue was attempted. Part 1's +[reactivity gate](run-dedup-generation-01.md#the-reactivity-gate) states this +correctly and is the reference. + +No caller may treat `QueueOutcomeNone` as success. In particular the reactivity +pipeline's `queue` closure appends to `ApplyTaskMutationResult.Runs` only for +`QueueOutcomeQueued`, so neither a suppression nor a no-op is reported as a +queued run. Returning `QueueOutcomeQueued` from a path that inserted nothing +would put a phantom run in that slice, which is the specific mis-report Part 2 +added the `Queued`-only rule to prevent.