fix(office): make repeat task reassignment work again after 24 hours (#3533)
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ import (
|
||||
officeroutines "github.com/kandev/kandev/internal/office/routines"
|
||||
officeservice "github.com/kandev/kandev/internal/office/service"
|
||||
officewakeup "github.com/kandev/kandev/internal/office/wakeup"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
taskservice "github.com/kandev/kandev/internal/task/service"
|
||||
@@ -267,7 +268,12 @@ func (a *routineWakeupAdapter) CreateWakeupRequest(
|
||||
}
|
||||
if err := a.repo.CreateWakeupRequest(ctx, row); err != nil {
|
||||
if errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) {
|
||||
return officeroutines.ErrWakeupAlreadyRequested
|
||||
runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, req.Reason, req.IdempotencyKey, req.AgentProfileID)
|
||||
// Wraps both sentinels so a caller can check either: routines
|
||||
// callers key off ErrWakeupAlreadyRequested to treat this as
|
||||
// success by another route, while errors.Is against the
|
||||
// sqlite-layer ErrWakeupIdempotencyConflict still matches.
|
||||
return fmt.Errorf("%w: %w", officeroutines.ErrWakeupAlreadyRequested, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package backendapp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"expvar"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
officeroutines "github.com/kandev/kandev/internal/office/routines"
|
||||
)
|
||||
|
||||
func newTestRoutineWakeupAdapter(t *testing.T) *routineWakeupAdapter {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("settings store init: %v", err)
|
||||
}
|
||||
repo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new office repo: %v", err)
|
||||
}
|
||||
return &routineWakeupAdapter{repo: repo}
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-004.6: a durable idempotency conflict on the wakeup
|
||||
// queue moves office_run_dedup_total{queue="wakeup",kind="durable"} through
|
||||
// ReportDurableDedup, not ReportInsertResult - the adapter does its own
|
||||
// errors.Is on the repository's sentinel and reports the conflict itself.
|
||||
func TestRoutineWakeupAdapter_CreateWakeupRequest_DurableConflictReportsCounter(t *testing.T) {
|
||||
adapter := newTestRoutineWakeupAdapter(t)
|
||||
ctx := context.Background()
|
||||
reason := "test_routine_wakeup_conflict_" + t.Name()
|
||||
key := "routine:r1:t1:tick:1234567890"
|
||||
|
||||
first := &officeroutines.WakeupRequest{
|
||||
ID: "wakeup-1", AgentProfileID: "agent-1", Source: "cron",
|
||||
Reason: reason, IdempotencyKey: key,
|
||||
}
|
||||
if err := adapter.CreateWakeupRequest(ctx, first); err != nil {
|
||||
t.Fatalf("first create: %v", err)
|
||||
}
|
||||
|
||||
second := &officeroutines.WakeupRequest{
|
||||
ID: "wakeup-2", AgentProfileID: "agent-1", Source: "cron",
|
||||
Reason: reason, IdempotencyKey: key,
|
||||
}
|
||||
err := adapter.CreateWakeupRequest(ctx, second)
|
||||
if !errors.Is(err, officesqlite.ErrWakeupIdempotencyConflict) {
|
||||
t.Fatalf("second create error = %v, want ErrWakeupIdempotencyConflict", err)
|
||||
}
|
||||
|
||||
// reason isn't in runs/service's bounded metricReasons allowlist, so the
|
||||
// label buckets it to "custom" rather than carrying it verbatim.
|
||||
if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=wakeup") {
|
||||
t.Fatal("expected office_run_dedup_total to carry a custom/durable/wakeup entry")
|
||||
}
|
||||
}
|
||||
|
||||
// counterHasLabel reports whether the named expvar.Map has an entry whose
|
||||
// key contains every given substring. Counters are process-global expvar
|
||||
// state (no reset hook), so tests assert presence/growth rather than exact
|
||||
// values that could collide with other tests in the same binary run.
|
||||
func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool {
|
||||
t.Helper()
|
||||
v := expvar.Get(mapName)
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map %q not registered", mapName)
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q is not a *expvar.Map", mapName)
|
||||
}
|
||||
found := false
|
||||
m.Do(func(kv expvar.KeyValue) {
|
||||
matches := true
|
||||
for _, s := range substrs {
|
||||
if !strings.Contains(kv.Key, s) {
|
||||
matches = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
found = true
|
||||
}
|
||||
})
|
||||
return found
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// approvalHandlerFixture wires the minimal stack needed to exercise the
|
||||
@@ -72,7 +73,9 @@ func (s *silentActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _,
|
||||
|
||||
type silentRunQueuer struct{}
|
||||
|
||||
func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil }
|
||||
func (s *silentRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) {
|
||||
return runsservice.QueueOutcomeQueued, nil
|
||||
}
|
||||
|
||||
// seedApprovalAgent creates an agent_profiles row with the given role
|
||||
// and permissions, returning the persisted instance.
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@@ -29,7 +30,7 @@ type AgentWriter interface {
|
||||
|
||||
// RunQueuer enqueues run requests for agent instances.
|
||||
type RunQueuer interface {
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error)
|
||||
}
|
||||
|
||||
// ApprovalService handles approval CRUD and decide logic.
|
||||
@@ -215,6 +216,7 @@ func (s *ApprovalService) queueApprovalRun(ctx context.Context, approval *Approv
|
||||
approval.ID, approval.Type, approval.Status, approval.DecisionNote,
|
||||
)
|
||||
idempotencyKey := "approval:" + approval.ID
|
||||
return s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID,
|
||||
_, err := s.runs.QueueRun(ctx, approval.RequestedByAgentProfileID,
|
||||
"approval_resolved", payload, idempotencyKey)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/approvals"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// noopActivityLogger implements shared.ActivityLogger without importing shared.
|
||||
@@ -23,7 +24,9 @@ func (n *noopActivityLogger) LogActivityWithRun(_ context.Context, _, _, _, _, _
|
||||
// noopRunQueuer implements approvals.RunQueuer as a no-op.
|
||||
type noopRunQueuer struct{}
|
||||
|
||||
func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) error { return nil }
|
||||
func (n *noopRunQueuer) QueueRun(_ context.Context, _, _, _, _ string) (runsservice.QueueOutcome, error) {
|
||||
return runsservice.QueueOutcomeQueued, nil
|
||||
}
|
||||
|
||||
type fakeAgentWriter struct {
|
||||
statuses map[string]string
|
||||
|
||||
@@ -103,7 +103,7 @@ func (s *ChannelService) SetupChannel(ctx context.Context, channel *models.Chann
|
||||
// workflow_step_participants. Channel tasks have no workflow_step_id;
|
||||
// the participant row is keyed at (step_id="", task_id) and the
|
||||
// runner projection still resolves it.
|
||||
if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil {
|
||||
if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil {
|
||||
_ = s.repo.DeleteChannel(ctx, channel.ID)
|
||||
return fmt.Errorf("set channel task assignee: %w", err)
|
||||
}
|
||||
|
||||
@@ -141,6 +141,7 @@ func newTestDeps(t *testing.T) *testDeps {
|
||||
project_id TEXT DEFAULT '',
|
||||
assignee_agent_profile_id TEXT DEFAULT '',
|
||||
assignee_user_id TEXT NOT NULL DEFAULT '',
|
||||
assignment_generation INTEGER NOT NULL DEFAULT 0,
|
||||
labels TEXT DEFAULT '[]',
|
||||
metadata TEXT DEFAULT '{}',
|
||||
identifier TEXT DEFAULT '',
|
||||
|
||||
@@ -74,8 +74,11 @@ type Repository interface {
|
||||
GetRunsByCommentIDs(ctx context.Context, commentIDs []string) (map[string]sqlite.CommentRunStatus, error)
|
||||
UpdateTaskState(ctx context.Context, taskID, state string) error
|
||||
GetTaskExecutionFields(ctx context.Context, taskID string) (*sqlite.TaskExecutionFields, error)
|
||||
// UpdateTaskAssignee returns the task's assignment_generation after the
|
||||
// bump, read back inside the same transaction that wrote the runner
|
||||
// seat (see the sqlite implementation's doc comment).
|
||||
UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error)
|
||||
UpdateTaskStateIfWorkflowStep(ctx context.Context, taskID, expectedStepID, state string) (bool, error)
|
||||
UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error
|
||||
UpdateTaskPriority(ctx context.Context, taskID, priority string) error
|
||||
UpdateTaskProjectID(ctx context.Context, taskID, projectID string) error
|
||||
GetTaskProjectID(ctx context.Context, taskID string) (string, error)
|
||||
@@ -270,6 +273,11 @@ type MarkFixedHandler interface {
|
||||
type TaskReactivityChange struct {
|
||||
NewStatus *string
|
||||
NewAssigneeID *string
|
||||
// AssignmentGeneration is the value UpdateTaskAssignee's transaction
|
||||
// committed and read back, carried here rather than re-read. Nil means
|
||||
// the caller could not supply one (e.g. the read-back itself failed);
|
||||
// the pipeline then enqueues the task_assigned wake keyless.
|
||||
AssignmentGeneration *int64
|
||||
// PrevAssigneeID is the assignee BEFORE the mutation. Required when
|
||||
// NewAssigneeID is set so the pipeline can detect a real change and
|
||||
// hand off the previous assignee's session.
|
||||
|
||||
@@ -920,7 +920,8 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge
|
||||
zap.String("task_id", taskID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
if err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID); err != nil {
|
||||
generation, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -928,14 +929,16 @@ func (s *DashboardService) SetTaskAssigneeAsAgent(ctx context.Context, callerAge
|
||||
|
||||
// Reactivity pipeline — wakes the new assignee with task_assigned
|
||||
// and hard-cancels the previous assignee's active session.
|
||||
s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID)
|
||||
s.runReactivityForAssigneeChange(ctx, taskID, prevAssignee, assigneeID, callerAgentID, generation)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runReactivityForAssigneeChange invokes the reactivity pipeline for an
|
||||
// assignee change. Best-effort — failures are logged, never propagated.
|
||||
// generation is the value UpdateTaskAssignee's transaction just committed
|
||||
// and read back; it is carried onto the mutation rather than re-read.
|
||||
func (s *DashboardService) runReactivityForAssigneeChange(
|
||||
ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string,
|
||||
ctx context.Context, taskID, prevAssigneeID, newAssigneeID, callerAgentID string, generation int64,
|
||||
) {
|
||||
if s.reactivity == nil {
|
||||
return
|
||||
@@ -945,10 +948,11 @@ func (s *DashboardService) runReactivityForAssigneeChange(
|
||||
actorType = "agent"
|
||||
}
|
||||
change := TaskReactivityChange{
|
||||
NewAssigneeID: &newAssigneeID,
|
||||
PrevAssigneeID: prevAssigneeID,
|
||||
ActorID: callerAgentID,
|
||||
ActorType: actorType,
|
||||
NewAssigneeID: &newAssigneeID,
|
||||
AssignmentGeneration: &generation,
|
||||
PrevAssigneeID: prevAssigneeID,
|
||||
ActorID: callerAgentID,
|
||||
ActorType: actorType,
|
||||
}
|
||||
// preStatus="" — assignee changes don't depend on the prev status.
|
||||
result, err := s.reactivity.ApplyTaskMutation(ctx, taskID, "", change)
|
||||
@@ -963,7 +967,10 @@ func (s *DashboardService) runReactivityForAssigneeChange(
|
||||
// Flip the prev assignee's office session row to COMPLETED so it leaves
|
||||
// the active sessions list. The reactivity pipeline already hard-cancels
|
||||
// the running execution above; this is the persistent-row counterpart.
|
||||
if prevAssigneeID != "" && s.sessionTerm != nil {
|
||||
// A same-agent reassignment is not a handoff — the pipeline above never
|
||||
// interrupts it — so this must not terminate the agent's own live
|
||||
// session out from under it.
|
||||
if prevAssigneeID != "" && prevAssigneeID != newAssigneeID && s.sessionTerm != nil {
|
||||
if err := s.sessionTerm.TerminateOfficeSession(ctx, taskID, prevAssigneeID, sessionTermReasonReassigned); err != nil {
|
||||
s.logger.Warn("terminate prev-assignee office session failed",
|
||||
zap.String("task_id", taskID),
|
||||
@@ -975,7 +982,7 @@ func (s *DashboardService) runReactivityForAssigneeChange(
|
||||
// the user isn't asked to triage a failure they already worked
|
||||
// around by reassigning. Counter is intentionally not reset — the
|
||||
// root cause may still be unfixed for the old agent.
|
||||
if prevAssigneeID != "" && s.failureNotifier != nil {
|
||||
if prevAssigneeID != "" && prevAssigneeID != newAssigneeID && s.failureNotifier != nil {
|
||||
s.failureNotifier.OnAssigneeChanged(ctx, taskID, prevAssigneeID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,18 @@ type recordingTerminator struct {
|
||||
calls []termCall
|
||||
}
|
||||
|
||||
type recordingFailureNotifier struct {
|
||||
calls []struct {
|
||||
taskID, agentID string
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recordingFailureNotifier) OnAssigneeChanged(_ context.Context, taskID, agentID string) {
|
||||
r.calls = append(r.calls, struct {
|
||||
taskID, agentID string
|
||||
}{taskID: taskID, agentID: agentID})
|
||||
}
|
||||
|
||||
type termCall struct {
|
||||
taskID, agentID, reason string
|
||||
}
|
||||
@@ -85,12 +97,14 @@ func (r *recordingReactivity) ApplyTaskMutation(_ context.Context, _ string, _ s
|
||||
func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
rt := &recordingTerminator{}
|
||||
fn := &recordingFailureNotifier{}
|
||||
deps.svc.SetSessionTerminator(rt)
|
||||
deps.svc.SetFailureNotifier(fn)
|
||||
deps.svc.SetReactivityApplier(&recordingReactivity{result: &dashboard.TaskReactivityResult{}})
|
||||
|
||||
insertTestTask(t, deps.db, "task-r", "ws-r", "Reassign", "todo", 2)
|
||||
// Seed prev assignee directly via the underlying repo update.
|
||||
if err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil {
|
||||
if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-r", "agent-prev"); err != nil {
|
||||
t.Fatalf("seed prev assignee: %v", err)
|
||||
}
|
||||
|
||||
@@ -105,4 +119,38 @@ func TestSetTaskAssignee_TerminatesPrevSession(t *testing.T) {
|
||||
if got.taskID != "task-r" || got.agentID != "agent-prev" {
|
||||
t.Errorf("term call: got %+v", got)
|
||||
}
|
||||
if len(fn.calls) != 1 || fn.calls[0].taskID != "task-r" || fn.calls[0].agentID != "agent-prev" {
|
||||
t.Fatalf("expected prior-assignee failure notification, got %+v", fn.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetTaskAssignee_SameAgent_DoesNotTerminateSession is the regression
|
||||
// test for Review round 3 Finding 5: a repeat assignment to the agent that
|
||||
// already holds the seat must not flip that agent's own live session row to
|
||||
// COMPLETED. The reactivity pipeline correctly declines to hard-cancel this
|
||||
// case (AC-OFFICE-RUN-DEDUP-001.3 treats it as a real occurrence, not an
|
||||
// interrupt), and the persisted-row side effect must agree.
|
||||
func TestSetTaskAssignee_SameAgent_DoesNotTerminateSession(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
rt := &recordingTerminator{}
|
||||
fn := &recordingFailureNotifier{}
|
||||
deps.svc.SetSessionTerminator(rt)
|
||||
deps.svc.SetFailureNotifier(fn)
|
||||
deps.svc.SetReactivityApplier(&recordingReactivity{result: &dashboard.TaskReactivityResult{}})
|
||||
|
||||
insertTestTask(t, deps.db, "task-same", "ws-r", "Reassign", "todo", 2)
|
||||
if _, err := deps.repo.UpdateTaskAssignee(context.Background(), "task-same", "agent-x"); err != nil {
|
||||
t.Fatalf("seed prev assignee: %v", err)
|
||||
}
|
||||
|
||||
if err := deps.svc.SetTaskAssigneeAsAgent(context.Background(), "", "task-same", "agent-x"); err != nil {
|
||||
t.Fatalf("set assignee: %v", err)
|
||||
}
|
||||
|
||||
if len(rt.calls) != 0 {
|
||||
t.Fatalf("same-agent reassignment must not terminate the agent's own session, got %d calls (%+v)", len(rt.calls), rt.calls)
|
||||
}
|
||||
if len(fn.calls) != 0 {
|
||||
t.Fatalf("same-agent reassignment must not dismiss its failure inbox entry, got %+v", fn.calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
taskservice "github.com/kandev/kandev/internal/task/service"
|
||||
|
||||
"go.uber.org/zap"
|
||||
@@ -349,7 +350,11 @@ func (s *OnboardingService) maybeCreateOnboardingTask(
|
||||
return ""
|
||||
}
|
||||
if s.runQueuer != nil {
|
||||
if wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned,
|
||||
// A third task_assigned producer alongside queueTaskAssignedRun; it is
|
||||
// never handed the assigning transaction's generation, so it enqueues
|
||||
// keyless rather than deriving a divergent key.
|
||||
runsservice.ReportKeylessEnqueue(runReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "onboarding_no_generation")
|
||||
if _, wakeErr := s.runQueuer.QueueRun(ctx, agentID, runReasonTaskAssigned,
|
||||
fmt.Sprintf(`{"task_id":%q}`, taskID), ""); wakeErr != nil {
|
||||
s.logger.Warn("enqueue onboarding run failed", zap.Error(wakeErr))
|
||||
}
|
||||
|
||||
@@ -625,6 +625,7 @@ func (r *Repository) runTaskPriorityRecreate() error {
|
||||
{"external_id", `ALTER TABLE tasks ADD COLUMN external_id TEXT COLLATE BINARY`},
|
||||
{"external_id_settled_at", `ALTER TABLE tasks ADD COLUMN external_id_settled_at TIMESTAMP`},
|
||||
{"assignee_user_id", `ALTER TABLE tasks ADD COLUMN assignee_user_id TEXT NOT NULL DEFAULT ''`},
|
||||
{"assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`},
|
||||
}
|
||||
for _, column := range legacyColumns {
|
||||
if _, err := conn.ExecContext(ctx, column.stmt); err != nil && !db.IsDuplicateColumnError(err) {
|
||||
@@ -687,7 +688,8 @@ func taskPriorityMigrationStatements() []string {
|
||||
checkout_run_id TEXT,
|
||||
external_id TEXT COLLATE BINARY,
|
||||
external_id_settled_at TIMESTAMP,
|
||||
assignee_user_id TEXT NOT NULL DEFAULT ''
|
||||
assignee_user_id TEXT NOT NULL DEFAULT '',
|
||||
assignment_generation INTEGER NOT NULL DEFAULT 0
|
||||
)`,
|
||||
// archived_by_cascade_id and external_id/external_id_settled_at are
|
||||
// added to the task schema by task/repository/sqlite/base.go
|
||||
@@ -707,7 +709,7 @@ func taskPriorityMigrationStatements() []string {
|
||||
origin, project_id,
|
||||
labels, identifier,
|
||||
checkout_agent_id, checkout_at, checkout_run_id,
|
||||
external_id, external_id_settled_at, assignee_user_id
|
||||
external_id, external_id_settled_at, assignee_user_id, assignment_generation
|
||||
) SELECT
|
||||
id, COALESCE(workspace_id,''), COALESCE(workflow_id,''),
|
||||
COALESCE(workflow_step_id,''), title, COALESCE(description,''),
|
||||
@@ -722,7 +724,7 @@ func taskPriorityMigrationStatements() []string {
|
||||
COALESCE(labels,'[]'), identifier,
|
||||
checkout_agent_id, checkout_at, checkout_run_id,
|
||||
external_id, external_id_settled_at,
|
||||
COALESCE(assignee_user_id,'')
|
||||
COALESCE(assignee_user_id,''), COALESCE(assignment_generation,0)
|
||||
FROM tasks`,
|
||||
`DROP TABLE tasks`,
|
||||
`ALTER TABLE tasks_priority_new RENAME TO tasks`,
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
)
|
||||
|
||||
// TestMigrate_PriorityRebuildPreservesAssignmentGeneration is the regression
|
||||
// test for the same rebuild hazard TestMigrate_PriorityRebuildPreservesAssignee
|
||||
// covers one column over: taskPriorityMigrationStatements' INSERT ... SELECT
|
||||
// carries tasks.assignment_generation through
|
||||
// COALESCE(assignment_generation,0) (docs/specs/office/system-design/
|
||||
// run-dedup-generation-02.md#failure-and-recovery). That expression had zero
|
||||
// coverage before this test — the sibling assignee test seeds a table where
|
||||
// runTaskPriorityRecreate's own defensive ALTER TABLE ADD COLUMN supplies the
|
||||
// column fresh (every row defaults to 0), which never exercises copying a
|
||||
// genuinely non-zero value through the recreate. Here the legacy table
|
||||
// already carries the column with a non-zero value, as an install upgrading
|
||||
// mid-feature (assignment_generation shipped, priority rebuild still pending)
|
||||
// would.
|
||||
func TestMigrate_PriorityRebuildPreservesAssignmentGeneration(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db") + "?_journal_mode=WAL"
|
||||
db, err := sqlx.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT NOT NULL DEFAULT '',
|
||||
workflow_id TEXT NOT NULL DEFAULT '',
|
||||
workflow_step_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
state TEXT DEFAULT 'TODO',
|
||||
priority INTEGER DEFAULT 0,
|
||||
position INTEGER DEFAULT 0,
|
||||
metadata TEXT DEFAULT '{}',
|
||||
is_ephemeral INTEGER NOT NULL DEFAULT 0,
|
||||
parent_id TEXT DEFAULT '',
|
||||
archived_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
origin TEXT DEFAULT 'manual',
|
||||
project_id TEXT DEFAULT '',
|
||||
labels TEXT DEFAULT '[]',
|
||||
identifier TEXT,
|
||||
assignee_user_id TEXT NOT NULL DEFAULT '',
|
||||
assignment_generation INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
`); err != nil {
|
||||
t.Fatalf("seed legacy schema: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO tasks (id, workspace_id, title, assignment_generation)
|
||||
VALUES ('task-1', 'ws-1', 'reassigned task', 3)
|
||||
`); err != nil {
|
||||
t.Fatalf("seed reassigned task: %v", err)
|
||||
}
|
||||
|
||||
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init office repo (run migrations): %v", err)
|
||||
}
|
||||
|
||||
var generation int64
|
||||
if err := db.Get(&generation, `SELECT assignment_generation FROM tasks WHERE id = 'task-1'`); err != nil {
|
||||
t.Fatalf("tasks.assignment_generation missing after priority rebuild: %v", err)
|
||||
}
|
||||
if generation != 3 {
|
||||
t.Fatalf("assignment_generation lost by priority rebuild: got %d, want 3", generation)
|
||||
}
|
||||
}
|
||||
@@ -165,21 +165,32 @@ func (r *Repository) UpdateTaskStateIfWorkflowStep(
|
||||
// task_id) so the projection's per-task runner clause still resolves it
|
||||
// (the (step_id="" / step_id="") match holds because the SELECT joins
|
||||
// step_id = task.workflow_step_id which is also "").
|
||||
func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) error {
|
||||
//
|
||||
// This is one of the two assignment_generation bump sites (the other is
|
||||
// insertTaskTx -> upsertRunnerInTx on create). It increments
|
||||
// tasks.assignment_generation unconditionally on every committed call —
|
||||
// including a repeat assignment to the agent that already holds the seat,
|
||||
// which is a real occurrence, not a no-op — and reads the new value back
|
||||
// inside this same transaction before Commit, returning it so callers carry
|
||||
// it forward instead of re-reading it later (a later re-read could observe a
|
||||
// different, more recent occurrence's value). A read-back failure rolls the
|
||||
// whole assignment back rather than commit a write whose generation could
|
||||
// not be reported.
|
||||
func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error) {
|
||||
var stepID string
|
||||
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(
|
||||
`SELECT COALESCE(workflow_step_id, '') FROM tasks WHERE id = ?`),
|
||||
taskID).Scan(&stepID)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return fmt.Errorf("task not found: %s", taskID)
|
||||
return 0, fmt.Errorf("task not found: %s", taskID)
|
||||
}
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
@@ -188,7 +199,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID
|
||||
DELETE FROM workflow_step_participants
|
||||
WHERE step_id = ? AND task_id = ? AND role = 'runner'
|
||||
`), stepID, taskID); err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
var existing string
|
||||
@@ -201,7 +212,7 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID
|
||||
if _, err := tx.ExecContext(ctx, tx.Rebind(
|
||||
`UPDATE workflow_step_participants SET agent_profile_id = ? WHERE id = ?`),
|
||||
assigneeID, existing); err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
case sql.ErrNoRows:
|
||||
if _, err := tx.ExecContext(ctx, tx.Rebind(`
|
||||
@@ -209,19 +220,30 @@ func (r *Repository) UpdateTaskAssignee(ctx context.Context, taskID, assigneeID
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position, created_at)
|
||||
VALUES (?, ?, ?, 'runner', ?, 0, 0, ?)
|
||||
`), newParticipantUUID(), stepID, taskID, assigneeID, time.Now().UTC()); err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
default:
|
||||
return probeErr
|
||||
return 0, probeErr
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.ExecContext(ctx, tx.Rebind(`
|
||||
UPDATE tasks SET updated_at = CURRENT_TIMESTAMP WHERE id = ?
|
||||
UPDATE tasks SET assignment_generation = assignment_generation + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?
|
||||
`), taskID); err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
return tx.Commit()
|
||||
|
||||
var generation int64
|
||||
if err := tx.QueryRowxContext(ctx, tx.Rebind(
|
||||
`SELECT assignment_generation FROM tasks WHERE id = ?`),
|
||||
taskID).Scan(&generation); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return generation, nil
|
||||
}
|
||||
|
||||
// TaskBasicInfo contains the minimal task fields needed for prompt building.
|
||||
|
||||
@@ -330,9 +330,13 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one"); err != nil {
|
||||
gen1, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-one")
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskAssignee (insert): %v", err)
|
||||
}
|
||||
if gen1 != 1 {
|
||||
t.Fatalf("generation = %d, want 1 for the first assignment", gen1)
|
||||
}
|
||||
if got := taskRunner(t, repo, "as-1"); got != "agent-one" {
|
||||
t.Fatalf("runner = %q, want agent-one", got)
|
||||
}
|
||||
@@ -340,10 +344,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) {
|
||||
t.Fatalf("runner rows = %d, want 1", n)
|
||||
}
|
||||
|
||||
// A second assignment updates the existing row instead of adding one.
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two"); err != nil {
|
||||
// A second assignment updates the existing row instead of adding one,
|
||||
// and bumps the generation again.
|
||||
gen2, err := repo.UpdateTaskAssignee(ctx, "as-1", "agent-two")
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskAssignee (update): %v", err)
|
||||
}
|
||||
if gen2 != 2 {
|
||||
t.Fatalf("generation = %d, want 2 for the second assignment", gen2)
|
||||
}
|
||||
if got := taskRunner(t, repo, "as-1"); got != "agent-two" {
|
||||
t.Errorf("runner = %q, want agent-two", got)
|
||||
}
|
||||
@@ -351,10 +360,15 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) {
|
||||
t.Errorf("runner rows = %d, want the row updated in place", n)
|
||||
}
|
||||
|
||||
// An empty assignee deletes the runner row.
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-1", ""); err != nil {
|
||||
// An empty assignee deletes the runner row and still bumps the
|
||||
// generation: an unassignment is itself an assignment write.
|
||||
gen3, err := repo.UpdateTaskAssignee(ctx, "as-1", "")
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskAssignee (clear): %v", err)
|
||||
}
|
||||
if gen3 != 3 {
|
||||
t.Fatalf("generation = %d, want 3 for the clear", gen3)
|
||||
}
|
||||
if n := runnerRowCount(t, repo, "as-1"); n != 0 {
|
||||
t.Errorf("runner rows = %d, want 0 after clearing", n)
|
||||
}
|
||||
@@ -362,7 +376,7 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) {
|
||||
t.Errorf("runner = %q, want empty after clearing", got)
|
||||
}
|
||||
|
||||
err := repo.UpdateTaskAssignee(ctx, "missing", "agent-one")
|
||||
_, err = repo.UpdateTaskAssignee(ctx, "missing", "agent-one")
|
||||
if err == nil {
|
||||
t.Fatal("UpdateTaskAssignee(missing) = nil error, want not-found")
|
||||
}
|
||||
@@ -371,6 +385,34 @@ func TestUpdateTaskAssignee_WritesUpdatesAndClearsRunnerRow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration covers AC-001.3: a
|
||||
// repeat assignment to the agent that already holds the seat is a real
|
||||
// occurrence (the operator is asking for the work again), not a no-op, so it
|
||||
// must mint a new generation rather than leave the counter unchanged.
|
||||
func TestUpdateTaskAssignee_RepeatAssignmentBumpsGeneration(t *testing.T) {
|
||||
repo := newSearchTestRepo(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO tasks (id, workspace_id, workflow_step_id, title, created_at, updated_at)
|
||||
VALUES ('as-repeat', 'ws-1', 'step-1', 'Repeat assign', datetime('now'), datetime('now'))
|
||||
`); err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
gen1, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a")
|
||||
if err != nil {
|
||||
t.Fatalf("first assignment: %v", err)
|
||||
}
|
||||
gen2, err := repo.UpdateTaskAssignee(ctx, "as-repeat", "agent-a")
|
||||
if err != nil {
|
||||
t.Fatalf("repeat assignment: %v", err)
|
||||
}
|
||||
if gen2 != gen1+1 {
|
||||
t.Fatalf("repeat assignment generation = %d, want %d (one more than %d)", gen2, gen1+1, gen1)
|
||||
}
|
||||
}
|
||||
|
||||
// Tasks created outside a workflow have no workflow_step_id; the runner row
|
||||
// is keyed on the empty step so the projection still resolves it.
|
||||
func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) {
|
||||
@@ -384,7 +426,7 @@ func TestUpdateTaskAssignee_WorksWithoutAWorkflowStep(t *testing.T) {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil {
|
||||
if _, err := repo.UpdateTaskAssignee(ctx, "as-nostep", "agent-channel"); err != nil {
|
||||
t.Fatalf("UpdateTaskAssignee: %v", err)
|
||||
}
|
||||
if got := taskRunner(t, repo, "as-nostep"); got != "agent-channel" {
|
||||
@@ -405,13 +447,13 @@ func TestUpdateTaskAssignee_LeavesOtherTasksAlone(t *testing.T) {
|
||||
t.Fatalf("seed tasks: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil {
|
||||
if _, err := repo.UpdateTaskAssignee(ctx, "as-a", "agent-a"); err != nil {
|
||||
t.Fatalf("assign as-a: %v", err)
|
||||
}
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil {
|
||||
if _, err := repo.UpdateTaskAssignee(ctx, "as-b", "agent-b"); err != nil {
|
||||
t.Fatalf("assign as-b: %v", err)
|
||||
}
|
||||
if err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil {
|
||||
if _, err := repo.UpdateTaskAssignee(ctx, "as-b", ""); err != nil {
|
||||
t.Fatalf("clear as-b: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ func newSearchTestRepo(t *testing.T) *sqlite.Repository {
|
||||
workflow_step_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
assignee_user_id TEXT NOT NULL DEFAULT '',
|
||||
assignment_generation INTEGER NOT NULL DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
state TEXT DEFAULT 'TODO',
|
||||
priority TEXT NOT NULL DEFAULT 'medium' CHECK (priority IN ('critical','high','medium','low')),
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.9: the cron dispatch path threads the trigger's
|
||||
// claimed tick into the wakeup-request's idempotency key rather than
|
||||
// re-reading trigger.next_run_at after TickScheduledTriggers has already
|
||||
// advanced it to the next slot. Driving this through TickScheduledTriggers
|
||||
// (not the private key builder) is what actually pins the wiring: a future
|
||||
// change that re-reads the trigger row after the advance would still pass
|
||||
// the unit-level key-format tests but fail here.
|
||||
func TestTickScheduledTriggers_WakeupKeyNamesClaimedTick_NotAdvancedRow(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID,
|
||||
Kind: "cron",
|
||||
CronExpression: "* * * * *",
|
||||
Timezone: "UTC",
|
||||
Enabled: true,
|
||||
}
|
||||
if err := svc.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
triggers, err := svc.ListRoutineTriggers(ctx, routine.ID)
|
||||
if err != nil || len(triggers) != 1 {
|
||||
t.Fatalf("list triggers: %v (n=%d)", err, len(triggers))
|
||||
}
|
||||
claimedTick := *triggers[0].NextRunAt
|
||||
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
if err := svc.TickScheduledTriggers(ctx, claimedTick.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("tick scheduled triggers: %v", err)
|
||||
}
|
||||
if len(enq.created) != 1 {
|
||||
t.Fatalf("expected 1 wakeup-request created, got %d", len(enq.created))
|
||||
}
|
||||
|
||||
want := fmt.Sprintf("routine:%s:%s:tick:%d", routine.ID, triggers[0].ID, claimedTick.Unix())
|
||||
got := enq.created[0].IdempotencyKey
|
||||
if got != want {
|
||||
t.Fatalf("idempotency key = %q, want %q (the claimed tick, not the row's advanced next_run_at)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.2: two manual "Fire now" calls inside the same
|
||||
// minute are two distinct occurrences (an operator asking twice) and must
|
||||
// mint two distinct keys - the exact collision the old unix-minute key
|
||||
// format reached (both fires share triggerID == "" and can share a minute).
|
||||
func TestFireManual_TwoFiresInSameMinute_MintDistinctKeys(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
routine.ConcurrencyPolicy = "always_create"
|
||||
if err := svc.UpdateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("update routine: %v", err)
|
||||
}
|
||||
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "one"}); err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
if _, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "two"}); err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
if len(enq.created) != 2 {
|
||||
t.Fatalf("expected 2 wakeup-requests created, got %d", len(enq.created))
|
||||
}
|
||||
first, second := enq.created[0].IdempotencyKey, enq.created[1].IdempotencyKey
|
||||
if first == "" || second == "" {
|
||||
t.Fatalf("expected non-empty keys, got %q and %q", first, second)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package routines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
)
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.1 / .4: a cron fire's key names the claimed
|
||||
// scheduled slot, and redelivering the same slot (the same claimedTick)
|
||||
// reproduces the same key byte for byte.
|
||||
func TestBuildRoutineIdempotencyKey_Cron_UsesClaimedTick(t *testing.T) {
|
||||
tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-ignored")
|
||||
|
||||
want := fmt.Sprintf("routine:%s:%s:tick:%d", "routine-1", "trigger-1", tick.Unix())
|
||||
if key != want {
|
||||
t.Fatalf("key = %q, want %q", key, want)
|
||||
}
|
||||
|
||||
// A second claim of the identical slot (redelivery) mints the same key.
|
||||
repeat := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tick, "run-different")
|
||||
if repeat != key {
|
||||
t.Fatalf("redelivery of the same slot produced %q, want %q (identical to the first)", repeat, key)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.2: two distinct cron slots (a genuinely later
|
||||
// tick) mint two distinct keys.
|
||||
func TestBuildRoutineIdempotencyKey_Cron_DistinctTicksDiffer(t *testing.T) {
|
||||
tickOne := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC)
|
||||
tickTwo := tickOne.Add(time.Minute)
|
||||
|
||||
keyOne := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickOne, "run-a")
|
||||
keyTwo := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", &tickTwo, "run-b")
|
||||
|
||||
if keyOne == keyTwo {
|
||||
t.Fatalf("distinct cron slots must mint distinct keys, both got %q", keyOne)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-003.3: a cron fire with no claimed tick (the
|
||||
// exported entry point called directly with a nil tick, bypassing the
|
||||
// live processCronTrigger guard) has no occurrence identity and goes
|
||||
// keyless with cause=unresolved.
|
||||
func TestBuildRoutineIdempotencyKey_Cron_NoClaimedTickGoesKeyless(t *testing.T) {
|
||||
key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "", nil, "run-1")
|
||||
if key != "" {
|
||||
t.Fatalf("key = %q, want empty (keyless) for a cron fire with no claimed tick", key)
|
||||
}
|
||||
}
|
||||
|
||||
// Manual and webhook fires claim no slot, so two fires of one routine are
|
||||
// two distinct occurrences by design: each RoutineRun.ID mints its own key.
|
||||
func TestBuildRoutineIdempotencyKey_ManualAndWebhook_UseRoutineRunID(t *testing.T) {
|
||||
for _, source := range []string{"manual", "webhook"} {
|
||||
key := buildRoutineIdempotencyKey(source, "routine-1", "", "", nil, "run-1")
|
||||
want := "routine:routine-1:run:run-1"
|
||||
if key != want {
|
||||
t.Errorf("source=%q key = %q, want %q", source, key, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Two distinct manual fires -> two distinct keys (the collision the
|
||||
// old unix-minute key format could reach).
|
||||
first := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-a")
|
||||
second := buildRoutineIdempotencyKey("manual", "routine-1", "", "", nil, "run-b")
|
||||
if first == second {
|
||||
t.Fatalf("two distinct manual fires must mint distinct keys, both got %q", first)
|
||||
}
|
||||
}
|
||||
|
||||
// An unrecognised RoutineRun.Source names no occurrence this table covers
|
||||
// and goes keyless with cause=unresolved, the same direction as a cron
|
||||
// fire with no claimed tick.
|
||||
func TestBuildRoutineIdempotencyKey_UnrecognisedSourceGoesKeyless(t *testing.T) {
|
||||
key := buildRoutineIdempotencyKey("some_future_source", "routine-1", "", "", nil, "run-1")
|
||||
if key != "" {
|
||||
t.Fatalf("key = %q, want empty (keyless) for an unrecognised source", key)
|
||||
}
|
||||
}
|
||||
|
||||
// An explicit request key (a webhook delivery header, say) always wins,
|
||||
// even over a claimed cron tick, so a caller-supplied idempotency key never
|
||||
// collides with the source-derived identity scheme.
|
||||
func TestBuildRoutineIdempotencyKey_ExplicitKeyTakesPriority(t *testing.T) {
|
||||
tick := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
key := buildRoutineIdempotencyKey(shared.RoutineSourceCron, "routine-1", "trigger-1", "delivery-abc", &tick, "run-1")
|
||||
want := fmt.Sprintf("routine:%s:%s:%s", "routine-1", shared.RoutineSourceCron, "delivery-abc")
|
||||
if key != want {
|
||||
t.Fatalf("key = %q, want %q (explicit key must take priority over claimedTick)", key, want)
|
||||
}
|
||||
|
||||
// Two webhook deliveries with distinct explicit keys mint distinct keys.
|
||||
first := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-1", nil, "run-a")
|
||||
second := buildRoutineIdempotencyKey("webhook", "routine-1", "", "delivery-2", nil, "run-a")
|
||||
if first == second {
|
||||
t.Fatalf("two distinct explicit keys must mint distinct keys, both got %q", first)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
taskservice "github.com/kandev/kandev/internal/task/service"
|
||||
)
|
||||
|
||||
@@ -445,6 +446,11 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin
|
||||
zap.String("trigger_id", trigger.ID), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
// The claimed tick: UpdateTriggerNextRun below advances the trigger
|
||||
// row's next_run_at to the next slot, so trigger.NextRunAt is captured
|
||||
// here, before that write, and carried through to the key builder
|
||||
// rather than re-read afterward.
|
||||
claimedTick := trigger.NextRunAt
|
||||
if err := s.repo.UpdateTriggerNextRun(ctx, trigger.ID, &advanceTo); err != nil {
|
||||
s.logger.Warn("update trigger next_run_at failed",
|
||||
zap.String("trigger_id", trigger.ID), zap.Error(err))
|
||||
@@ -458,7 +464,7 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin
|
||||
// runCount-1 as missed_ticks ("you missed N since the last fire").
|
||||
missedForPayload = runCount - 1
|
||||
}
|
||||
_, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload)
|
||||
_, err = s.DispatchRoutineRunWithMissed(ctx, routine, trigger, shared.RoutineSourceCron, nil, missedForPayload, claimedTick)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -530,7 +536,7 @@ func (s *RoutineService) DispatchRoutineRun(
|
||||
source string,
|
||||
provided map[string]string,
|
||||
) (*RoutineRun, error) {
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0)
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", 0, nil)
|
||||
}
|
||||
|
||||
// DispatchRoutineRunWithIdempotencyKey dispatches a fire with an explicit
|
||||
@@ -544,7 +550,7 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey(
|
||||
provided map[string]string,
|
||||
idempotencyKey string,
|
||||
) (*RoutineRun, error) {
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0)
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, idempotencyKey, 0, nil)
|
||||
}
|
||||
|
||||
// DispatchRoutineRunWithMissed is the cron-tick entry point that
|
||||
@@ -554,6 +560,12 @@ func (s *RoutineService) DispatchRoutineRunWithIdempotencyKey(
|
||||
// learns "you missed N-1 ticks" via wakeup.RoutinePayload.MissedTicks.
|
||||
// Manual fires (UI / API) call DispatchRoutineRun directly with no
|
||||
// missed-tick attribution.
|
||||
//
|
||||
// claimedTick is the scheduled tick processCronTrigger claimed off the
|
||||
// trigger row before advancing it — the dedup key's occurrence identity
|
||||
// for a cron fire. It travels as a parameter rather than being re-read
|
||||
// from the trigger row, which by dispatch time already names the next
|
||||
// slot.
|
||||
func (s *RoutineService) DispatchRoutineRunWithMissed(
|
||||
ctx context.Context,
|
||||
routine *Routine,
|
||||
@@ -561,8 +573,9 @@ func (s *RoutineService) DispatchRoutineRunWithMissed(
|
||||
source string,
|
||||
provided map[string]string,
|
||||
missedTicks int,
|
||||
claimedTick *time.Time,
|
||||
) (*RoutineRun, error) {
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks)
|
||||
return s.dispatchRoutineRun(ctx, routine, trigger, source, provided, "", missedTicks, claimedTick)
|
||||
}
|
||||
|
||||
func (s *RoutineService) dispatchRoutineRun(
|
||||
@@ -573,6 +586,7 @@ func (s *RoutineService) dispatchRoutineRun(
|
||||
provided map[string]string,
|
||||
idempotencyKey string,
|
||||
missedTicks int,
|
||||
claimedTick *time.Time,
|
||||
) (*RoutineRun, error) {
|
||||
now := time.Now().UTC()
|
||||
defaults := parseDeclaredDefaults(routine.Variables)
|
||||
@@ -640,7 +654,7 @@ func (s *RoutineService) dispatchRoutineRun(
|
||||
return run, nil
|
||||
}
|
||||
|
||||
if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil {
|
||||
if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks, claimedTick); err != nil {
|
||||
return run, err
|
||||
}
|
||||
// A nil error only means materialiseRoutineRun's terminal write
|
||||
@@ -683,11 +697,12 @@ func (s *RoutineService) materialiseRoutineRun(
|
||||
source string,
|
||||
idempotencyKey string,
|
||||
missedTicks int,
|
||||
claimedTick *time.Time,
|
||||
) error {
|
||||
if tmpl.Title != "" && s.workflowEnsurer != nil && s.taskCreator != nil {
|
||||
return s.materialiseHeavyRoutineRun(ctx, routine, run, title, description)
|
||||
}
|
||||
return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks)
|
||||
return s.materialiseLightweightRoutineRun(ctx, routine, run, vars, source, idempotencyKey, missedTicks, claimedTick)
|
||||
}
|
||||
|
||||
// materialiseHeavyRoutineRun creates a real task in the routine system
|
||||
@@ -746,11 +761,12 @@ func (s *RoutineService) materialiseLightweightRoutineRun(
|
||||
source string,
|
||||
idempotencyKey string,
|
||||
missedTicks int,
|
||||
claimedTick *time.Time,
|
||||
) error {
|
||||
if s.wakeup == nil || routine.AssigneeAgentProfileID == "" {
|
||||
return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusDone)
|
||||
}
|
||||
idemKey := buildRoutineIdempotencyKey(routine.ID, run.TriggerID, source, run.ID, idempotencyKey, run.StartedAt)
|
||||
idemKey := buildRoutineIdempotencyKey(source, routine.ID, run.TriggerID, idempotencyKey, claimedTick, run.ID)
|
||||
payloadStr, _ := marshalRoutinePayload(routine.ID, vars, missedTicks)
|
||||
req := &WakeupRequest{
|
||||
ID: uuid.New().String(),
|
||||
@@ -805,27 +821,37 @@ func (s *RoutineService) finalizeLightweightRun(
|
||||
}
|
||||
|
||||
// buildRoutineIdempotencyKey composes the source-level dedup key for a
|
||||
// routine fire. Cron fires use the trigger and minute bucket. Manual and
|
||||
// webhook fires use a unique run identity unless the caller supplies an
|
||||
// explicit request key, so distinct event deliveries never collide.
|
||||
// routine fire. An explicit request key (a webhook delivery header, say)
|
||||
// always wins, so distinct event deliveries never collide. Otherwise the
|
||||
// occurrence identity depends on the source: a cron fire claims a scheduled
|
||||
// slot off the trigger row (ClaimTrigger's compare-and-swap means exactly
|
||||
// one RoutineRun exists per slot, so the slot - not the run - is the
|
||||
// occurrence, and a catch-up collapsing several missed ticks into one fire
|
||||
// is still one occurrence). A manual or webhook fire claims no slot, so
|
||||
// RoutineRun.ID is its only durable distinguishing identity, and two such
|
||||
// fires are two distinct occurrences by design.
|
||||
//
|
||||
// A cron source with no claimed tick, or a source this table does not
|
||||
// recognise, has no occurrence identity to name and goes keyless.
|
||||
func buildRoutineIdempotencyKey(
|
||||
routineID, triggerID, source, runID, explicitKey string, startedAt *time.Time,
|
||||
source, routineID, triggerID, explicitKey string, claimedTick *time.Time, routineRunID string,
|
||||
) string {
|
||||
if explicitKey != "" {
|
||||
return fmt.Sprintf("routine:%s:%s:%s", routineID, source, explicitKey)
|
||||
}
|
||||
if source != shared.RoutineSourceCron {
|
||||
return fmt.Sprintf("routine:%s:%s:%s", routineID, source, runID)
|
||||
switch source {
|
||||
case shared.RoutineSourceCron:
|
||||
if claimedTick == nil {
|
||||
runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "cron_no_claimed_tick")
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("routine:%s:%s:tick:%d", routineID, triggerID, claimedTick.Unix())
|
||||
case "manual", "webhook":
|
||||
return fmt.Sprintf("routine:%s:run:%s", routineID, routineRunID)
|
||||
default:
|
||||
runsservice.ReportKeylessEnqueue(shared.RoutineDispatchReason(source), runsservice.KeylessCauseUnresolved, "unrecognised_routine_source")
|
||||
return ""
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if startedAt != nil {
|
||||
now = *startedAt
|
||||
}
|
||||
minute := now.Unix() / 60
|
||||
if triggerID == "" {
|
||||
return fmt.Sprintf("routine:%s:%s:%d", routineID, source, minute)
|
||||
}
|
||||
return fmt.Sprintf("routine:%s:%s:%d", routineID, triggerID, minute)
|
||||
}
|
||||
|
||||
// marshalRoutinePayload renders the wakeup-request payload for a
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// CommentWriter is the comment mutation dependency used by runtime actions.
|
||||
@@ -254,7 +255,7 @@ type ApprovalRequester interface {
|
||||
|
||||
// RunSpawner is the run queue dependency used by runtime actions.
|
||||
type RunSpawner interface {
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error)
|
||||
}
|
||||
|
||||
// AgentModifier is the agent update dependency used by runtime actions.
|
||||
@@ -565,7 +566,22 @@ type SpawnAgentRunInput struct {
|
||||
IdempotencyKey string `json:"idempotency_key"`
|
||||
}
|
||||
|
||||
// maxSpawnAgentRunReasonLength bounds SpawnAgentRunInput.Reason. Reason is
|
||||
// agent-supplied and reaches office_run_dedup_total /
|
||||
// office_run_dedup_keyless_total as an expvar.Map label; those maps never
|
||||
// evict, so an unbounded Reason would let a caller grow them without limit.
|
||||
const maxSpawnAgentRunReasonLength = 100
|
||||
|
||||
// SpawnAgentRun queues a run for an agent in the same workspace.
|
||||
//
|
||||
// A non-empty agent-supplied key is prefixed with the calling run's id
|
||||
// (agent:<callerRunID>:<key>) so a retry of the same run reuses the run id
|
||||
// and still dedupes, while a later run gets a different prefix and is not
|
||||
// suppressed. With no caller run id the request enqueues keyless
|
||||
// (cause=unresolved) rather than risk colliding across runs. An empty key is
|
||||
// NOT prefixed: the agent expressed no dedup intent (cause=by_design), and
|
||||
// prefixing it would collapse every no-dedup-intent call inside one run onto
|
||||
// a single key, suppressing every call after the first.
|
||||
func (a *Actions) SpawnAgentRun(
|
||||
ctx context.Context,
|
||||
runCtx RunContext,
|
||||
@@ -577,6 +593,9 @@ func (a *Actions) SpawnAgentRun(
|
||||
if a.deps.Runs == nil || a.deps.AgentModifier == nil {
|
||||
return fmt.Errorf("%w: runs", ErrRuntimeDependencyMissing)
|
||||
}
|
||||
if len(input.Reason) > maxSpawnAgentRunReasonLength {
|
||||
return ErrReasonTooLong
|
||||
}
|
||||
target, err := a.deps.AgentModifier.GetAgentInstance(ctx, input.AgentID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -592,7 +611,17 @@ func (a *Actions) SpawnAgentRun(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), input.IdempotencyKey)
|
||||
key := ""
|
||||
switch {
|
||||
case input.IdempotencyKey == "":
|
||||
runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseByDesign, "")
|
||||
case runCtx.RunID != "":
|
||||
key = fmt.Sprintf("agent:%s:%s", runCtx.RunID, input.IdempotencyKey)
|
||||
default:
|
||||
runsservice.ReportKeylessEnqueue(input.Reason, runsservice.KeylessCauseUnresolved, "no_caller_run")
|
||||
}
|
||||
_, err = a.deps.Runs.QueueRun(ctx, target.ID, input.Reason, string(payload), key)
|
||||
return err
|
||||
}
|
||||
|
||||
// ModifyAgentInput contains agent fields an authorized runtime may update.
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
func TestCapabilitiesMarshalProjectCapabilityKeys(t *testing.T) {
|
||||
@@ -1221,14 +1222,14 @@ type spawnRunCall struct {
|
||||
func (r *recordingRunSpawner) QueueRun(
|
||||
_ context.Context,
|
||||
agentInstanceID, reason, payload, idempotencyKey string,
|
||||
) error {
|
||||
) (runsservice.QueueOutcome, error) {
|
||||
r.calls = append(r.calls, spawnRunCall{
|
||||
AgentID: agentInstanceID,
|
||||
Reason: reason,
|
||||
Payload: payload,
|
||||
IdempotencyKey: idempotencyKey,
|
||||
})
|
||||
return nil
|
||||
return runsservice.QueueOutcomeQueued, nil
|
||||
}
|
||||
|
||||
type recordingAgentModifier struct {
|
||||
|
||||
@@ -20,4 +20,10 @@ var (
|
||||
// either) in a workspace that has at least one project to choose from.
|
||||
// Caller-correctable: the caller should retry with an explicit project_id.
|
||||
ErrProjectRequired = fmt.Errorf("project_id is required")
|
||||
// ErrReasonTooLong is returned when SpawnAgentRunInput.Reason exceeds
|
||||
// maxSpawnAgentRunReasonLength. Reason is agent-supplied and becomes a
|
||||
// label on the process-global office_run_dedup_total /
|
||||
// office_run_dedup_keyless_total expvar maps, which never evict entries;
|
||||
// an unbounded value lets a caller grow those maps without limit.
|
||||
ErrReasonTooLong = fmt.Errorf("reason exceeds max length of %d characters", maxSpawnAgentRunReasonLength)
|
||||
)
|
||||
|
||||
@@ -478,7 +478,7 @@ func (h *Handler) respondRuntimeError(
|
||||
targetID string,
|
||||
err error,
|
||||
) {
|
||||
if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) {
|
||||
if errors.Is(err, errTaskTitleRequired) || errors.Is(err, ErrProjectRequired) || errors.Is(err, ErrReasonTooLong) {
|
||||
h.appendDeniedRunEvent(c.Request.Context(), runCtx, action, targetType, targetID, err)
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"expvar"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// spawnAgentRunKeylessCounterHasLabel reports whether the process-global
|
||||
// office_run_dedup_keyless_total expvar map carries an entry for the given
|
||||
// reason and cause. Mirrors internal/runs/service's own counterHasLabel test
|
||||
// helper — that one lives in a different package (service_test) and cannot
|
||||
// be imported here.
|
||||
func spawnAgentRunKeylessCounterHasLabel(t *testing.T, reason, cause string) bool {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_run_dedup_keyless_total")
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map office_run_dedup_keyless_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map")
|
||||
}
|
||||
found := false
|
||||
m.Do(func(kv expvar.KeyValue) {
|
||||
if strings.Contains(kv.Key, "reason="+reason) && strings.Contains(kv.Key, "cause="+cause) {
|
||||
found = true
|
||||
}
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
// TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID is the AC-001.7
|
||||
// regression: a non-empty agent-supplied key is prefixed with the calling
|
||||
// run's id (agent:<callerRunID>:<key>) before reaching QueueRun, so a retry
|
||||
// of the same run reuses the run id and still dedupes, while a later run
|
||||
// (different RunID) produces a different key and is not suppressed. Before
|
||||
// this test, the only coverage touching SpawnAgentRun's key logic was the
|
||||
// unrelated cross-workspace denial test — the prefix itself was unpinned.
|
||||
func TestActionsSpawnAgentRun_PrefixesKeyWithCallerRunID(t *testing.T) {
|
||||
agents := &recordingAgentModifier{
|
||||
agents: map[string]*models.AgentInstance{
|
||||
"agent-2": {ID: "agent-2", WorkspaceID: "ws-1"},
|
||||
},
|
||||
}
|
||||
runs := &recordingRunSpawner{}
|
||||
actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents})
|
||||
runCtx := RunContext{
|
||||
WorkspaceID: "ws-1",
|
||||
RunID: "run-caller-1",
|
||||
Capabilities: Capabilities{
|
||||
CanSpawnAgentRun: true,
|
||||
},
|
||||
}
|
||||
|
||||
if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{
|
||||
AgentID: "agent-2",
|
||||
Reason: "custom_reason",
|
||||
IdempotencyKey: "custom-key",
|
||||
}); err != nil {
|
||||
t.Fatalf("SpawnAgentRun: %v", err)
|
||||
}
|
||||
|
||||
if len(runs.calls) != 1 {
|
||||
t.Fatalf("run spawner calls = %d, want 1", len(runs.calls))
|
||||
}
|
||||
want := "agent:run-caller-1:custom-key"
|
||||
if got := runs.calls[0].IdempotencyKey; got != want {
|
||||
t.Fatalf("idempotency key = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey proves
|
||||
// the prefix's other half: the SAME agent-supplied key from a DIFFERENT
|
||||
// calling run does not collide, because the caller run id varies the key.
|
||||
func TestActionsSpawnAgentRun_DifferentCallerRun_ProducesDifferentKey(t *testing.T) {
|
||||
agents := &recordingAgentModifier{
|
||||
agents: map[string]*models.AgentInstance{
|
||||
"agent-2": {ID: "agent-2", WorkspaceID: "ws-1"},
|
||||
},
|
||||
}
|
||||
runs := &recordingRunSpawner{}
|
||||
actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents})
|
||||
caps := Capabilities{CanSpawnAgentRun: true}
|
||||
|
||||
for _, callerRunID := range []string{"run-a", "run-b"} {
|
||||
runCtx := RunContext{WorkspaceID: "ws-1", RunID: callerRunID, Capabilities: caps}
|
||||
if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{
|
||||
AgentID: "agent-2",
|
||||
Reason: "custom_reason",
|
||||
IdempotencyKey: "same-key",
|
||||
}); err != nil {
|
||||
t.Fatalf("SpawnAgentRun (caller %s): %v", callerRunID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(runs.calls) != 2 {
|
||||
t.Fatalf("run spawner calls = %d, want 2", len(runs.calls))
|
||||
}
|
||||
if runs.calls[0].IdempotencyKey == runs.calls[1].IdempotencyKey {
|
||||
t.Fatalf("two different caller runs produced the same key %q", runs.calls[0].IdempotencyKey)
|
||||
}
|
||||
}
|
||||
|
||||
// TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign proves the third
|
||||
// branch: the agent expressed no dedup intent (empty IdempotencyKey), so the
|
||||
// call enqueues keyless without a prefix — prefixing an empty key would
|
||||
// collapse every no-dedup-intent call inside one run onto a single key,
|
||||
// suppressing every call after the first.
|
||||
func TestActionsSpawnAgentRun_EmptyKey_NotPrefixed_ByDesign(t *testing.T) {
|
||||
agents := &recordingAgentModifier{
|
||||
agents: map[string]*models.AgentInstance{
|
||||
"agent-2": {ID: "agent-2", WorkspaceID: "ws-1"},
|
||||
},
|
||||
}
|
||||
runs := &recordingRunSpawner{}
|
||||
actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents})
|
||||
runCtx := RunContext{
|
||||
WorkspaceID: "ws-1",
|
||||
RunID: "run-caller-2",
|
||||
Capabilities: Capabilities{CanSpawnAgentRun: true},
|
||||
}
|
||||
|
||||
if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{
|
||||
AgentID: "agent-2",
|
||||
Reason: "test_spawn_empty_key",
|
||||
}); err != nil {
|
||||
t.Fatalf("SpawnAgentRun: %v", err)
|
||||
}
|
||||
|
||||
if len(runs.calls) != 1 {
|
||||
t.Fatalf("run spawner calls = %d, want 1", len(runs.calls))
|
||||
}
|
||||
if got := runs.calls[0].IdempotencyKey; got != "" {
|
||||
t.Fatalf("idempotency key = %q, want empty (keyless by design)", got)
|
||||
}
|
||||
// Reason isn't in runs/service's bounded metricReasons allowlist, so it
|
||||
// buckets to "custom" on the label rather than surviving verbatim.
|
||||
if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "by_design") {
|
||||
t.Fatal("expected office_run_dedup_keyless_total to carry a custom/by_design entry")
|
||||
}
|
||||
}
|
||||
|
||||
// TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved proves
|
||||
// the fourth branch: a non-empty agent-supplied key with no caller run id to
|
||||
// prefix it enqueues keyless (cause=unresolved) rather than risk colliding
|
||||
// across unrelated runs.
|
||||
func TestActionsSpawnAgentRun_NoCallerRunID_EnqueuesKeylessUnresolved(t *testing.T) {
|
||||
agents := &recordingAgentModifier{
|
||||
agents: map[string]*models.AgentInstance{
|
||||
"agent-2": {ID: "agent-2", WorkspaceID: "ws-1"},
|
||||
},
|
||||
}
|
||||
runs := &recordingRunSpawner{}
|
||||
actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents})
|
||||
runCtx := RunContext{
|
||||
WorkspaceID: "ws-1",
|
||||
RunID: "",
|
||||
Capabilities: Capabilities{CanSpawnAgentRun: true},
|
||||
}
|
||||
|
||||
if err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{
|
||||
AgentID: "agent-2",
|
||||
Reason: "test_spawn_no_caller_run",
|
||||
IdempotencyKey: "custom-key",
|
||||
}); err != nil {
|
||||
t.Fatalf("SpawnAgentRun: %v", err)
|
||||
}
|
||||
|
||||
if len(runs.calls) != 1 {
|
||||
t.Fatalf("run spawner calls = %d, want 1", len(runs.calls))
|
||||
}
|
||||
if got := runs.calls[0].IdempotencyKey; got != "" {
|
||||
t.Fatalf("idempotency key = %q, want empty (keyless, no caller run to prefix with)", got)
|
||||
}
|
||||
// Reason isn't in runs/service's bounded metricReasons allowlist, so it
|
||||
// buckets to "custom" on the label rather than surviving verbatim.
|
||||
if !spawnAgentRunKeylessCounterHasLabel(t, "custom", "unresolved") {
|
||||
t.Fatal("expected office_run_dedup_keyless_total to carry a custom/unresolved entry")
|
||||
}
|
||||
}
|
||||
|
||||
// TestActionsSpawnAgentRun_ReasonTooLong_Rejected proves Reason is bounded
|
||||
// before it can reach office_run_dedup_total /
|
||||
// office_run_dedup_keyless_total as an expvar.Map label: those maps never
|
||||
// evict entries, so an unvalidated agent-supplied Reason would let a caller
|
||||
// grow them without limit. An over-length Reason is rejected with
|
||||
// ErrReasonTooLong and never reaches the run spawner.
|
||||
func TestActionsSpawnAgentRun_ReasonTooLong_Rejected(t *testing.T) {
|
||||
agents := &recordingAgentModifier{
|
||||
agents: map[string]*models.AgentInstance{
|
||||
"agent-2": {ID: "agent-2", WorkspaceID: "ws-1"},
|
||||
},
|
||||
}
|
||||
runs := &recordingRunSpawner{}
|
||||
actions := NewActions(ActionDependencies{Runs: runs, AgentModifier: agents})
|
||||
runCtx := RunContext{
|
||||
WorkspaceID: "ws-1",
|
||||
RunID: "run-caller-3",
|
||||
Capabilities: Capabilities{CanSpawnAgentRun: true},
|
||||
}
|
||||
|
||||
err := actions.SpawnAgentRun(context.Background(), runCtx, SpawnAgentRunInput{
|
||||
AgentID: "agent-2",
|
||||
Reason: strings.Repeat("a", maxSpawnAgentRunReasonLength+1),
|
||||
})
|
||||
if !errors.Is(err, ErrReasonTooLong) {
|
||||
t.Fatalf("SpawnAgentRun error = %v, want ErrReasonTooLong", err)
|
||||
}
|
||||
if len(runs.calls) != 0 {
|
||||
t.Fatalf("run spawner calls = %d, want 0 (rejected before enqueue)", len(runs.calls))
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ func (a *DashboardApprovalAdapter) QueueApprovalRuns(
|
||||
DecisionComment: w.DecisionComment,
|
||||
IdempotencyKey: w.IdempotencyKey,
|
||||
}
|
||||
if err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil {
|
||||
if _, err := a.scheduler.QueueRunCtx(ctx, w.AgentID, c); err != nil {
|
||||
a.scheduler.logger.Warn("approval run failed: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,12 +75,13 @@ func (a *DashboardReactivityAdapter) ApplyTaskMutation(
|
||||
// two packages.
|
||||
func convertChangeToMutation(c dashboard.TaskReactivityChange) TaskMutation {
|
||||
out := TaskMutation{
|
||||
NewStatus: c.NewStatus,
|
||||
NewAssigneeID: c.NewAssigneeID,
|
||||
ReopenIntent: c.ReopenIntent,
|
||||
ResumeIntent: c.ResumeIntent,
|
||||
ActorID: c.ActorID,
|
||||
ActorType: c.ActorType,
|
||||
NewStatus: c.NewStatus,
|
||||
NewAssigneeID: c.NewAssigneeID,
|
||||
AssignmentGeneration: c.AssignmentGeneration,
|
||||
ReopenIntent: c.ReopenIntent,
|
||||
ResumeIntent: c.ResumeIntent,
|
||||
ActorID: c.ActorID,
|
||||
ActorType: c.ActorType,
|
||||
}
|
||||
if c.Comment != nil {
|
||||
out.Comment = &MutationComment{
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// windowedDedupCounterValue reads the current value of one
|
||||
// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the
|
||||
// label has never been reported. Callers snapshot before/after and assert
|
||||
// the exact delta.
|
||||
func windowedDedupCounterValue(t *testing.T, reason string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_run_dedup_total")
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map office_run_dedup_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("office_run_dedup_total is not a *expvar.Map")
|
||||
}
|
||||
iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs")
|
||||
if iv == nil {
|
||||
return 0
|
||||
}
|
||||
i, ok := iv.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason)
|
||||
}
|
||||
return i.Value()
|
||||
}
|
||||
|
||||
// TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome closes a
|
||||
// phantom-green gap on the office-owned queue implementation the design
|
||||
// names as least observable (run-dedup-generation-02.md#test-strategy):
|
||||
// no test asserted SchedulerService.QueueRun actually calls
|
||||
// ReportWindowedDedup, so deleting that call would leave every existing
|
||||
// test green while AC-OFFICE-RUN-DEDUP-004.1 silently went unmet on the
|
||||
// path a reassignment travels.
|
||||
func TestSchedulerService_QueueRun_ReportsWindowedDedupOutcome(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-queue-run-outcome")
|
||||
ctx := context.Background()
|
||||
|
||||
key := "idem-key-scheduler-outcome"
|
||||
first, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key)
|
||||
if err != nil {
|
||||
t.Fatalf("first enqueue: %v", err)
|
||||
}
|
||||
if first != runsservice.QueueOutcomeQueued {
|
||||
t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first)
|
||||
}
|
||||
|
||||
before := windowedDedupCounterValue(t, RunReasonTaskAssigned)
|
||||
|
||||
second, err := ss.QueueRun(ctx, "agent-queue-run-outcome", RunReasonTaskAssigned, "{}", key)
|
||||
if err != nil {
|
||||
t.Fatalf("second enqueue: %v", err)
|
||||
}
|
||||
if second != runsservice.QueueOutcomeDeduped {
|
||||
t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second)
|
||||
}
|
||||
|
||||
after := windowedDedupCounterValue(t, RunReasonTaskAssigned)
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", RunReasonTaskAssigned, after, before+1)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/runs/commentkeys"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// Canonical lowercase status values used inside the pipeline. Backend
|
||||
@@ -46,10 +49,17 @@ type TaskMutation struct {
|
||||
// What's changing.
|
||||
NewStatus *string // nil = unchanged
|
||||
NewAssigneeID *string
|
||||
NewPriority *string
|
||||
Comment *MutationComment // user/agent comment if this mutation includes one
|
||||
ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress)
|
||||
ResumeIntent bool // explicit resume=true (validated upstream to require comment)
|
||||
// AssignmentGeneration is the value the assigning transaction committed,
|
||||
// carried here rather than re-read (a producer reading the task's
|
||||
// current generation after the fact can observe a later occurrence's
|
||||
// value — see docs/specs/office/system-design/run-dedup-generation-01.md
|
||||
// #carrying-the-generation). Nil means the caller could not supply one;
|
||||
// reactToAssigneeChange then enqueues keyless rather than guess.
|
||||
AssignmentGeneration *int64
|
||||
NewPriority *string
|
||||
Comment *MutationComment // user/agent comment if this mutation includes one
|
||||
ReopenIntent bool // explicit reopen=true (or status moves done|cancelled → todo|in_progress)
|
||||
ResumeIntent bool // explicit resume=true (validated upstream to require comment)
|
||||
|
||||
// Who is acting.
|
||||
ActorID string
|
||||
@@ -115,13 +125,17 @@ func (ss *SchedulerService) ApplyTaskMutation(
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
if err := ss.QueueRunCtx(ctx, agentID, c); err != nil {
|
||||
outcome, err := ss.QueueRunCtx(ctx, agentID, c)
|
||||
if err != nil {
|
||||
ss.logger.Error("reactivity run failed",
|
||||
zap.String("agent", agentID),
|
||||
zap.String("reason", c.Reason),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
if outcome != runsservice.QueueOutcomeQueued {
|
||||
return
|
||||
}
|
||||
res.Runs = append(res.Runs, QueuedRunSummary{
|
||||
AgentID: agentID, Reason: c.Reason, TaskID: c.TaskID,
|
||||
})
|
||||
@@ -133,7 +147,12 @@ func (ss *SchedulerService) ApplyTaskMutation(
|
||||
}
|
||||
|
||||
// --- Assignee handoff ---
|
||||
if change.NewAssigneeID != nil && *change.NewAssigneeID != task.AssigneeAgentProfileID {
|
||||
// Fires on every non-nil NewAssigneeID, including a repeat assignment to
|
||||
// the agent that already holds the seat: that is a real occurrence (the
|
||||
// operator is asking for the work again), not a no-op, so this no longer
|
||||
// gates on the two ids differing. reactToAssigneeChange itself guards the
|
||||
// session interrupt.
|
||||
if change.NewAssigneeID != nil {
|
||||
ss.reactToAssigneeChange(task, *change.NewAssigneeID, change, queue, res)
|
||||
}
|
||||
|
||||
@@ -180,7 +199,13 @@ func (ss *SchedulerService) reactToStatusChange(
|
||||
ss.cascadeReviewRequested(ctx, task, change, queue)
|
||||
|
||||
case prev == statusBlocked && next != statusBlocked:
|
||||
// Unblocked — wake assignee with task_unblocked.
|
||||
// Unblocked — no durable occurrence row to key on: keyless by design.
|
||||
// Reported only when there's an assignee to enqueue for — queue()
|
||||
// silently drops an empty agent id, so reporting unconditionally
|
||||
// would count an enqueue that never happened.
|
||||
if task.AssigneeAgentProfileID != "" {
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskUnblocked, runsservice.KeylessCauseByDesign, "")
|
||||
}
|
||||
queue(task.AssigneeAgentProfileID, RunContext{
|
||||
Reason: RunReasonTaskUnblocked,
|
||||
TaskID: task.ID,
|
||||
@@ -190,7 +215,7 @@ func (ss *SchedulerService) reactToStatusChange(
|
||||
})
|
||||
|
||||
case (prev == statusDone || prev == statusCancelled) && (next == statusTodo || next == statusInProgress):
|
||||
// Reopen — different reason if a comment was attached.
|
||||
// Reopen — different reason (and dedup identity) if a comment was attached.
|
||||
reason := RunReasonTaskReopened
|
||||
commentID := ""
|
||||
if change.Comment != nil {
|
||||
@@ -201,13 +226,23 @@ func (ss *SchedulerService) reactToStatusChange(
|
||||
// Explicit resume always uses the comment-flavoured reason.
|
||||
reason = RunReasonTaskReopenedComment
|
||||
}
|
||||
var key string
|
||||
if reason == RunReasonTaskReopenedComment {
|
||||
key = fmt.Sprintf("%s:%s:%s", RunReasonTaskReopenedComment, commentID, task.AssigneeAgentProfileID)
|
||||
} else if task.AssigneeAgentProfileID != "" {
|
||||
// Silent reopen (no comment) — no durable occurrence row: keyless
|
||||
// by design. Reported only when there's an assignee to enqueue
|
||||
// for, matching the task_unblocked case above.
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskReopened, runsservice.KeylessCauseByDesign, "")
|
||||
}
|
||||
queue(task.AssigneeAgentProfileID, RunContext{
|
||||
Reason: reason,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: change.ActorID,
|
||||
ActorType: change.ActorType,
|
||||
CommentID: commentID,
|
||||
Reason: reason,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: change.ActorID,
|
||||
ActorType: change.ActorType,
|
||||
CommentID: commentID,
|
||||
IdempotencyKey: key,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -222,6 +257,13 @@ func (ss *SchedulerService) reactToStatusChange(
|
||||
//
|
||||
// The previous assignee is NOT separately notified — interrupting their
|
||||
// run is the signal that they're no longer in charge.
|
||||
//
|
||||
// ApplyTaskMutation now calls this for every non-nil NewAssigneeID,
|
||||
// including a repeat assignment to the agent that already holds the seat.
|
||||
// The interrupt must NOT fire for that case — it would hard-cancel the
|
||||
// agent's own in-flight run — so unlike the removed caller-side equality
|
||||
// gate, this comparison stays local to the interrupt decision and does not
|
||||
// also guard the wake.
|
||||
func (ss *SchedulerService) reactToAssigneeChange(
|
||||
task *TaskSnapshot,
|
||||
newAssigneeID string,
|
||||
@@ -229,25 +271,36 @@ func (ss *SchedulerService) reactToAssigneeChange(
|
||||
queue func(string, RunContext),
|
||||
res *ApplyTaskMutationResult,
|
||||
) {
|
||||
// Cancel the prior assignee's session if there was one. We re-use
|
||||
// InterruptSessionID — status→cancelled also sets it; either reason
|
||||
// for hard-cancelling produces the same downstream call.
|
||||
if task.AssigneeAgentProfileID != "" && res.InterruptSessionID == "" {
|
||||
if task.AssigneeAgentProfileID != "" && newAssigneeID != task.AssigneeAgentProfileID {
|
||||
res.InterruptSessionID = task.ID
|
||||
}
|
||||
|
||||
// Wake the new assignee.
|
||||
if newAssigneeID == "" {
|
||||
// Unassignment: the interrupt above (if any) already fired; queue's
|
||||
// own empty-agent-id guard would catch this too, but there is no key
|
||||
// to build or keyless cause to report for a run that will never be
|
||||
// attempted.
|
||||
return
|
||||
}
|
||||
|
||||
commentID := ""
|
||||
if change.Comment != nil {
|
||||
commentID = change.Comment.ID
|
||||
}
|
||||
var key string
|
||||
if change.AssignmentGeneration != nil {
|
||||
key = dedupkeys.AssignmentKey(task.ID, newAssigneeID, *change.AssignmentGeneration)
|
||||
} else {
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "nil_mutation_generation")
|
||||
}
|
||||
queue(newAssigneeID, RunContext{
|
||||
Reason: RunReasonTaskAssigned,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: change.ActorID,
|
||||
ActorType: change.ActorType,
|
||||
CommentID: commentID,
|
||||
Reason: RunReasonTaskAssigned,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: change.ActorID,
|
||||
ActorType: change.ActorType,
|
||||
CommentID: commentID,
|
||||
IdempotencyKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -262,13 +315,15 @@ func (ss *SchedulerService) reactToComment(
|
||||
|
||||
// Assignee wake — skip if self-comment or task is closed.
|
||||
if !comment.SkipAssigneeWake && !selfComment && !closed {
|
||||
key := commentkeys.TaskComment(comment.ID) + ":" + task.AssigneeAgentProfileID
|
||||
queue(task.AssigneeAgentProfileID, RunContext{
|
||||
Reason: RunReasonTaskComment,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: comment.AuthorID,
|
||||
ActorType: comment.AuthorType,
|
||||
CommentID: comment.ID,
|
||||
Reason: RunReasonTaskComment,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: comment.AuthorID,
|
||||
ActorType: comment.AuthorType,
|
||||
CommentID: comment.ID,
|
||||
IdempotencyKey: key,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -286,13 +341,15 @@ func (ss *SchedulerService) reactToComment(
|
||||
if agentID == comment.AuthorID {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s:%s:%s", RunReasonTaskMentioned, comment.ID, agentID)
|
||||
queue(agentID, RunContext{
|
||||
Reason: RunReasonTaskMentioned,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: comment.AuthorID,
|
||||
ActorType: comment.AuthorType,
|
||||
CommentID: comment.ID,
|
||||
Reason: RunReasonTaskMentioned,
|
||||
TaskID: task.ID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ActorID: comment.AuthorID,
|
||||
ActorType: comment.AuthorType,
|
||||
CommentID: comment.ID,
|
||||
IdempotencyKey: key,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -320,6 +377,7 @@ func (ss *SchedulerService) cascadeReviewRequested(
|
||||
zap.String("task_id", task.ID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
reported := map[string]struct{}{}
|
||||
for _, p := range parts {
|
||||
if p.AgentProfileID == "" {
|
||||
continue
|
||||
@@ -327,6 +385,14 @@ func (ss *SchedulerService) cascadeReviewRequested(
|
||||
if p.Role != models.ParticipantRoleReviewer && p.Role != models.ParticipantRoleApprover {
|
||||
continue
|
||||
}
|
||||
// No durable occurrence row to key on: keyless by design, reported
|
||||
// once per distinct agent rather than per role — an agent seated as
|
||||
// both reviewer and approver is one recipient, and queue()'s own
|
||||
// seen-map dedup only attempts one enqueue for it.
|
||||
if _, dup := reported[p.AgentProfileID]; !dup {
|
||||
reported[p.AgentProfileID] = struct{}{}
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskReviewRequested, runsservice.KeylessCauseByDesign, "")
|
||||
}
|
||||
queue(p.AgentProfileID, RunContext{
|
||||
Reason: RunReasonTaskReviewRequested,
|
||||
TaskID: task.ID,
|
||||
@@ -352,19 +418,34 @@ func (ss *SchedulerService) cascadeBlockersResolved(
|
||||
}
|
||||
for _, blockedID := range blockedTaskIDs {
|
||||
// Verify all OTHER blockers are also resolved.
|
||||
ready, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID)
|
||||
if err != nil || !ready {
|
||||
ready, blockers, err := ss.allBlockersResolvedExcept(ctx, blockedID, task.ID)
|
||||
if err != nil {
|
||||
ss.logger.Error("check blockers resolved failed",
|
||||
zap.String("task_id", blockedID), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if !ready {
|
||||
continue
|
||||
}
|
||||
assignee, err := ss.repo.GetTaskAssignee(ctx, blockedID)
|
||||
if err != nil || assignee == "" {
|
||||
continue
|
||||
}
|
||||
blockerIDs := make([]string, 0, len(blockers))
|
||||
for _, b := range blockers {
|
||||
blockerIDs = append(blockerIDs, b.BlockerTaskID)
|
||||
}
|
||||
if len(blockerIDs) == 0 {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s:%s:%s:%s",
|
||||
RunReasonTaskBlockersResolved, blockedID, assignee, dedupkeys.BlockerDigest(blockerIDs))
|
||||
queue(assignee, RunContext{
|
||||
Reason: RunReasonTaskBlockersResolved,
|
||||
TaskID: blockedID,
|
||||
WorkspaceID: task.WorkspaceID,
|
||||
ResolvedBlockerTaskID: task.ID,
|
||||
IdempotencyKey: key,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -443,13 +524,17 @@ func childrenCompletedIdempotencyKey(parentID, agentID string, children []sqlite
|
||||
}
|
||||
|
||||
// allBlockersResolvedExcept returns true if every blocker on `taskID`
|
||||
// other than `excludeBlockerID` is in a terminal step.
|
||||
// other than `excludeBlockerID` is in a terminal step, alongside the full
|
||||
// blocker-task-id set it read to decide — the caller digests that same
|
||||
// slice for the wake's dedup key rather than re-reading it (AC-001.9
|
||||
// applied to a set: a second read could observe a different set than the
|
||||
// one this readiness decision was actually made against).
|
||||
func (ss *SchedulerService) allBlockersResolvedExcept(
|
||||
ctx context.Context, taskID, excludeBlockerID string,
|
||||
) (bool, error) {
|
||||
) (bool, []*models.TaskBlocker, error) {
|
||||
blockers, err := ss.repo.ListTaskBlockers(ctx, taskID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
for _, b := range blockers {
|
||||
if b.BlockerTaskID == excludeBlockerID {
|
||||
@@ -457,13 +542,13 @@ func (ss *SchedulerService) allBlockersResolvedExcept(
|
||||
}
|
||||
done, err := ss.repo.IsTaskInTerminalStep(ctx, b.BlockerTaskID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, nil, err
|
||||
}
|
||||
if !done {
|
||||
return false, nil
|
||||
return false, blockers, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
return true, blockers, nil
|
||||
}
|
||||
|
||||
// normalisedStatus maps both backend uppercase task states (TODO,
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt
|
||||
// pins the reactivity gate relaxation this feature made: ApplyTaskMutation
|
||||
// now calls reactToAssigneeChange for EVERY non-nil NewAssigneeID, including
|
||||
// a repeat assignment to the agent that already holds the seat, because that
|
||||
// is a real occurrence (the operator asking for the work again) rather than
|
||||
// a no-op. Before this feature the caller-side equality gate suppressed it
|
||||
// entirely. The relocated interrupt-comparison guard must still hold: a
|
||||
// same-agent repeat must NOT hard-cancel the agent's own in-flight session.
|
||||
// Before this test, ApplyTaskMutation/reactToStatusChange/
|
||||
// reactToAssigneeChange were at 0.0% coverage.
|
||||
func TestApplyTaskMutation_SameAgentRepeatAssignment_QueuesWithoutInterrupt(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-repeat")
|
||||
|
||||
task := &TaskSnapshot{
|
||||
ID: "task-repeat-assign",
|
||||
WorkspaceID: "ws-1",
|
||||
AssigneeAgentProfileID: "agent-repeat", // already holds the seat
|
||||
}
|
||||
newAssignee := "agent-repeat"
|
||||
gen := int64(1)
|
||||
change := TaskMutation{
|
||||
NewAssigneeID: &newAssignee,
|
||||
AssignmentGeneration: &gen,
|
||||
ActorID: "user-1",
|
||||
ActorType: "user",
|
||||
}
|
||||
|
||||
res, err := ss.ApplyTaskMutation(context.Background(), task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
if res.InterruptSessionID != "" {
|
||||
t.Fatalf("same-agent repeat assignment set InterruptSessionID = %q, want empty (must not cancel its own run)", res.InterruptSessionID)
|
||||
}
|
||||
found := false
|
||||
for _, r := range res.Runs {
|
||||
if r.AgentID == "agent-repeat" && r.Reason == RunReasonTaskAssigned {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected a task_assigned run queued for agent-repeat, got %+v", res.Runs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee
|
||||
// is the positive counterpart: reassigning to a DIFFERENT agent must still
|
||||
// hard-cancel the previous assignee's session — the interrupt guard's
|
||||
// comparison (moved inside reactToAssigneeChange by this feature) must not
|
||||
// have accidentally suppressed the case it was already handling.
|
||||
func TestApplyTaskMutation_DifferentAgentReassignment_InterruptsPreviousAssignee(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-old")
|
||||
createChildrenCompletedAgent(t, repo, "agent-new")
|
||||
|
||||
task := &TaskSnapshot{
|
||||
ID: "task-reassign",
|
||||
WorkspaceID: "ws-1",
|
||||
AssigneeAgentProfileID: "agent-old",
|
||||
}
|
||||
newAssignee := "agent-new"
|
||||
gen := int64(1)
|
||||
change := TaskMutation{
|
||||
NewAssigneeID: &newAssignee,
|
||||
AssignmentGeneration: &gen,
|
||||
ActorID: "user-1",
|
||||
ActorType: "user",
|
||||
}
|
||||
|
||||
res, err := ss.ApplyTaskMutation(context.Background(), task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
if res.InterruptSessionID != task.ID {
|
||||
t.Fatalf("InterruptSessionID = %q, want %q (previous assignee's session must be cancelled)", res.InterruptSessionID, task.ID)
|
||||
}
|
||||
found := false
|
||||
for _, r := range res.Runs {
|
||||
if r.AgentID == "agent-new" && r.Reason == RunReasonTaskAssigned {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected a task_assigned run queued for agent-new, got %+v", res.Runs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyTaskMutation_StatusChangeAndCommentDispatch drives
|
||||
// ApplyTaskMutation's NewStatus and Comment branches together (an unblock
|
||||
// plus an attached comment), covering the dispatcher itself rather than only
|
||||
// the reactToStatusChange/reactToComment helpers it delegates to.
|
||||
func TestApplyTaskMutation_StatusChangeAndCommentDispatch(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-dispatch")
|
||||
|
||||
task := &TaskSnapshot{
|
||||
ID: "task-dispatch",
|
||||
WorkspaceID: "ws-1",
|
||||
State: "BLOCKED",
|
||||
AssigneeAgentProfileID: "agent-dispatch",
|
||||
}
|
||||
newStatus := "todo"
|
||||
change := TaskMutation{
|
||||
NewStatus: &newStatus,
|
||||
Comment: &MutationComment{ID: "comment-dispatch", AuthorType: "user", AuthorID: "user-1"},
|
||||
ActorID: "user-1",
|
||||
ActorType: "user",
|
||||
}
|
||||
|
||||
res, err := ss.ApplyTaskMutation(context.Background(), task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
reasons := map[string]bool{}
|
||||
for _, r := range res.Runs {
|
||||
reasons[r.Reason] = true
|
||||
}
|
||||
if !reasons[RunReasonTaskUnblocked] {
|
||||
t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskUnblocked, res.Runs)
|
||||
}
|
||||
if !reasons[RunReasonTaskComment] {
|
||||
t.Fatalf("expected %s among dispatched runs, got %+v", RunReasonTaskComment, res.Runs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
)
|
||||
|
||||
// insertBlockerRelationship records that taskID is blocked by
|
||||
// blockerTaskID, and gives blockerTaskID the given (already-terminal or
|
||||
// not) state.
|
||||
func insertBlockerRelationship(
|
||||
t *testing.T, repo *officesqlite.Repository, taskID, blockerTaskID, blockerState string,
|
||||
) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`,
|
||||
blockerTaskID, blockerState,
|
||||
); err != nil {
|
||||
t.Fatalf("insert blocker task %s: %v", blockerTaskID, err)
|
||||
}
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`INSERT INTO task_blockers (task_id, blocker_task_id, created_at) VALUES (?, ?, CURRENT_TIMESTAMP)`,
|
||||
taskID, blockerTaskID,
|
||||
); err != nil {
|
||||
t.Fatalf("insert task_blockers row (%s blocked by %s): %v", taskID, blockerTaskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey is the
|
||||
// regression test for both blocker-resolution producers converging on the
|
||||
// same key shape (AC-OFFICE-RUN-DEDUP-002.1, SR-59's resolution: only the
|
||||
// digest need be byte-identical, not the whole key/opID — see
|
||||
// docs/specs/office/system-design/run-dedup-generation-03.md#the-shared-key-builders).
|
||||
// Before this test, office/scheduler.cascadeBlockersResolved and
|
||||
// allBlockersResolvedExcept had zero coverage: nothing exercised the
|
||||
// blocked task's wake at all, let alone the specific
|
||||
// dedupkeys.BlockerDigest call.
|
||||
func TestCascadeBlockersResolved_QueuesRunWithSharedDigestKey(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-1")
|
||||
queue := newChildrenCompletedQueue(t, ss)
|
||||
ctx := context.Background()
|
||||
|
||||
// blocked-1 is blocked by blocker-a (already done) and blocker-b (the
|
||||
// one resolving now). setupChildrenCompletedParent gives blocked-1 a
|
||||
// runner participant row so GetTaskAssignee resolves it to agent-1 —
|
||||
// reused here even though it's named for the children-completed tests,
|
||||
// because it does exactly what a blocked task's assignee wiring needs.
|
||||
setupChildrenCompletedParent(t, ss, "blocked-1", "agent-1")
|
||||
insertBlockerRelationship(t, repo, "blocked-1", "blocker-a", "COMPLETED")
|
||||
insertBlockerRelationship(t, repo, "blocked-1", "blocker-b", "IN_PROGRESS")
|
||||
|
||||
// blocker-b finishes: it's the second and last outstanding blocker.
|
||||
if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-b'`); err != nil {
|
||||
t.Fatalf("complete blocker-b: %v", err)
|
||||
}
|
||||
ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-b", WorkspaceID: "ws-1"}, queue)
|
||||
|
||||
runs, err := repo.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
var got *string
|
||||
for _, run := range runs {
|
||||
if run.Reason == RunReasonTaskBlockersResolved && run.AgentProfileID == "agent-1" {
|
||||
got = run.IdempotencyKey
|
||||
break
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatalf("no %s run persisted for agent-1: runs=%#v", RunReasonTaskBlockersResolved, runs)
|
||||
}
|
||||
|
||||
// The key's digest segment must be exactly what the OTHER blocker
|
||||
// producer (office/service.resolveAndWakeIfUnblocked) would derive
|
||||
// for the identical blocker set, in either read order — that's the
|
||||
// structural convergence AC-002.1 requires.
|
||||
digestAscending := dedupkeys.BlockerDigest([]string{"blocker-a", "blocker-b"})
|
||||
digestDescending := dedupkeys.BlockerDigest([]string{"blocker-b", "blocker-a"})
|
||||
if digestAscending != digestDescending {
|
||||
t.Fatalf("BlockerDigest is not order-independent: %q vs %q", digestAscending, digestDescending)
|
||||
}
|
||||
want := fmt.Sprintf("%s:blocked-1:agent-1:%s", RunReasonTaskBlockersResolved, digestAscending)
|
||||
if *got != want {
|
||||
t.Fatalf("idempotency key = %q, want %q", *got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCascadeBlockersResolved_StillBlocked_DoesNotQueue proves the
|
||||
// negative: cascadeBlockersResolved must not wake the blocked task while
|
||||
// another blocker is still outstanding.
|
||||
func TestCascadeBlockersResolved_StillBlocked_DoesNotQueue(t *testing.T) {
|
||||
repo := newReactivityTestRepo(t)
|
||||
ss := newChildrenCompletedTestScheduler(t, repo)
|
||||
createChildrenCompletedAgent(t, repo, "agent-1")
|
||||
queue := newChildrenCompletedQueue(t, ss)
|
||||
ctx := context.Background()
|
||||
|
||||
setupChildrenCompletedParent(t, ss, "blocked-2", "agent-1")
|
||||
insertBlockerRelationship(t, repo, "blocked-2", "blocker-c", "IN_PROGRESS") // never finishes
|
||||
insertBlockerRelationship(t, repo, "blocked-2", "blocker-d", "IN_PROGRESS")
|
||||
|
||||
if _, err := repo.ExecRaw(ctx, `UPDATE tasks SET state = 'COMPLETED' WHERE id = 'blocker-d'`); err != nil {
|
||||
t.Fatalf("complete blocker-d: %v", err)
|
||||
}
|
||||
ss.cascadeBlockersResolved(ctx, &TaskSnapshot{ID: "blocker-d", WorkspaceID: "ws-1"}, queue)
|
||||
|
||||
if got := runsCountForReason(t, ss, RunReasonTaskBlockersResolved); got != 0 {
|
||||
t.Fatalf("persisted runs = %d, want 0 (blocker-c is still unresolved)", got)
|
||||
}
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func createChildrenCompletedAgent(t *testing.T, repo *officesqlite.Repository, i
|
||||
func newChildrenCompletedQueue(t *testing.T, ss *SchedulerService) func(string, RunContext) {
|
||||
t.Helper()
|
||||
return func(agentID string, c RunContext) {
|
||||
if err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil {
|
||||
if _, err := ss.QueueRunCtx(context.Background(), agentID, c); err != nil {
|
||||
t.Fatalf("QueueRunCtx: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/runs/commentkeys"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
)
|
||||
|
||||
// newAuditScheduler wires a real repo + real *service.Service (reused from
|
||||
// reactivity_children_completed_test.go's newChildrenCompletedTestScheduler)
|
||||
// so every producer under audit — including the two (task_blockers_resolved,
|
||||
// task_review_requested) that read participants/blockers off the repo, and
|
||||
// task_mentioned, which resolves mentions via ss.svc.ListAgentInstances —
|
||||
// can run unmodified.
|
||||
func newAuditScheduler(t *testing.T) (*SchedulerService, *officesqlite.Repository) {
|
||||
t.Helper()
|
||||
repo := newReactivityTestRepo(t)
|
||||
return newChildrenCompletedTestScheduler(t, repo), repo
|
||||
}
|
||||
|
||||
// requireOneCallKey finds the single recorded queue call for (reason, agentID)
|
||||
// and returns its IdempotencyKey, failing the test if no such call exists.
|
||||
func requireOneCallKey(t *testing.T, calls []recordedQueueCall, reason, agentID string) string {
|
||||
t.Helper()
|
||||
for _, c := range calls {
|
||||
if c.agentID == agentID && c.ctx.Reason == reason {
|
||||
return c.ctx.IdempotencyKey
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls)
|
||||
return ""
|
||||
}
|
||||
|
||||
// requireOneCallKeyless is requireOneCallKey's keyless counterpart: it
|
||||
// asserts the matching call carries NO idempotency key, rather than
|
||||
// asserting a value.
|
||||
func requireOneCallKeyless(t *testing.T, calls []recordedQueueCall, reason, agentID string) {
|
||||
t.Helper()
|
||||
for _, c := range calls {
|
||||
if c.agentID == agentID && c.ctx.Reason == reason {
|
||||
if c.ctx.IdempotencyKey != "" {
|
||||
t.Fatalf("%s call for agent %q carries key %q, want keyless", reason, agentID, c.ctx.IdempotencyKey)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s call recorded for agent %q: calls=%#v", reason, agentID, calls)
|
||||
}
|
||||
|
||||
// auditKeylessCounterValue returns the current value of the process-global
|
||||
// office_run_dedup_keyless_total expvar map (internal/runs/service) for the
|
||||
// exact "reason=<reason>;cause=<cause>" label, or 0 if that label has never
|
||||
// been incremented. Duplicated locally rather than imported: the counter map
|
||||
// is unexported and the sibling helper in internal/runs/service/dedup_test.go
|
||||
// lives in a different package. Callers snapshot before and after a producer
|
||||
// call and assert the delta, rather than merely that the label exists — a
|
||||
// mere-existence check would pass even if this specific call site failed to
|
||||
// report, as long as some other test already set that label.
|
||||
func auditKeylessCounterValue(t *testing.T, reason, cause string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_run_dedup_keyless_total")
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map office_run_dedup_keyless_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("office_run_dedup_keyless_total is not a *expvar.Map")
|
||||
}
|
||||
iv := m.Get("reason=" + reason + ";cause=" + cause)
|
||||
if iv == nil {
|
||||
return 0
|
||||
}
|
||||
i, ok := iv.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("counter value for reason=%s;cause=%s is not *expvar.Int", reason, cause)
|
||||
}
|
||||
return i.Value()
|
||||
}
|
||||
|
||||
// TestReactivityProducerKeyAudit is the Part 2-required "producer-audit
|
||||
// key-format table test" (docs/specs/office/system-design/
|
||||
// run-dedup-generation-02.md#test-strategy): a table over every reactivity
|
||||
// producer named in the "Generation sources per producer" table
|
||||
// (…-01.md#generation-sources-per-producer), asserting each one derives the
|
||||
// spec's exact key shape, or reports the spec's exact keyless cause when the
|
||||
// reason has no durable occurrence to key on. Review round 1 found 6 of this
|
||||
// file's 8 run reasons silently doing neither — this is the design's own
|
||||
// named enforcement mechanism whose absence is why that gap shipped
|
||||
// undetected, and what stops the next reason added to reactivity.go from
|
||||
// reintroducing it silently.
|
||||
func TestReactivityProducerKeyAudit(t *testing.T) {
|
||||
t.Run("task_assigned", func(t *testing.T) {
|
||||
ss, _ := newAuditScheduler(t)
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
task := &TaskSnapshot{ID: "task-audit-assigned", WorkspaceID: "ws-1"}
|
||||
gen := int64(3)
|
||||
res := &ApplyTaskMutationResult{}
|
||||
ss.reactToAssigneeChange(task, "agent-assigned", TaskMutation{
|
||||
AssignmentGeneration: &gen, ActorID: "user-1", ActorType: "user",
|
||||
}, queue, res)
|
||||
|
||||
key := requireOneCallKey(t, calls, RunReasonTaskAssigned, "agent-assigned")
|
||||
if want := dedupkeys.AssignmentKey(task.ID, "agent-assigned", gen); key != want {
|
||||
t.Fatalf("task_assigned key = %q, want %q", key, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_blockers_resolved", func(t *testing.T) {
|
||||
ss, repo := newAuditScheduler(t)
|
||||
setupChildrenCompletedParent(t, ss, "blocked-audit", "agent-blocked")
|
||||
insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-a", "COMPLETED")
|
||||
insertBlockerRelationship(t, repo, "blocked-audit", "blocker-audit-b", "COMPLETED")
|
||||
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
ss.cascadeBlockersResolved(context.Background(),
|
||||
&TaskSnapshot{ID: "blocker-audit-b", WorkspaceID: "ws-1"}, queue)
|
||||
|
||||
key := requireOneCallKey(t, calls, RunReasonTaskBlockersResolved, "agent-blocked")
|
||||
digest := dedupkeys.BlockerDigest([]string{"blocker-audit-a", "blocker-audit-b"})
|
||||
want := RunReasonTaskBlockersResolved + ":blocked-audit:agent-blocked:" + digest
|
||||
if key != want {
|
||||
t.Fatalf("task_blockers_resolved key = %q, want %q", key, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_comment", func(t *testing.T) {
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-comment", WorkspaceID: "ws-1", AssigneeAgentProfileID: "agent-assignee"}
|
||||
comment := &MutationComment{ID: "comment-audit-1", Body: "status update", AuthorType: "user", AuthorID: "user-1"}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
ss.reactToComment(context.Background(), task, comment, queue)
|
||||
|
||||
key := requireOneCallKey(t, calls, RunReasonTaskComment, "agent-assignee")
|
||||
want := commentkeys.TaskComment("comment-audit-1") + ":agent-assignee"
|
||||
if key != want {
|
||||
t.Fatalf("task_comment key = %q, want %q", key, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_mentioned", func(t *testing.T) {
|
||||
ss, repo := newAuditScheduler(t)
|
||||
createChildrenCompletedAgent(t, repo, "agent-mentioned")
|
||||
task := &TaskSnapshot{ID: "task-audit-mentioned", WorkspaceID: "ws-1"}
|
||||
comment := &MutationComment{
|
||||
ID: "comment-audit-2", AuthorType: "user", AuthorID: "user-1",
|
||||
// The mention token regex allows internal spaces (for multi-word
|
||||
// agent names), so it greedily consumes everything after the "@"
|
||||
// up to end of string or a disallowed character. Keeping the
|
||||
// mention last with nothing trailing bounds the capture to
|
||||
// exactly the agent name.
|
||||
Body: "cc @agent-mentioned",
|
||||
// Isolate the mention path from the assignee-wake path (task has
|
||||
// no assignee here, but SkipAssigneeWake documents the intent).
|
||||
SkipAssigneeWake: true,
|
||||
}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
ss.reactToComment(context.Background(), task, comment, queue)
|
||||
|
||||
key := requireOneCallKey(t, calls, RunReasonTaskMentioned, "agent-mentioned")
|
||||
want := RunReasonTaskMentioned + ":comment-audit-2:agent-mentioned"
|
||||
if key != want {
|
||||
t.Fatalf("task_mentioned key = %q, want %q", key, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_reopened_via_comment", func(t *testing.T) {
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-reopen-comment", WorkspaceID: "ws-1", State: "DONE", AssigneeAgentProfileID: "agent-reopen"}
|
||||
res := &ApplyTaskMutationResult{}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
change := TaskMutation{
|
||||
Comment: &MutationComment{ID: "comment-audit-3", AuthorType: "user", AuthorID: "user-1"},
|
||||
ActorID: "user-1",
|
||||
ActorType: "user",
|
||||
}
|
||||
ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res)
|
||||
|
||||
key := requireOneCallKey(t, calls, RunReasonTaskReopenedComment, "agent-reopen")
|
||||
want := RunReasonTaskReopenedComment + ":comment-audit-3:agent-reopen"
|
||||
if key != want {
|
||||
t.Fatalf("task_reopened_via_comment key = %q, want %q", key, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_unblocked", func(t *testing.T) {
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-unblocked", WorkspaceID: "ws-1", State: "BLOCKED", AssigneeAgentProfileID: "agent-unblocked"}
|
||||
res := &ApplyTaskMutationResult{}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design")
|
||||
change := TaskMutation{ActorID: "user-1", ActorType: "user"}
|
||||
ss.reactToStatusChange(context.Background(), task, "todo", change, queue, res)
|
||||
|
||||
requireOneCallKeyless(t, calls, RunReasonTaskUnblocked, "agent-unblocked")
|
||||
if got, want := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"), before+1; got != want {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want %d", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_unblocked_no_assignee_does_not_report", func(t *testing.T) {
|
||||
// Regression test for Review round 2 Finding 3: an unassigned task
|
||||
// must not increment the keyless counter for an enqueue that never
|
||||
// happens. Goes through ApplyTaskMutation (not reactToStatusChange
|
||||
// directly) so the real queue closure's empty-agent-id guard
|
||||
// (reactivity.go's `if agentID == "" { return }`) is exercised —
|
||||
// that guard is exactly what silently swallowed the enqueue while
|
||||
// FINDING 3's ReportKeylessEnqueue call still fired unconditionally.
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-unblocked-noassignee", WorkspaceID: "ws-1", State: "BLOCKED"}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design")
|
||||
newStatus := "todo"
|
||||
change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"}
|
||||
res, err := ss.ApplyTaskMutation(context.Background(), task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
if len(res.Runs) != 0 {
|
||||
t.Fatalf("expected no queued runs for an unassigned task_unblocked, got %#v", res.Runs)
|
||||
}
|
||||
if got := auditKeylessCounterValue(t, RunReasonTaskUnblocked, "by_design"); got != before {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_unblocked,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_reopened", func(t *testing.T) {
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-reopen-silent", WorkspaceID: "ws-1", State: "CANCELLED", AssigneeAgentProfileID: "agent-silent-reopen"}
|
||||
res := &ApplyTaskMutationResult{}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design")
|
||||
change := TaskMutation{ActorID: "user-1", ActorType: "user"}
|
||||
ss.reactToStatusChange(context.Background(), task, "in_progress", change, queue, res)
|
||||
|
||||
requireOneCallKeyless(t, calls, RunReasonTaskReopened, "agent-silent-reopen")
|
||||
if got, want := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"), before+1; got != want {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want %d", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_reopened_no_assignee_does_not_report", func(t *testing.T) {
|
||||
// Regression test for Review round 2 Finding 3, silent-reopen branch.
|
||||
// Goes through ApplyTaskMutation for the same reason as the
|
||||
// task_unblocked case above: the empty-agent-id guard lives in the
|
||||
// real queue closure, not in reactToStatusChange itself.
|
||||
ss, _ := newAuditScheduler(t)
|
||||
task := &TaskSnapshot{ID: "task-audit-reopen-silent-noassignee", WorkspaceID: "ws-1", State: "CANCELLED"}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design")
|
||||
newStatus := "in_progress"
|
||||
change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"}
|
||||
res, err := ss.ApplyTaskMutation(context.Background(), task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
if len(res.Runs) != 0 {
|
||||
t.Fatalf("expected no queued runs for an unassigned silent reopen, got %#v", res.Runs)
|
||||
}
|
||||
if got := auditKeylessCounterValue(t, RunReasonTaskReopened, "by_design"); got != before {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_reopened,by_design} = %d, want unchanged %d (no enqueue attempted)", got, before)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_review_requested", func(t *testing.T) {
|
||||
ss, repo := newAuditScheduler(t)
|
||||
ctx := context.Background()
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review', 'ws-1', 'step-audit')
|
||||
`); err != nil {
|
||||
t.Fatalf("insert task: %v", err)
|
||||
}
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO workflow_step_participants
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position)
|
||||
VALUES ('p-audit-reviewer', 'step-audit', 'task-audit-review', 'reviewer', 'agent-reviewer-audit', 1, 0)
|
||||
`); err != nil {
|
||||
t.Fatalf("insert participant: %v", err)
|
||||
}
|
||||
|
||||
task := &TaskSnapshot{ID: "task-audit-review", WorkspaceID: "ws-1"}
|
||||
var calls []recordedQueueCall
|
||||
queue := func(agentID string, c RunContext) {
|
||||
calls = append(calls, recordedQueueCall{agentID: agentID, ctx: c})
|
||||
}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design")
|
||||
ss.cascadeReviewRequested(ctx, task, TaskMutation{ActorID: "user-1", ActorType: "user"}, queue)
|
||||
|
||||
requireOneCallKeyless(t, calls, RunReasonTaskReviewRequested, "agent-reviewer-audit")
|
||||
if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("task_review_requested_dual_role_recipient_counts_once", func(t *testing.T) {
|
||||
// Regression test for Review round 3 Finding 4: an agent seated as
|
||||
// both reviewer and approver on the same task is one recipient, not
|
||||
// two. Driven through the real ApplyTaskMutation (not a bare
|
||||
// closure) so the real queue closure's seen-map dedup — which
|
||||
// silently absorbs the second queue() call for the same agent —
|
||||
// is what proves the counter must match the number of enqueue
|
||||
// attempts actually made, not the number of participant rows read.
|
||||
ss, repo := newAuditScheduler(t)
|
||||
ctx := context.Background()
|
||||
createChildrenCompletedAgent(t, repo, "agent-dual-role")
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO tasks (id, workspace_id, workflow_step_id) VALUES ('task-audit-review-dual', 'ws-1', 'step-audit')
|
||||
`); err != nil {
|
||||
t.Fatalf("insert task: %v", err)
|
||||
}
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO workflow_step_participants
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position)
|
||||
VALUES ('p-audit-dual-reviewer', 'step-audit', 'task-audit-review-dual', 'reviewer', 'agent-dual-role', 1, 0)
|
||||
`); err != nil {
|
||||
t.Fatalf("insert reviewer participant: %v", err)
|
||||
}
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
INSERT INTO workflow_step_participants
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position)
|
||||
VALUES ('p-audit-dual-approver', 'step-audit', 'task-audit-review-dual', 'approver', 'agent-dual-role', 1, 1)
|
||||
`); err != nil {
|
||||
t.Fatalf("insert approver participant: %v", err)
|
||||
}
|
||||
|
||||
task := &TaskSnapshot{ID: "task-audit-review-dual", WorkspaceID: "ws-1", State: statusTodo}
|
||||
before := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design")
|
||||
newStatus := statusInReview
|
||||
change := TaskMutation{NewStatus: &newStatus, ActorID: "user-1", ActorType: "user"}
|
||||
res, err := ss.ApplyTaskMutation(ctx, task, change)
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyTaskMutation: %v", err)
|
||||
}
|
||||
|
||||
queued := 0
|
||||
for _, r := range res.Runs {
|
||||
if r.Reason == RunReasonTaskReviewRequested && r.AgentID == "agent-dual-role" {
|
||||
queued++
|
||||
}
|
||||
}
|
||||
if queued != 1 {
|
||||
t.Fatalf("expected exactly 1 queued task_review_requested run for the dual-role agent, got %d (%#v)", queued, res.Runs)
|
||||
}
|
||||
if got, want := auditKeylessCounterValue(t, RunReasonTaskReviewRequested, "by_design"), before+1; got != want {
|
||||
t.Fatalf("office_run_dedup_keyless_total{task_review_requested,by_design} = %d, want %d (one recipient, not one row per role)", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// ErrRoutingNotSupported is returned by TaskStarter.StartTaskWithRoute
|
||||
@@ -107,13 +108,13 @@ type RunContext struct {
|
||||
// task_changes_requested has the context inline.
|
||||
DecisionComment string `json:"decision_comment,omitempty"`
|
||||
|
||||
// IdempotencyKey, when non-empty, overrides the default
|
||||
// "{reason}:{taskID}:{agentID}" key QueueRunCtx mints. The default
|
||||
// key is permanently unique per (reason, task, agent) — fine for
|
||||
// reasons that fire at most once per task, wrong for a reason that
|
||||
// can legitimately recur (e.g. task_children_completed across
|
||||
// repeated delegation waves). Callers that recur build their own
|
||||
// key that changes with the thing that makes each occurrence
|
||||
// IdempotencyKey is the dedup identity QueueRunCtx passes through
|
||||
// verbatim. Empty means no dedup — the run enqueues keyless — not
|
||||
// "derive one for me": QueueRunCtx no longer synthesises a
|
||||
// "{reason}:{taskID}:{agentID}" default, which was permanently unique
|
||||
// per (reason, task, agent) and silently swallowed every later
|
||||
// legitimate occurrence for the same triple. Callers that want dedup
|
||||
// build a key that changes with the thing that makes each occurrence
|
||||
// distinct. Excluded from the JSON payload: it must never change
|
||||
// encodeRunContext's output shape, which CoalesceRun compares for
|
||||
// equality and taskIDFromPayload parses.
|
||||
@@ -248,32 +249,30 @@ func (ss *SchedulerService) SetProjectSkillDirResolver(fn func(agentTypeID strin
|
||||
func (ss *SchedulerService) QueueRun(
|
||||
ctx context.Context,
|
||||
agentInstanceID, reason, payload, idempotencyKey string,
|
||||
) error {
|
||||
) (runsservice.QueueOutcome, error) {
|
||||
if err := ss.guardAgentStatus(ctx, agentInstanceID); err != nil {
|
||||
return err
|
||||
return runsservice.QueueOutcomeNone, err
|
||||
}
|
||||
|
||||
if idempotencyKey != "" {
|
||||
dup, err := ss.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours)
|
||||
if err != nil {
|
||||
return fmt.Errorf("idempotency check: %w", err)
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err)
|
||||
}
|
||||
if dup {
|
||||
ss.logger.Debug("run skipped (idempotent)",
|
||||
zap.String("key", idempotencyKey))
|
||||
return nil
|
||||
return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil
|
||||
}
|
||||
}
|
||||
|
||||
coalesced, err := ss.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("coalesce check: %w", err)
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err)
|
||||
}
|
||||
if coalesced {
|
||||
ss.logger.Debug("run coalesced",
|
||||
zap.String("agent", agentInstanceID),
|
||||
zap.String("reason", reason))
|
||||
return nil
|
||||
return runsservice.QueueOutcomeCoalesced, nil
|
||||
}
|
||||
|
||||
var idemKeyPtr *string
|
||||
@@ -290,36 +289,37 @@ func (ss *SchedulerService) QueueRun(
|
||||
IdempotencyKey: idemKeyPtr,
|
||||
RequestedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := ss.repo.CreateRun(ctx, req); err != nil {
|
||||
return fmt.Errorf("enqueue run: %w", err)
|
||||
insertErr := ss.repo.CreateRun(ctx, req)
|
||||
outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr)
|
||||
if err != nil {
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err)
|
||||
}
|
||||
if outcome == runsservice.QueueOutcomeDeduped {
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
ss.logger.Info("run queued",
|
||||
zap.String("id", req.ID),
|
||||
zap.String("agent", agentInstanceID),
|
||||
zap.String("reason", reason))
|
||||
return nil
|
||||
return runsservice.QueueOutcomeQueued, nil
|
||||
}
|
||||
|
||||
// QueueRunCtx is the typed variant of QueueRun that takes a
|
||||
// structured RunContext. The context is JSON-encoded into the
|
||||
// payload column so the agent runtime can deserialise it. The
|
||||
// idempotency key is c.IdempotencyKey when the caller set one;
|
||||
// otherwise it defaults to "{reason}:{taskID}:{agentID}" so the same
|
||||
// agent never gets two runs for the same task+reason within the
|
||||
// idempotency window.
|
||||
// QueueRunCtx is the typed variant of QueueRun that takes a structured
|
||||
// RunContext. The context is JSON-encoded into the payload column so the
|
||||
// agent runtime can deserialise it. The idempotency key is c.IdempotencyKey
|
||||
// verbatim — an empty key enqueues with no dedup identity rather than
|
||||
// falling back to a "{reason}:{taskID}:{agentID}" default that would be
|
||||
// permanently unique per (reason, task, agent) and silently swallow every
|
||||
// later legitimate occurrence for the same triple.
|
||||
func (ss *SchedulerService) QueueRunCtx(
|
||||
ctx context.Context, agentInstanceID string, c RunContext,
|
||||
) error {
|
||||
) (runsservice.QueueOutcome, error) {
|
||||
payload, err := encodeRunContext(c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode run context: %w", err)
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("encode run context: %w", err)
|
||||
}
|
||||
idempotencyKey := c.IdempotencyKey
|
||||
if idempotencyKey == "" {
|
||||
idempotencyKey = fmt.Sprintf("%s:%s:%s", c.Reason, c.TaskID, agentInstanceID)
|
||||
}
|
||||
return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, idempotencyKey)
|
||||
return ss.QueueRun(ctx, agentInstanceID, c.Reason, payload, c.IdempotencyKey)
|
||||
}
|
||||
|
||||
func encodeRunContext(c RunContext) (string, error) {
|
||||
|
||||
@@ -40,7 +40,7 @@ func launchedWorkingAgent(
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, project_id, title, created_at, updated_at)
|
||||
VALUES (?, 'ws-1', ?, 'Working status task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
|
||||
taskID, project.ID)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ func newTestService(t *testing.T, overrides ...service.ServiceOptions) *service.
|
||||
state TEXT NOT NULL DEFAULT 'TODO',
|
||||
title TEXT DEFAULT '',
|
||||
assignee_user_id TEXT NOT NULL DEFAULT '',
|
||||
assignment_generation INTEGER NOT NULL DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
identifier TEXT DEFAULT '',
|
||||
workflow_id TEXT DEFAULT '',
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
"github.com/kandev/kandev/internal/workflow/engine"
|
||||
)
|
||||
|
||||
// newBlockersConvergenceTestRepo builds a real office repository (real
|
||||
// migrations, so task_blockers is the production schema) plus the minimal
|
||||
// hand-rolled tasks table resolveAndWakeIfUnblocked reads state from —
|
||||
// tasks is owned by the task package's schema, which office's migrations
|
||||
// only reference by foreign key.
|
||||
func newBlockersConvergenceTestRepo(t *testing.T) *sqlite.Repository {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("settings store: %v", err)
|
||||
}
|
||||
repo, err := sqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := repo.ExecRaw(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT DEFAULT '',
|
||||
state TEXT DEFAULT ''
|
||||
)
|
||||
`); err != nil {
|
||||
t.Fatalf("create tasks table: %v", err)
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
// convergenceDispatcher records every HandleTrigger call so the test can
|
||||
// assert the exact operation id resolveAndWakeIfUnblocked derived.
|
||||
type convergenceDispatcher struct {
|
||||
mu sync.Mutex
|
||||
calls []convergenceCall
|
||||
}
|
||||
|
||||
type convergenceCall struct {
|
||||
taskID string
|
||||
trigger engine.Trigger
|
||||
payload any
|
||||
opID string
|
||||
}
|
||||
|
||||
func (d *convergenceDispatcher) HandleTrigger(
|
||||
_ context.Context, taskID string, trigger engine.Trigger, payload any, opID string,
|
||||
) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.calls = append(d.calls, convergenceCall{taskID, trigger, payload, opID})
|
||||
return nil
|
||||
}
|
||||
|
||||
func newConvergenceTestService(t *testing.T, repo *sqlite.Repository, dispatcher *convergenceDispatcher) *Service {
|
||||
t.Helper()
|
||||
log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "console"})
|
||||
if err != nil {
|
||||
t.Fatalf("logger: %v", err)
|
||||
}
|
||||
s := &Service{repo: repo, logger: log}
|
||||
s.SetWorkflowEngineDispatcher(dispatcher)
|
||||
return s
|
||||
}
|
||||
|
||||
// TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved
|
||||
// is the AC-OFFICE-RUN-DEDUP-002.1 convergence test for the second blocker
|
||||
// producer. office/scheduler.cascadeBlockersResolved (the other half of the
|
||||
// same AC) already has a regression test
|
||||
// (reactivity_blockers_resolved_test.go), but that test never exercises
|
||||
// this package's resolveAndWakeIfUnblocked — it only proves
|
||||
// dedupkeys.BlockerDigest is order-independent in isolation. Review round 2
|
||||
// found the AC's own explicit test requirement ("both producers derive a
|
||||
// byte-identical digest, driven through the shared builder from both
|
||||
// packages") unproven by any executing test. This test drives
|
||||
// resolveAndWakeIfUnblocked directly and asserts its persisted operation id
|
||||
// carries the exact digest office/scheduler's producer would derive for the
|
||||
// identical blocker set.
|
||||
func TestResolveAndWakeIfUnblocked_DerivesSameDigestAsCascadeBlockersResolved(t *testing.T) {
|
||||
repo := newBlockersConvergenceTestRepo(t)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, row := range []struct{ id, state string }{
|
||||
{"blocked-1", "IN_PROGRESS"},
|
||||
{"blocker-conv-a", "COMPLETED"},
|
||||
{"blocker-conv-b", "COMPLETED"},
|
||||
} {
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`,
|
||||
row.id, row.state,
|
||||
); err != nil {
|
||||
t.Fatalf("insert task %s: %v", row.id, err)
|
||||
}
|
||||
}
|
||||
for _, blockerID := range []string{"blocker-conv-a", "blocker-conv-b"} {
|
||||
if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{
|
||||
TaskID: "blocked-1", BlockerTaskID: blockerID,
|
||||
}); err != nil {
|
||||
t.Fatalf("create blocker relationship for %s: %v", blockerID, err)
|
||||
}
|
||||
}
|
||||
|
||||
dispatcher := &convergenceDispatcher{}
|
||||
s := newConvergenceTestService(t, repo, dispatcher)
|
||||
|
||||
if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-1", "blocker-conv-b"); err != nil {
|
||||
t.Fatalf("resolveAndWakeIfUnblocked: %v", err)
|
||||
}
|
||||
|
||||
dispatcher.mu.Lock()
|
||||
calls := append([]convergenceCall{}, dispatcher.calls...)
|
||||
dispatcher.mu.Unlock()
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("HandleTrigger calls = %d, want 1: %#v", len(calls), calls)
|
||||
}
|
||||
call := calls[0]
|
||||
if call.taskID != "blocked-1" {
|
||||
t.Fatalf("taskID = %q, want blocked-1", call.taskID)
|
||||
}
|
||||
if call.trigger != engine.TriggerOnBlockerResolved {
|
||||
t.Fatalf("trigger = %q, want %q", call.trigger, engine.TriggerOnBlockerResolved)
|
||||
}
|
||||
|
||||
// The digest office/scheduler.cascadeBlockersResolved would derive for
|
||||
// the identical blocker set — proving the two independently-implemented
|
||||
// producers converge on the same occurrence identity, per AC-002.1.
|
||||
digest := dedupkeys.BlockerDigest([]string{"blocker-conv-a", "blocker-conv-b"})
|
||||
want := "blockers_resolved:blocked-1:" + digest
|
||||
if call.opID != want {
|
||||
t.Fatalf("operation id = %q, want %q", call.opID, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch proves the
|
||||
// negative: a task with another outstanding blocker must not wake.
|
||||
func TestResolveAndWakeIfUnblocked_StillBlocked_DoesNotDispatch(t *testing.T) {
|
||||
repo := newBlockersConvergenceTestRepo(t)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, row := range []struct{ id, state string }{
|
||||
{"blocked-2", "IN_PROGRESS"},
|
||||
{"blocker-conv-c", "IN_PROGRESS"}, // never finishes
|
||||
{"blocker-conv-d", "COMPLETED"},
|
||||
} {
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`INSERT INTO tasks (id, workspace_id, state) VALUES (?, 'ws-1', ?)`,
|
||||
row.id, row.state,
|
||||
); err != nil {
|
||||
t.Fatalf("insert task %s: %v", row.id, err)
|
||||
}
|
||||
}
|
||||
for _, blockerID := range []string{"blocker-conv-c", "blocker-conv-d"} {
|
||||
if err := repo.CreateTaskBlocker(ctx, &models.TaskBlocker{
|
||||
TaskID: "blocked-2", BlockerTaskID: blockerID,
|
||||
}); err != nil {
|
||||
t.Fatalf("create blocker relationship for %s: %v", blockerID, err)
|
||||
}
|
||||
}
|
||||
|
||||
dispatcher := &convergenceDispatcher{}
|
||||
s := newConvergenceTestService(t, repo, dispatcher)
|
||||
|
||||
if err := s.resolveAndWakeIfUnblocked(ctx, "blocked-2", "blocker-conv-d"); err != nil {
|
||||
t.Fatalf("resolveAndWakeIfUnblocked: %v", err)
|
||||
}
|
||||
|
||||
dispatcher.mu.Lock()
|
||||
defer dispatcher.mu.Unlock()
|
||||
if len(dispatcher.calls) != 0 {
|
||||
t.Fatalf("expected no HandleTrigger calls while blocker-conv-c is still outstanding, got %#v", dispatcher.calls)
|
||||
}
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func TestAdmitRun_NoEvaluatorWired_UnattendedCancels(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -128,7 +128,7 @@ func TestAdmitRun_NoEvaluatorWired_AttendedLaunches(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-no-evaluator-attended", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-no-evaluator-attended"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T)
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-evaluator-fault", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-evaluator-fault"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -229,7 +229,7 @@ func TestAdmitRun_UnevaluatedPolicy_NamesPolicyInDeferral(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-unevaluated-policy", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-unevaluated-policy"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -279,7 +279,7 @@ func TestFinishPolicyBlock_ClearsStaleAgentWorkingStatus(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -308,7 +308,7 @@ func TestCancelBudgetRun_ClearsStaleAgentWorkingStatus(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -345,7 +345,7 @@ func TestFailRunNoEscalation_ClearsStaleAgentWorkingStatus(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -388,7 +388,7 @@ func TestProcessRun_MissingAgentReleasesPriorCheckout(t *testing.T) {
|
||||
INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-routed-missing', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-routed-missing"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -445,7 +445,7 @@ func TestProcessRun_WorkspaceLookupErrorReleasesPriorCheckout(t *testing.T) {
|
||||
INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-routed-lookup-error', 'ws-1', 'Routed task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-routed-lookup-error"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -496,7 +496,7 @@ func TestAdmitRun_RepeatedDeferral_AttemptIncrementsPerActivity(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-attempt-increments", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-attempt-increments"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -536,7 +536,7 @@ func TestAdmitRun_DefaultCeiling_BlocksUnattendedRunWithZeroPolicies(t *testing.
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling", int64(600_000))
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -621,7 +621,7 @@ func TestAdmitRun_PricingDegradedBlock_UnmeasurableOutcome(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -652,7 +652,7 @@ func TestAdmitRun_DefaultCeiling_ExemptsAttendedRun(t *testing.T) {
|
||||
}
|
||||
insertTestCostEvent(t, svc, agent.ID, "task-default-ceiling-attended", int64(600_000))
|
||||
insertTestTask(t, svc, "task-default-ceiling-attended-run", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-default-ceiling-attended-run"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -711,7 +711,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -732,7 +732,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -752,7 +752,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -775,7 +775,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
@@ -794,7 +794,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -825,7 +825,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -847,7 +847,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -870,7 +870,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -891,7 +891,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -918,7 +918,7 @@ func TestBudgetAdmission_ActionsAreDistinguishable(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
@@ -47,7 +47,7 @@ func TestAdmitRun_MetricBlockedByLimit(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func TestAdmitRun_MetricBlockedPricingDegraded(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -104,7 +104,7 @@ func TestAdmitRun_MetricDeferredEvaluatorFault(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-metric-deferred-fault", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-metric-deferred-fault"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -126,7 +126,7 @@ func TestAdmitRun_MetricBlockedAbsentEvaluator(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestAdmitRun_MetricDeferredWorkspaceLookup(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -192,7 +192,7 @@ func TestProcessRun_AgentRowDeleted_Cancels(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -239,7 +239,7 @@ func TestAdmitRun_MetricCancelledNoWorkspace(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -271,7 +271,7 @@ func TestAdmitRun_MetricCancelledStaleDeferral(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -302,7 +302,7 @@ func TestAdmitRun_MetricAdmittedDefault(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -335,7 +335,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaDefault(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -369,7 +369,7 @@ func TestAdmitRun_MetricAdmittedDegradedWindow_ViaStoredPolicyBypassingGate5(t *
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-metric-degraded-attended", "ws-1")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-metric-degraded-attended"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ func (s *Service) SetupChannel(ctx context.Context, channel *models.Channel) err
|
||||
// the runner row is keyed against an empty step_id; the office
|
||||
// repo's runner projection still resolves it because it joins
|
||||
// step_id = task.workflow_step_id (also empty).
|
||||
if err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil {
|
||||
if _, err := s.repo.UpdateTaskAssignee(ctx, taskID, channel.AgentProfileID); err != nil {
|
||||
_ = s.repo.DeleteChannel(ctx, channel.ID)
|
||||
return fmt.Errorf("set channel task assignee: %w", err)
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func TestLoadContinuationSummary_AgentScope_RoundTripsThroughRealWriterAndReader
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "agent-scope-a")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "agent-scope-a", service.RunReasonTaskAssigned, "{}", "continuation-agent-scope",
|
||||
); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
@@ -147,7 +147,7 @@ func TestLoadContinuationSummary_ScopeSurvivesCoalesceAfterClaim(t *testing.T) {
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "agent-scope-coalesce")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "agent-scope-coalesce", service.RunReasonTaskAssigned, "{}", "continuation-scope-coalesce",
|
||||
); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
"github.com/kandev/kandev/internal/runs/commentkeys"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
"github.com/kandev/kandev/internal/workflow/engine"
|
||||
)
|
||||
|
||||
@@ -107,12 +109,16 @@ type TaskMovedData struct {
|
||||
SessionID string `json:"session_id"`
|
||||
}
|
||||
|
||||
// TaskUpdatedData represents the payload of a task.updated event.
|
||||
// TaskUpdatedData represents the payload of a task.created / task.updated
|
||||
// event. AssignmentGeneration is carried on the extra map by
|
||||
// publishTaskEventWithExtra's caller at task-creation time (never re-read);
|
||||
// nil means the publishing event predates this field or is not a creation.
|
||||
type TaskUpdatedData struct {
|
||||
TaskID string `json:"task_id"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
AssigneeAgentProfileID string `json:"assignee_agent_profile_id"`
|
||||
Title string `json:"title"`
|
||||
AssignmentGeneration *int64 `json:"assignment_generation"`
|
||||
}
|
||||
|
||||
// CommentPostedData represents a comment event payload.
|
||||
@@ -994,22 +1000,39 @@ func (s *Service) handleTaskCreated(ctx context.Context, event *bus.Event) error
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, true)
|
||||
return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, true)
|
||||
}
|
||||
|
||||
// handleTaskUpdated fires a task_assigned run when an agent is assigned.
|
||||
// handleTaskUpdated fires a task_assigned run when an agent is assigned. No
|
||||
// production update path can put a new agent into the runner seat (see
|
||||
// office/repository/sqlite/tasks.go's UpdateTaskAssignee vs the four inert
|
||||
// syncRunnerInTx writers), so this stays subscribed as a redelivery and
|
||||
// defensive path rather than a live assignment occurrence.
|
||||
func (s *Service) handleTaskUpdated(ctx context.Context, event *bus.Event) error {
|
||||
data, err := decodeEventData[TaskUpdatedData](event)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, false)
|
||||
return s.queueTaskAssignedRun(ctx, data.TaskID, data.AssigneeAgentProfileID, data.AssignmentGeneration, false)
|
||||
}
|
||||
|
||||
// queueTaskAssignedRun fires a task_assigned run for the given occurrence.
|
||||
//
|
||||
// When fallbackToStoredRunner is set and the event carried no assignee, the
|
||||
// agent is recovered through a fresh post-commit GetTaskExecutionFields read
|
||||
// (task.created's payload can omit it) while the generation, if any, still
|
||||
// describes the payload's own occurrence — not necessarily this freshly-read
|
||||
// agent's. Task event publication is asynchronous (a per-task FIFO drainer),
|
||||
// so a reassignment can commit between the event being built and this
|
||||
// handler running, and the two halves would then name different occurrences.
|
||||
// Forcing keyless whenever the fallback actually recovers an agent this way
|
||||
// avoids mis-keying at the cost of a possible duplicate wake, which
|
||||
// AC-003.2 already accepts.
|
||||
func (s *Service) queueTaskAssignedRun(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
agentProfileID string,
|
||||
assignmentGeneration *int64,
|
||||
fallbackToStoredRunner bool,
|
||||
) error {
|
||||
if taskID == "" {
|
||||
@@ -1025,15 +1048,23 @@ func (s *Service) queueTaskAssignedRun(
|
||||
if fields == nil || !fields.IsFromOffice {
|
||||
return nil
|
||||
}
|
||||
fellBackToStoredRunner := false
|
||||
if agentProfileID == "" && fallbackToStoredRunner {
|
||||
agentProfileID = fields.AssigneeAgentProfileID
|
||||
fellBackToStoredRunner = agentProfileID != ""
|
||||
}
|
||||
if agentProfileID == "" {
|
||||
return nil
|
||||
}
|
||||
payload := mustJSON(map[string]string{"task_id": taskID})
|
||||
key := fmt.Sprintf("task_assigned:%s:%s", taskID, agentProfileID)
|
||||
return s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key)
|
||||
var key string
|
||||
if fellBackToStoredRunner || assignmentGeneration == nil {
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseUnresolved, "event_missing_generation")
|
||||
} else {
|
||||
key = dedupkeys.AssignmentKey(taskID, agentProfileID, *assignmentGeneration)
|
||||
}
|
||||
_, err = s.QueueRun(ctx, agentProfileID, RunReasonTaskAssigned, payload, key)
|
||||
return err
|
||||
}
|
||||
|
||||
// handleTaskMoved keeps the legacy named-step activity fallback and queues
|
||||
@@ -1120,7 +1151,9 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blockerIDs := make([]string, 0, len(blockers))
|
||||
for _, b := range blockers {
|
||||
blockerIDs = append(blockerIDs, b.BlockerTaskID)
|
||||
if b.BlockerTaskID == resolvedBlockerID {
|
||||
continue
|
||||
}
|
||||
@@ -1129,7 +1162,10 @@ func (s *Service) resolveAndWakeIfUnblocked(ctx context.Context, blockedTaskID,
|
||||
return err // still blocked
|
||||
}
|
||||
}
|
||||
key := fmt.Sprintf("blockers_resolved:%s", blockedTaskID)
|
||||
if len(blockerIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
key := fmt.Sprintf("blockers_resolved:%s:%s", blockedTaskID, dedupkeys.BlockerDigest(blockerIDs))
|
||||
return s.dispatchEngineTrigger(ctx, blockedTaskID, engine.TriggerOnBlockerResolved,
|
||||
engine.OnBlockerResolvedPayload{
|
||||
ResolvedBlockerIDs: []string{resolvedBlockerID},
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestHandleAgentCompleted_WarnsWhenReviewDecisionMissing(t *testing.T) {
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}`
|
||||
if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -91,7 +91,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyReviewStartedReason(t *testing.T) {
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-1","agent_profile_id":"reviewer-1"}`
|
||||
if err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "reviewer-1", "review_started", payload, "review-started-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -139,7 +139,7 @@ func TestHandleAgentCompleted_WarnsOnStageIDKeyedPayload(t *testing.T) {
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","stage_type":"approval","stage_id":"step-1"}`
|
||||
if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "approval-stage-id-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -179,7 +179,7 @@ func TestHandleAgentCompleted_WarnsOnLegacyApprovalStartedReason(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "approver-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "approver-1")
|
||||
payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-approval","agent_profile_id":"approver-1"}`
|
||||
if err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "approver-1", "approval_started", payload, "approval-started-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -219,7 +219,7 @@ func TestHandleAgentCompleted_WarnsOnTaskReviewRequested(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-review-requested", "approval")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","role":"approver"}`
|
||||
if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskReviewRequested, payload, "task-review-requested-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -259,7 +259,7 @@ func TestHandleAgentCompleted_UsesAuthoritativeStageTypeWhenPayloadOmitsIt(t *te
|
||||
svc.ExecSQL(t, `INSERT INTO workflow_steps (id, stage_type) VALUES (?, ?)`, "step-authoritative-decision", "approval")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","workflow_step_id":"step-authoritative-decision"}`
|
||||
if err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "approver-1", service.RunReasonTaskAssigned, payload, "authoritative-decision-no-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -301,7 +301,7 @@ func TestHandleAgentCompleted_NoWarnWhenReviewDecisionRecorded(t *testing.T) {
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "reviewer-1")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","stage_type":"review","workflow_step_id":"step-1"}`
|
||||
if err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "reviewer-1", service.RunReasonTaskAssigned, payload, "review-with-decision"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -347,7 +347,7 @@ func TestHandleAgentCompleted_NoWarnForWorkStage(t *testing.T) {
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "builder-1")
|
||||
|
||||
payload := `{"task_id":"` + taskID + `","stage_type":"work","workflow_step_id":"step-1"}`
|
||||
if err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "builder-1", service.RunReasonTaskAssigned, payload, "work-stage"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
|
||||
@@ -322,7 +322,7 @@ func queueTaskAssignedRunForAgentFailedTests(
|
||||
) *models.Run {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, agentID, service.RunReasonTaskAssigned, `{"task_id":"`+taskID+`"}`, "",
|
||||
); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
@@ -565,7 +565,7 @@ func TestEngineDispatcher_PathBEscalation_DoesNotFireAgentErrorTrigger(t *testin
|
||||
}
|
||||
createTestAgent(t, svc, "ws-1", "worker-pathb")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-pathb", service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "",
|
||||
); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestHandleAgentCompleted_RecordsFinalAgentMessageAsOutputSummary(t *testing
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-init",
|
||||
); err != nil {
|
||||
@@ -127,7 +127,7 @@ func TestHandleAgentCompleted_TruncatesOutputSummaryAt500Chars(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-truncate",
|
||||
); err != nil {
|
||||
@@ -175,7 +175,7 @@ func TestHandleAgentCompleted_NoAgentMessageStaysBestEffort(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-none",
|
||||
); err != nil {
|
||||
@@ -216,7 +216,7 @@ func TestHandleTasklessAgentCompleted_RecordsOutputSummaryAndKeepsContinuationSu
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-taskless")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-taskless", service.RunReasonTaskAssigned, "{}", "run-output-taskless",
|
||||
); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
@@ -268,7 +268,7 @@ func TestHandleAgentCompleted_LastAgentMessageWinsOverLaterUserMessage(t *testin
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-ordering",
|
||||
); err != nil {
|
||||
@@ -318,7 +318,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) {
|
||||
// Run 1: queue, claim, produce an agent message, complete. The
|
||||
// session (sess-1) is reused for run 2 below, mirroring how office
|
||||
// task-bound sessions persist across turns.
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-reuse-1",
|
||||
); err != nil {
|
||||
@@ -359,7 +359,7 @@ func TestHandleAgentCompleted_SecondRunOnReusedSessionStaysEmpty(t *testing.T) {
|
||||
// Run 2: reuses sess-1 but produces no new agent message before
|
||||
// completing (e.g. a turn that only made tool calls, or was cut
|
||||
// short). No new task_session_messages row is inserted here.
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskComment,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-reuse-2",
|
||||
); err != nil {
|
||||
@@ -390,7 +390,7 @@ func TestHandleAgentCompleted_DoesNotClearExistingSummaryWithoutMessage(t *testi
|
||||
ctx := context.Background()
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-preserve"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
@@ -423,7 +423,7 @@ func TestHandleAgentCompleted_UsesTurnScopedMessageForSharedSession(t *testing.T
|
||||
ctx := context.Background()
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-turn"); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
"github.com/kandev/kandev/internal/workflow/engine"
|
||||
)
|
||||
|
||||
@@ -42,7 +43,8 @@ func (d *queueRunDispatcher) HandleTrigger(
|
||||
case engine.TriggerOnComment:
|
||||
p, _ := payload.(engine.OnCommentPayload)
|
||||
body, _ := json.Marshal(map[string]string{"task_id": taskID, "comment_id": p.CommentID})
|
||||
return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID)
|
||||
_, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskComment, string(body), opID)
|
||||
return err
|
||||
case engine.TriggerOnBlockerResolved:
|
||||
p, _ := payload.(engine.OnBlockerResolvedPayload)
|
||||
var resolved string
|
||||
@@ -50,10 +52,12 @@ func (d *queueRunDispatcher) HandleTrigger(
|
||||
resolved = p.ResolvedBlockerIDs[0]
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"task_id": taskID, "resolved_blocker_id": resolved})
|
||||
return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID)
|
||||
_, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskBlockersResolved, string(body), opID)
|
||||
return err
|
||||
case engine.TriggerOnChildrenCompleted:
|
||||
body, _ := json.Marshal(map[string]string{"task_id": taskID})
|
||||
return d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID)
|
||||
_, err := d.svc.QueueRun(ctx, assignee, service.RunReasonTaskChildrenCompleted, string(body), opID)
|
||||
return err
|
||||
case engine.TriggerOnApprovalResolved:
|
||||
p, _ := payload.(engine.OnApprovalResolvedPayload)
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
@@ -61,7 +65,8 @@ func (d *queueRunDispatcher) HandleTrigger(
|
||||
"status": p.Status,
|
||||
"decision_note": p.Note,
|
||||
})
|
||||
return d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID)
|
||||
_, err := d.svc.QueueRun(ctx, assignee, service.RunReasonApprovalResolved, string(body), opID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -289,6 +294,64 @@ func TestTaskCreated_WakesAssigneeFromStoredRunner(t *testing.T) {
|
||||
t.Fatalf("expected task_assigned run for worker-created, got %#v", runs)
|
||||
}
|
||||
|
||||
func TestTaskCreated_WithAssignmentIdentity_UsesGenerationKey(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-created-keyed")
|
||||
insertTestTask(t, svc, "task-created-keyed", "ws-1")
|
||||
svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project', assignment_generation = 1 WHERE id = ?`, "task-created-keyed")
|
||||
setTestTaskAssignee(t, svc, "task-created-keyed", "worker-created-keyed")
|
||||
|
||||
event := bus.NewEvent(events.TaskCreated, "task-service", map[string]any{
|
||||
"task_id": "task-created-keyed",
|
||||
"assignee_agent_profile_id": "worker-created-keyed",
|
||||
"assignment_generation": int64(1),
|
||||
})
|
||||
if err := eb.Publish(ctx, events.TaskCreated, event); err != nil {
|
||||
t.Fatalf("publish task created event: %v", err)
|
||||
}
|
||||
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list creation runs: %v", err)
|
||||
}
|
||||
wantKey := dedupkeys.AssignmentKey("task-created-keyed", "worker-created-keyed", 1)
|
||||
var creationRunCount int
|
||||
for _, run := range runs {
|
||||
if run.AgentProfileID != "worker-created-keyed" || run.Reason != service.RunReasonTaskAssigned {
|
||||
continue
|
||||
}
|
||||
creationRunCount++
|
||||
if run.IdempotencyKey == nil || *run.IdempotencyKey != wantKey {
|
||||
t.Fatalf("creation run key = %v, want %q", run.IdempotencyKey, wantKey)
|
||||
}
|
||||
}
|
||||
if creationRunCount != 1 {
|
||||
t.Fatalf("creation event queued %d runs, want 1", creationRunCount)
|
||||
}
|
||||
|
||||
// A replay after the first run is claimed must still use the same durable
|
||||
// key and avoid inserting a second run.
|
||||
svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE reason = ? AND idempotency_key = ?`, service.RunReasonTaskAssigned, wantKey)
|
||||
if err := eb.Publish(ctx, events.TaskCreated, event); err != nil {
|
||||
t.Fatalf("replay task created event: %v", err)
|
||||
}
|
||||
allRuns, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list replay runs: %v", err)
|
||||
}
|
||||
var totalTaskAssigned int
|
||||
for _, run := range allRuns {
|
||||
if run.AgentProfileID == "worker-created-keyed" && run.Reason == service.RunReasonTaskAssigned {
|
||||
totalTaskAssigned++
|
||||
}
|
||||
}
|
||||
if totalTaskAssigned != 1 {
|
||||
t.Fatalf("replayed creation event queued %d runs, want 1", totalTaskAssigned)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskCreated_NoopWhenNoStoredRunner(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -488,6 +488,9 @@ func (s *Service) autoPauseAgent(
|
||||
return nil
|
||||
}
|
||||
|
||||
// requeueRunForTask is a manual resume: the failed run's own id is the
|
||||
// occurrence identity (falling back to the task id if it is somehow empty),
|
||||
// so a duplicate "Mark fixed" click dedupes instead of double-queuing.
|
||||
func (s *Service) requeueRunForTask(
|
||||
ctx context.Context, agentID, taskID, failedRunID string,
|
||||
) error {
|
||||
@@ -497,7 +500,8 @@ func (s *Service) requeueRunForTask(
|
||||
identity = taskID
|
||||
}
|
||||
key := fmt.Sprintf("%s:%s:%s", RunReasonManualResumeAfterFailure, agentID, identity)
|
||||
return s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key)
|
||||
_, err := s.QueueRun(ctx, agentID, RunReasonManualResumeAfterFailure, payload, key)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) publishRunFailed(
|
||||
|
||||
@@ -26,7 +26,7 @@ func queueAndReadRun(
|
||||
ctx := context.Background()
|
||||
payload := mustMarshalJSON(map[string]string{"task_id": taskID})
|
||||
idem := agentID + ":" + taskID
|
||||
if err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agentID, service.RunReasonTaskAssigned, payload, idem); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
rows, err := svc.ListRuns(ctx, "ws-1")
|
||||
@@ -255,7 +255,7 @@ func TestMarkAgentPausedFixed_RecoversAgent(t *testing.T) {
|
||||
|
||||
// Proves the guardAgentStatus rejection is gone: QueueRun must
|
||||
// succeed now that the agent is actually idle.
|
||||
if err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, "agent-recover", service.RunReasonTaskAssigned,
|
||||
mustMarshalJSON(map[string]string{"task_id": "agent-recover-task-a"}),
|
||||
"agent-recover:post-fix"); err != nil {
|
||||
t.Fatalf("expected QueueRun to succeed after mark fixed, got: %v", err)
|
||||
|
||||
@@ -227,7 +227,12 @@ func (s *Service) queueCEOAgentError(
|
||||
"run_id": run.ID,
|
||||
"error": errMsg,
|
||||
})
|
||||
_ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, "")
|
||||
// The failed run's own id makes this occurrence identity: one escalation
|
||||
// per failed run per CEO, but a later run by the same agent that also
|
||||
// fails escalates again instead of being silently swallowed by a
|
||||
// permanently-unique-per-pair key.
|
||||
key := fmt.Sprintf("agent_error:%s:%s", run.ID, ceos[0].ID)
|
||||
_, _ = s.QueueRun(ctx, ceos[0].ID, RunReasonAgentError, payload, key)
|
||||
}
|
||||
|
||||
// retryDelayWithJitter returns the base delay for a given retry index
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestHandleRunFailure_CEOOwnFailure_DoesNotSelfEscalate(t *testing.T) {
|
||||
}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, ceo.ID, service.RunReasonAgentError, `{}`, "",
|
||||
); err != nil {
|
||||
t.Fatalf("queue ceo run %d: %v", i, err)
|
||||
|
||||
@@ -226,7 +226,7 @@ func TestHandleRunFailure_RateLimitParsed_UsesResetTime(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -275,7 +275,7 @@ func TestHandleRunFailure_RateLimitNoParseable_UsesBackoff(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k2"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
@@ -325,7 +325,7 @@ func TestHandleRunFailure_NonRateLimit_UsesBackoff(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k3"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
|
||||
@@ -85,18 +85,17 @@ const IdempotencyWindowHours = 24
|
||||
func (s *Service) QueueRun(
|
||||
ctx context.Context,
|
||||
agentInstanceID, reason, payload, idempotencyKey string,
|
||||
) error {
|
||||
) (runsservice.QueueOutcome, error) {
|
||||
if err := s.guardAgentStatus(ctx, agentInstanceID); err != nil {
|
||||
return err
|
||||
return runsservice.QueueOutcomeNone, err
|
||||
}
|
||||
|
||||
if s.runsService != nil {
|
||||
_, err := s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{
|
||||
return s.runsService.QueueRun(ctx, runsservice.QueueRunRequest{
|
||||
Reason: reason,
|
||||
IdempotencyKey: idempotencyKey,
|
||||
Payload: payloadWithAgent(payload, agentInstanceID),
|
||||
})
|
||||
return err
|
||||
}
|
||||
return s.queueRunInline(ctx, agentInstanceID, reason, payload, idempotencyKey)
|
||||
}
|
||||
@@ -107,28 +106,26 @@ func (s *Service) QueueRun(
|
||||
func (s *Service) queueRunInline(
|
||||
ctx context.Context,
|
||||
agentInstanceID, reason, payload, idempotencyKey string,
|
||||
) error {
|
||||
) (runsservice.QueueOutcome, error) {
|
||||
if idempotencyKey != "" {
|
||||
dup, err := s.repo.CheckIdempotencyKey(ctx, idempotencyKey, IdempotencyWindowHours)
|
||||
if err != nil {
|
||||
return fmt.Errorf("idempotency check: %w", err)
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err)
|
||||
}
|
||||
if dup {
|
||||
s.logger.Debug("run skipped (idempotent)",
|
||||
zap.String("key", idempotencyKey))
|
||||
return nil
|
||||
return runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, idempotencyKey), nil
|
||||
}
|
||||
}
|
||||
|
||||
coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, reason, CoalesceWindowSeconds, payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("coalesce check: %w", err)
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err)
|
||||
}
|
||||
if coalesced {
|
||||
s.logger.Debug("run coalesced",
|
||||
zap.String("agent", agentInstanceID),
|
||||
zap.String("reason", reason))
|
||||
return nil
|
||||
return runsservice.QueueOutcomeCoalesced, nil
|
||||
}
|
||||
|
||||
var idemKeyPtr *string
|
||||
@@ -145,8 +142,13 @@ func (s *Service) queueRunInline(
|
||||
IdempotencyKey: idemKeyPtr,
|
||||
RequestedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := s.repo.CreateRun(ctx, req); err != nil {
|
||||
return fmt.Errorf("enqueue run: %w", err)
|
||||
insertErr := s.repo.CreateRun(ctx, req)
|
||||
outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, idempotencyKey, agentInstanceID, insertErr)
|
||||
if err != nil {
|
||||
return runsservice.QueueOutcomeNone, fmt.Errorf("enqueue run: %w", err)
|
||||
}
|
||||
if outcome == runsservice.QueueOutcomeDeduped {
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
s.logger.Info("run queued",
|
||||
@@ -155,7 +157,7 @@ func (s *Service) queueRunInline(
|
||||
zap.String("reason", reason))
|
||||
|
||||
s.publishRunQueued(ctx, req, idempotencyKey)
|
||||
return nil
|
||||
return runsservice.QueueOutcomeQueued, nil
|
||||
}
|
||||
|
||||
// payloadWithAgent decodes the JSON payload string and adds the
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) {
|
||||
|
||||
// Queue + claim a run for the task so handleAgentTurnMessageSaved
|
||||
// resolves a runID via GetClaimedRunByTaskID.
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "lifecycle-init",
|
||||
); err != nil {
|
||||
@@ -102,7 +102,7 @@ func TestRunLifecycle_StepCompleteEventsEmitted(t *testing.T) {
|
||||
// no longer find a claimed run. To exercise the persistence path
|
||||
// directly, we re-queue + claim once more and seed an activity
|
||||
// row tagged with the run id.
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskComment,
|
||||
`{"task_id":"`+taskID+`"}`, "lifecycle-second",
|
||||
); err != nil {
|
||||
@@ -131,7 +131,7 @@ func TestRunLifecycle_ErrorEventEmitted(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "lifecycle-error",
|
||||
); err != nil {
|
||||
@@ -249,7 +249,7 @@ func TestQueueRun_PublishesOfficeRunQueued(t *testing.T) {
|
||||
|
||||
const idemKey = "task_comment:cm-123"
|
||||
payload := `{"task_id":"` + taskID + `","comment_id":"cm-123"}`
|
||||
if err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, "worker-1", service.RunReasonTaskComment, payload, idemKey); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
@@ -293,7 +293,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskComment,
|
||||
`{"task_id":"`+taskID+`","comment_id":"cm-1"}`, "task_comment:cm-1",
|
||||
); err != nil {
|
||||
@@ -353,7 +353,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T)
|
||||
sourceTaskID := "source-task"
|
||||
insertTestTask(t, svc, sourceTaskID, "ws-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskComment,
|
||||
`{"task_id":"`+targetTaskID+`","source_task_id":"`+sourceTaskID+`","comment_id":"cm-source"}`,
|
||||
"task_comment:cm-source:target",
|
||||
@@ -406,7 +406,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-1")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-1")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-1", service.RunReasonTaskComment,
|
||||
`{"task_id":"`+taskID+`","comment_id":"cm-2"}`, "task_comment:cm-2",
|
||||
); err != nil {
|
||||
|
||||
@@ -2,12 +2,39 @@ package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// windowedDedupCounterValue reads the current value of one
|
||||
// office_run_dedup_total{reason,kind=windowed,queue=runs} label, or 0 if the
|
||||
// label has never been reported. Callers snapshot before/after and assert
|
||||
// the exact delta.
|
||||
func windowedDedupCounterValue(t *testing.T, reason string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_run_dedup_total")
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map office_run_dedup_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("office_run_dedup_total is not a *expvar.Map")
|
||||
}
|
||||
iv := m.Get("reason=" + reason + ";kind=windowed;queue=runs")
|
||||
if iv == nil {
|
||||
return 0
|
||||
}
|
||||
i, ok := iv.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("counter value for reason=%s;kind=windowed;queue=runs is not *expvar.Int", reason)
|
||||
}
|
||||
return i.Value()
|
||||
}
|
||||
|
||||
func TestQueueRun_Basic(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
@@ -17,7 +44,7 @@ func TestQueueRun_Basic(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1")
|
||||
_, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "key-1")
|
||||
if err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
@@ -47,11 +74,11 @@ func TestQueueRun_Idempotency(t *testing.T) {
|
||||
}
|
||||
|
||||
key := "idem-key-1"
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil {
|
||||
t.Fatalf("first enqueue: %v", err)
|
||||
}
|
||||
// Second enqueue with same key should be silently dropped.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key); err != nil {
|
||||
t.Fatalf("second enqueue: %v", err)
|
||||
}
|
||||
|
||||
@@ -61,6 +88,46 @@ func TestQueueRun_Idempotency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestQueueRun_Idempotency_ReportsWindowedDedupOutcome closes a phantom-green
|
||||
// gap: TestQueueRun_Idempotency above only asserted the resulting row count,
|
||||
// so deleting queueRunInline's ReportWindowedDedup call would leave it green
|
||||
// while AC-OFFICE-RUN-DEDUP-004.1's observability contract silently went
|
||||
// unmet on this queue implementation. Asserts the outcome value and the
|
||||
// office_run_dedup_total counter directly.
|
||||
func TestQueueRun_Idempotency_ReportsWindowedDedupOutcome(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-1", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
key := "idem-key-outcome"
|
||||
first, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key)
|
||||
if err != nil {
|
||||
t.Fatalf("first enqueue: %v", err)
|
||||
}
|
||||
if first != runsservice.QueueOutcomeQueued {
|
||||
t.Fatalf("first outcome = %v, want QueueOutcomeQueued", first)
|
||||
}
|
||||
|
||||
before := windowedDedupCounterValue(t, service.RunReasonTaskAssigned)
|
||||
|
||||
second, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", key)
|
||||
if err != nil {
|
||||
t.Fatalf("second enqueue: %v", err)
|
||||
}
|
||||
if second != runsservice.QueueOutcomeDeduped {
|
||||
t.Fatalf("second outcome = %v, want QueueOutcomeDeduped", second)
|
||||
}
|
||||
|
||||
after := windowedDedupCounterValue(t, service.RunReasonTaskAssigned)
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_run_dedup_total{%s,windowed,runs} = %d, want %d", service.RunReasonTaskAssigned, after, before+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueRun_SkipsPausedAgent(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
@@ -74,7 +141,7 @@ func TestQueueRun_SkipsPausedAgent(t *testing.T) {
|
||||
t.Fatalf("pause agent: %v", err)
|
||||
}
|
||||
|
||||
err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "")
|
||||
_, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for paused agent")
|
||||
}
|
||||
@@ -97,7 +164,7 @@ func TestQueueRun_SkipsStoppedAgent(t *testing.T) {
|
||||
t.Fatalf("stop agent: %v", err)
|
||||
}
|
||||
|
||||
err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "")
|
||||
_, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for stopped agent")
|
||||
}
|
||||
@@ -113,10 +180,10 @@ func TestQueueRun_Coalesce(t *testing.T) {
|
||||
}
|
||||
|
||||
// Two runs with the same agent + reason within coalesce window should merge.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("first: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("second: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestSchedulerTick_ContendedCheckoutRequeuesRun(t *testing.T) {
|
||||
t.Fatalf("seed checkout: ok=%v err=%v", ok, err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-contended-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestSchedulerTick_CheckoutErrorRetriesInsteadOfFalseFinish(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-checkout-db-error-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestSchedulerTick_InactiveAgentRunDoesNotStealCheckout(t *testing.T) {
|
||||
// Queue while idle (QueueRun itself refuses to queue for a paused
|
||||
// agent), then go paused before the tick runs - reproducing the race
|
||||
// where an agent is deactivated between queueing and processing.
|
||||
if err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, inactive.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-inactive-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func TestSchedulerTick_SameAgentPreCheckoutRunDoesNotStealOwnLiveCheckout(t *tes
|
||||
// queued while idle, then the agent goes paused before the tick picks
|
||||
// it up - reproducing the race where the holder is deactivated after
|
||||
// a second run was already queued behind its own live run.
|
||||
if err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, holder.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-same-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ func TestSchedulerTick_AgentCompletedReleasesTaskCheckout(t *testing.T) {
|
||||
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-checkout-release-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-checkout-release-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func TestSchedulerTick_AgentFailedReleasesTaskCheckout(t *testing.T) {
|
||||
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-checkout-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-checkout-fail-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -228,7 +228,7 @@ func TestSchedulerTick_TasklessAgentCompletedStampsRuntime(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
claimed, err := svc.ClaimNextRun(ctx)
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestCooldown_RecentFinish_GuardAllows(t *testing.T) {
|
||||
}
|
||||
|
||||
// Queue a run.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestCooldown_PastCooldown_ClaimedNormally(t *testing.T) {
|
||||
}
|
||||
|
||||
// Queue a run.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ func TestRetry_FailedRun_RetriedWithBackoff(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestRetry_FifthFailure_MarkedFailed(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -402,7 +402,7 @@ func TestIdleSkip_HeartbeatNoTasks_Skipped(t *testing.T) {
|
||||
}
|
||||
// Worker defaults to skip_idle_runs=true, no tasks assigned.
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -457,7 +457,7 @@ func TestIdleSkip_HeartbeatWithActionableTasks_Proceeds(t *testing.T) {
|
||||
// Assign an IN_PROGRESS task to this agent.
|
||||
insertActionableTask(t, svc, "task-inprog-1", agent.ID, "IN_PROGRESS")
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -499,7 +499,7 @@ func TestIdleSkip_HeartbeatSkipDisabled_Proceeds(t *testing.T) {
|
||||
`UPDATE agent_profiles SET skip_idle_runs = 0 WHERE id = ?`, agent.ID)
|
||||
|
||||
// No tasks assigned.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -535,7 +535,7 @@ func TestIdleSkip_NonHeartbeatRun_NotSkipped(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-event-1', 'ws-1', 'Event Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-event-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -569,7 +569,7 @@ func TestIdleSkip_CEODefaultFalse_NotSkipped(t *testing.T) {
|
||||
}
|
||||
// CEO has no tasks, but skip_idle_runs defaults to false.
|
||||
|
||||
if err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, ceo.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestSchedulerIntegration_RoutingReceivesBuiltPromptAndEnv(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at)
|
||||
VALUES ('task-routing-1', 'ws-1', 'ROUTING_PROMPT_SENTINEL_TITLE',
|
||||
'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-routing-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func TestSchedulerIntegration_SeatActionFlowsToPromptAndLaunch(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
|
||||
"task-decision", "ws-1", "step-decision", "Decision task", "Review the change")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-decision"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -263,7 +263,7 @@ func TestSchedulerIntegration_NonSeatRunDoesNotReceiveDecisionSkill(t *testing.T
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, workflow_step_id, title, description, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
|
||||
"task-non-seat", "ws-1", "step-non-seat", "Non-seat task", "Review the change")
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-non-seat"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -317,7 +317,7 @@ func TestSchedulerIntegration_RoutingFallThrough_FallsBackToLegacy(t *testing.T)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-ft-1', 'ws-1', 'Fall-through Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-ft-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -361,7 +361,7 @@ func TestSchedulerIntegration_RoutingParked_LeavesAgentIdle(t *testing.T) {
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-parked-1', 'ws-1', 'Parked Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-parked-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -402,7 +402,7 @@ func TestSchedulerIntegration_RoutingDispatchError_LeavesAgentIdle(t *testing.T)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-routing-err-1', 'ws-1', 'Routing Error Task', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-routing-err-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) {
|
||||
|
||||
// The run was queued while the task was on step-old. A later workflow
|
||||
// move must prevent the scheduler from launching that stale step.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-moved-step","workflow_step_id":"step-old"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
@@ -150,7 +150,7 @@ func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) {
|
||||
VALUES ('task-project-exec', 'ws-1', ?, 'Project executor task', 'desc',
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, project.ID)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-project-exec"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
@@ -205,7 +205,7 @@ func TestSchedulerIntegration_PausedAgentSkipped(t *testing.T) {
|
||||
}
|
||||
|
||||
// Queue while agent is active.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -244,10 +244,10 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) {
|
||||
}
|
||||
|
||||
// Queue two runs.
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, "k1"); err != nil {
|
||||
t.Fatalf("queue first: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskComment, `{"task_id":"t2"}`, "k2"); err != nil {
|
||||
t.Fatalf("queue second: %v", err)
|
||||
}
|
||||
|
||||
@@ -330,7 +330,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, tt.reason, tt.payload, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// maxRecoveryPerTick caps the number of unstarted tasks recovered in one tick.
|
||||
@@ -69,7 +70,8 @@ func (si *SchedulerIntegration) recoverUnstartedTasks(ctx context.Context, log *
|
||||
zap.String("agent_profile_id", t.AssigneeAgentProfileID))
|
||||
|
||||
payload := mustJSON(map[string]string{"task_id": t.ID})
|
||||
if err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID,
|
||||
runsservice.ReportKeylessEnqueue(RunReasonTaskAssigned, runsservice.KeylessCauseByDesign, "")
|
||||
if _, err := si.svc.QueueRun(ctx, t.AssigneeAgentProfileID,
|
||||
RunReasonTaskAssigned, payload, ""); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestSchedulerOutcome_AgentInactive_WritesOutcome(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ func TestSchedulerOutcome_IdleSkipped_WritesOutcome(t *testing.T) {
|
||||
}
|
||||
// Worker defaults to skip_idle_runs=true, no tasks assigned.
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestSchedulerOutcome_TaskTreeHeld_WritesOutcome(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-tree-outcome"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -184,7 +184,7 @@ func TestSchedulerOutcome_CheckoutError_RetriesInsteadOfFinishing(t *testing.T)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-checkout-error"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -250,7 +250,7 @@ func TestSchedulerOutcome_CheckoutUnavailable_RetriesInsteadOfFinishing(t *testi
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-checkout-unavail"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -286,7 +286,7 @@ func TestSchedulerOutcome_BudgetBlocked_WritesOutcome(t *testing.T) {
|
||||
insertTestTask(t, svc, "task-budget-outcome", "ws-1")
|
||||
insertTestCostEvent(t, svc, agent.ID, "task-budget-outcome", int64(600))
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-budget-outcome"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -312,7 +312,7 @@ func TestSchedulerOutcome_NoTaskStarter_FailsRun(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-no-launch"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -353,7 +353,7 @@ func TestSchedulerOutcome_Processed_WritesOutcome(t *testing.T) {
|
||||
createTestAgent(t, svc, "ws-1", "worker-processed")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-processed")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
if _, err := svc.QueueRun(
|
||||
ctx, "worker-processed", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "processed-outcome",
|
||||
); err != nil {
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestClaimNextRun_ClaimsQueued(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func TestFinishAndFailRun(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -178,7 +178,7 @@ func TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome(t *testing.T) {
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
req, _ := svc.ClaimNextRun(ctx)
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) {
|
||||
|
||||
// Mirrors wakeup/dispatcher.go's createFreshRun: reason from the
|
||||
// routine trigger, payload literally "{}" (no task_id).
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -196,7 +196,7 @@ func TestSchedulerTick_TaskBoundRunStillLaunches(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-wo35-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -256,7 +256,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) {
|
||||
// mirrors 3 ticks of the pre-installed coordinator heartbeat routine.
|
||||
const firesAtThreshold = 3
|
||||
for i := 0; i < firesAtThreshold; i++ {
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue taskless run %d: %v", i, err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
@@ -281,7 +281,7 @@ func TestSchedulerTick_TasklessRunsDoNotAutoPauseAgent(t *testing.T) {
|
||||
// failures: a task-bound run queued afterwards must still launch.
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-wo35-pause-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-wo35-pause-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue task-bound run: %v", err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
@@ -330,7 +330,7 @@ func TestSchedulerTick_TasklessRunFailure_PublishesResolvableWorkspaceEvent(t *t
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
@@ -391,7 +391,7 @@ func TestSchedulerTick_UnlaunchableRun_PublishesResolvableWorkspaceEvent(t *test
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-wo35-unlaunchable"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
@@ -446,7 +446,7 @@ func TestSchedulerTick_RepeatTasklessFailures_OnlyFirstStaysInInbox(t *testing.T
|
||||
const fires = 3
|
||||
var firstRunID string
|
||||
for i := 0; i < fires; i++ {
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue taskless run %d: %v", i, err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
@@ -512,7 +512,7 @@ func TestSchedulerTick_RepeatTasklessFailures_StayVisiblePerRoutineScope(t *test
|
||||
|
||||
queueRoutineFailure := func(scope, idempotencyKey string) string {
|
||||
t.Helper()
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, idempotencyKey); err != nil {
|
||||
t.Fatalf("queue routine %s: %v", scope, err)
|
||||
}
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package service_test
|
||||
|
||||
// This file proves the specific defect run-dedup-generation fixes: a
|
||||
// task_assigned dedup key that used to be stable for the life of a
|
||||
// (task, agent) pair (docs/specs/office/requirements/run-dedup-generation.md,
|
||||
// AC-OFFICE-SCHEDULER-001.7). idx_run_idempotency has no time bound, so a
|
||||
// stable key collided with the UNIQUE index forever, making a re-assignment
|
||||
// of the same task to the same agent a silent, permanent no-op. These tests
|
||||
// exercise the real bus -> handleTaskUpdated -> queueTaskAssignedRun path
|
||||
// (not just the dedupkeys.AssignmentKey unit) and assert on the persisted
|
||||
// idempotency_key, which none of the existing task_assigned tests do.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
"github.com/kandev/kandev/internal/runs/dedupkeys"
|
||||
)
|
||||
|
||||
func publishTaskAssigned(
|
||||
t *testing.T, ctx context.Context, eb bus.EventBus, taskID, agentID string, generation *int64,
|
||||
) {
|
||||
t.Helper()
|
||||
data := map[string]any{
|
||||
"task_id": taskID,
|
||||
"assignee_agent_profile_id": agentID,
|
||||
}
|
||||
if generation != nil {
|
||||
data["assignment_generation"] = *generation
|
||||
}
|
||||
event := bus.NewEvent(events.TaskUpdated, "test", data)
|
||||
if err := eb.Publish(ctx, events.TaskUpdated, event); err != nil {
|
||||
t.Fatalf("publish task updated event: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func gen(n int64) *int64 { return &n }
|
||||
|
||||
// taskAssignedRunsFor returns every task_assigned run queued for agentID on
|
||||
// the given task, in insertion order (ListRuns has no ordering guarantee
|
||||
// this test relies on beyond "all rows present").
|
||||
func taskAssignedRunsFor(t *testing.T, svc *service.Service, wsID, taskID, agentID string) []string {
|
||||
t.Helper()
|
||||
runs, err := svc.ListRuns(context.Background(), wsID)
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
var keys []string
|
||||
for _, run := range runs {
|
||||
if run.AgentProfileID != agentID || run.Reason != service.RunReasonTaskAssigned {
|
||||
continue
|
||||
}
|
||||
var payload struct {
|
||||
TaskID string `json:"task_id"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(run.Payload), &payload); err != nil {
|
||||
t.Fatalf("decode run payload: %v", err)
|
||||
}
|
||||
if payload.TaskID != taskID {
|
||||
continue
|
||||
}
|
||||
if run.IdempotencyKey == nil {
|
||||
keys = append(keys, "")
|
||||
continue
|
||||
}
|
||||
keys = append(keys, *run.IdempotencyKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestTaskAssigned_SameGenerationRedelivery_DedupesForever(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-gen")
|
||||
insertTestTask(t, svc, "task-gen-redeliver", "ws-1")
|
||||
svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-gen-redeliver")
|
||||
|
||||
publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0))
|
||||
publishTaskAssigned(t, ctx, eb, "task-gen-redeliver", "worker-gen", gen(0))
|
||||
|
||||
keys := taskAssignedRunsFor(t, svc, "ws-1", "task-gen-redeliver", "worker-gen")
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("redelivering the same generation queued %d runs, want exactly 1 (deduped): keys=%#v", len(keys), keys)
|
||||
}
|
||||
want := dedupkeys.AssignmentKey("task-gen-redeliver", "worker-gen", 0)
|
||||
if keys[0] != want {
|
||||
t.Fatalf("idempotency key = %q, want %q", keys[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskAssigned_GenerationBump_AvoidsPermanentCollision is the direct
|
||||
// regression test for the reported defect: reassigning the same task to the
|
||||
// same agent must NOT be a permanent no-op once the durable unique index
|
||||
// already holds a row for an earlier generation's key. Unlike
|
||||
// TestTaskAssigned_ReassignmentUsesAgentScopedIdempotency (which reassigns to
|
||||
// a *different* agent and never sets assignment_generation, so it never
|
||||
// exercised the keyed path at all), this drives the same agent through two
|
||||
// generations to prove the key itself changed.
|
||||
func TestTaskAssigned_GenerationBump_AvoidsPermanentCollision(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-regen")
|
||||
insertTestTask(t, svc, "task-regen", "ws-1")
|
||||
svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-regen")
|
||||
|
||||
// Occurrence 1: original assignment at generation 0. In production this
|
||||
// row's idempotency_key (task_assigned:task-regen:worker-regen:0) is
|
||||
// exactly the durable UNIQUE row that used to collide forever under the
|
||||
// pre-fix key format (task_assigned:<task>:<agent>, no generation).
|
||||
publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(0))
|
||||
|
||||
// The first run is claimed and finishes (agent did the work) long before
|
||||
// the reassignment — CoalesceRun only merges into a still-'queued' row,
|
||||
// so this ensures the second occurrence exercises the durable
|
||||
// idx_run_idempotency INSERT path, not the 5-second coalesce window.
|
||||
// This is exactly the "more than 24 hours later" gap in the bug report.
|
||||
svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`)
|
||||
|
||||
// Simulate a real reassignment: UpdateTaskAssignee bumps the stored
|
||||
// generation before the task.updated event carrying it is published.
|
||||
svc.ExecSQL(t, `UPDATE tasks SET assignment_generation = 1 WHERE id = ?`, "task-regen")
|
||||
publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1))
|
||||
|
||||
keys := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen")
|
||||
if len(keys) != 2 {
|
||||
t.Fatalf("reassigning the same agent at a new generation queued %d runs, want 2 (no false collision): keys=%#v", len(keys), keys)
|
||||
}
|
||||
wantGen0 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 0)
|
||||
wantGen1 := dedupkeys.AssignmentKey("task-regen", "worker-regen", 1)
|
||||
matchesInOrder := keys[0] == wantGen0 && keys[1] == wantGen1
|
||||
matchesReversed := keys[0] == wantGen1 && keys[1] == wantGen0
|
||||
if !matchesInOrder && !matchesReversed {
|
||||
t.Fatalf("keys = %#v, want %q and %q (one run per generation)", keys, wantGen0, wantGen1)
|
||||
}
|
||||
if keys[0] == keys[1] {
|
||||
t.Fatalf("both runs share idempotency key %q — this is the reported defect: a reassignment collided with the durable unique index", keys[0])
|
||||
}
|
||||
|
||||
// Redelivering generation 1 again — past the coalesce window and after
|
||||
// the generation-1 run has itself been claimed — must still dedupe via
|
||||
// the durable unique index: the fix must not turn every event into a
|
||||
// fresh key, only a genuine generation change.
|
||||
svc.ExecSQL(t, `UPDATE runs SET status = 'completed' WHERE status = 'queued'`)
|
||||
publishTaskAssigned(t, ctx, eb, "task-regen", "worker-regen", gen(1))
|
||||
keysAfterRedeliver := taskAssignedRunsFor(t, svc, "ws-1", "task-regen", "worker-regen")
|
||||
if len(keysAfterRedeliver) != 2 {
|
||||
t.Fatalf("redelivering generation 1 queued a 3rd run: keys=%#v", keysAfterRedeliver)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskAssigned_MissingGeneration_EnqueuesKeyless proves SR-55's
|
||||
// resolution: an event carrying no assignment_generation must never mint a
|
||||
// key with a guessed or zero generation — it goes keyless (empty
|
||||
// idempotency_key) so a possible duplicate wake is accepted (AC-003.2)
|
||||
// instead of a mis-keyed row that could collide with an unrelated
|
||||
// occurrence.
|
||||
func TestTaskAssigned_MissingGeneration_EnqueuesKeyless(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-nogen")
|
||||
insertTestTask(t, svc, "task-nogen", "ws-1")
|
||||
svc.ExecSQL(t, `UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "task-nogen")
|
||||
|
||||
publishTaskAssigned(t, ctx, eb, "task-nogen", "worker-nogen", nil)
|
||||
|
||||
keys := taskAssignedRunsFor(t, svc, "ws-1", "task-nogen", "worker-nogen")
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("got %d runs, want 1: keys=%#v", len(keys), keys)
|
||||
}
|
||||
if keys[0] != "" {
|
||||
t.Fatalf("idempotency key = %q, want empty (keyless enqueue)", keys[0])
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,10 @@ const participantTypeAgent = "agent"
|
||||
// by tests and a couple of internal callers; the dashboard's
|
||||
// permissioned variant is SetTaskAssigneeAsAgent.
|
||||
func (s *Service) SetTaskAssignee(ctx context.Context, taskID, assigneeID string) error {
|
||||
return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID)
|
||||
// generation discarded — this caller does not carry it to the event bus;
|
||||
// queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003).
|
||||
_, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetTaskAssigneeAsAgent checks can_assign_tasks for the given caller
|
||||
@@ -55,5 +58,8 @@ func (s *Service) SetTaskAssigneeAsAgent(ctx context.Context, callerAgentID, tas
|
||||
return shared.ErrForbidden
|
||||
}
|
||||
}
|
||||
return s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID)
|
||||
// generation discarded — this caller does not carry it to the event bus;
|
||||
// queueTaskAssignedRun falls through to keyless enqueue (AC-OFFICE-RUN-DEDUP-003).
|
||||
_, err := s.repo.UpdateTaskAssignee(ctx, taskID, assigneeID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -99,7 +99,7 @@ func TestSchedulerTick_LaunchesAgent(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, priority, created_at, updated_at)
|
||||
VALUES ('task-launch-1', 'ws-1', 'Build API', 'Implement endpoint', 'medium', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-launch-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ func TestSchedulerTick_LaunchIncludesRuntimeTokenEnv(t *testing.T) {
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-token-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-token-1","session_id":"sess-token"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -192,7 +192,7 @@ func TestSchedulerTick_SnapshotsRunSkills(t *testing.T) {
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-skill-1', 'ws-1', 'Review API', 'Review endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-skill-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -239,7 +239,7 @@ func TestSchedulerTick_KeepsRunClaimedUntilAgentCompletes(t *testing.T) {
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-life-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-life-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -301,7 +301,7 @@ func TestSchedulerTick_AgentStoppedFinishesRun(t *testing.T) {
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-stop-1', 'ws-1', 'Stop handler test', 'desc', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-stop-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -386,7 +386,7 @@ func TestSchedulerTick_StartTaskError_TriggersRetry(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-fail-1', 'ws-1', 'Failing Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-fail-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -439,7 +439,7 @@ func TestSchedulerTick_NoTaskStarter_FailsRunLoudly(t *testing.T) {
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-noop-1', 'ws-1', 'NoOp Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-noop-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -494,7 +494,7 @@ func TestSchedulerTick_NoTaskStarter_UpdatesFailureAccounting(t *testing.T) {
|
||||
|
||||
for i := 0; i < threshold; i++ {
|
||||
idempotencyKey := fmt.Sprintf("task-noop-accounting-%d", i)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-noop-accounting"}`, idempotencyKey); err != nil {
|
||||
t.Fatalf("queue failure %d: %v", i, err)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestIdleSkip_RoutineDispatchNoTasks_Skipped(t *testing.T) {
|
||||
}
|
||||
// Worker defaults to skip_idle_runs=true, no tasks assigned.
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ func TestIdleSkip_RoutineDispatch_CoordinatorNotSkipped(t *testing.T) {
|
||||
t.Fatalf("CEO role should default to SkipIdleRuns=false")
|
||||
}
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil {
|
||||
if _, err := svc.QueueRun(ctx, agent.ID, shared.RunReasonRoutineDispatchCron, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
"github.com/kandev/kandev/internal/workflow/engine"
|
||||
)
|
||||
|
||||
@@ -39,8 +40,11 @@ type AgentWriter interface {
|
||||
// Implemented by the run feature (and transitionally by office/service.Service).
|
||||
type RunQueuer interface {
|
||||
// QueueRun enqueues a run for agentInstanceID with the given reason, payload,
|
||||
// and optional idempotency key (empty string disables deduplication).
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) error
|
||||
// and optional idempotency key (empty string disables deduplication). The
|
||||
// returned QueueOutcome reports what actually happened (queued / deduped /
|
||||
// coalesced / none-on-error) so callers that need to distinguish a fresh
|
||||
// insert from a no-op don't have to infer it from side effects.
|
||||
QueueRun(ctx context.Context, agentInstanceID, reason, payload, idempotencyKey string) (runsservice.QueueOutcome, error)
|
||||
}
|
||||
|
||||
// WorkflowEngineDispatcher routes typed office task events through the
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/orchestrator/executor"
|
||||
"github.com/kandev/kandev/internal/orchestrator/messagequeue"
|
||||
"github.com/kandev/kandev/internal/orchestrator/watcher"
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
"github.com/kandev/kandev/internal/steptelemetry"
|
||||
"github.com/kandev/kandev/internal/sysprompt"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
@@ -2179,13 +2180,16 @@ func (s *Service) queueOfficeAutoStartRun(ctx context.Context, task *models.Task
|
||||
}
|
||||
|
||||
// officeAutoStartIdempotencyKey uses the immutable workflow-step transition
|
||||
// row as the per-entry component. A legacy event without that field uses the
|
||||
// task timestamp as a compatibility fallback until the event is republished.
|
||||
// row as the per-entry component. A zero stepTransitionID means the
|
||||
// per-occurrence identity is unavailable, so the enqueue goes keyless rather
|
||||
// than falling back to a time-derived key that would never suppress a
|
||||
// redelivery.
|
||||
func officeAutoStartIdempotencyKey(task *models.Task, agentProfileID, stepID string, stepTransitionID int64) string {
|
||||
entryID := strconv.FormatInt(stepTransitionID, 10)
|
||||
if stepTransitionID == 0 {
|
||||
entryID = "legacy:" + task.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
||||
runsservice.ReportKeylessEnqueue(officeAutoStartRunReason, runsservice.KeylessCauseUnresolved, "zero_step_transition")
|
||||
return ""
|
||||
}
|
||||
entryID := strconv.FormatInt(stepTransitionID, 10)
|
||||
return fmt.Sprintf("%s:%s:%s:%s:%s",
|
||||
officeAutoStartRunReason, task.ID, agentProfileID, stepID, entryID)
|
||||
}
|
||||
|
||||
@@ -36,6 +36,20 @@ func TestOfficeAutoStartIdempotencyKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless pins that a
|
||||
// zero StepTransitionID (no per-occurrence identity available) enqueues with
|
||||
// no key at all, rather than falling back to a time-derived key that would
|
||||
// never suppress a genuine redelivery.
|
||||
func TestOfficeAutoStartIdempotencyKey_ZeroStepTransitionGoesKeyless(t *testing.T) {
|
||||
task := &models.Task{ID: "t1", UpdatedAt: time.Now().UTC(), WorkflowStepTransitionID: 0}
|
||||
|
||||
key := officeAutoStartIdempotencyKey(task, "agent-1", "step2", task.WorkflowStepTransitionID)
|
||||
|
||||
if key != "" {
|
||||
t.Errorf("key = %q, want empty (keyless) for zero StepTransitionID", key)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoStartOfficeTaskLogsQueuedOutcomeAtInfo pins that a real insert
|
||||
// (QueueOutcomeQueued) is logged as an info-level "queued" message, and the
|
||||
// log is only emitted after the queue attempt resolves — not asserted
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// Package dedupkeys centralizes the shared dedup-key builders that more than
|
||||
// one office producer must derive identically for the same occurrence
|
||||
// (docs/specs/office/system-design/run-dedup-generation-01.md#convergent-producers,
|
||||
// #the-shared-key-builders). Placed beside internal/runs/commentkeys as a
|
||||
// sibling package so the office service, the office scheduler and the
|
||||
// orchestrator can all reach it without an import cycle.
|
||||
package dedupkeys
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AssignmentKey builds the canonical task_assigned dedup key. Both
|
||||
// task_assigned producers (office/service's queueTaskAssignedRun and
|
||||
// office/scheduler's reactToAssigneeChange) call this rather than
|
||||
// formatting the string themselves, which is what makes their convergence
|
||||
// structural instead of coincidental (AC-OFFICE-RUN-DEDUP-002.1).
|
||||
func AssignmentKey(taskID, agentProfileID string, generation int64) string {
|
||||
return fmt.Sprintf("task_assigned:%s:%s:%d", taskID, agentProfileID, generation)
|
||||
}
|
||||
|
||||
// BlockerDigest computes the shared digest over a blocker-task-id set. Both
|
||||
// blocker producers (office/service's resolveAndWakeIfUnblocked and
|
||||
// office/scheduler's cascadeBlockersResolved) call this so a set observed in
|
||||
// different read order still converges on the same digest. The encoding is
|
||||
// binding: sort ascending by byte value (NOT ListTaskBlockers's created_at,
|
||||
// which is not unique), join with a single comma, then SHA-256 the UTF-8
|
||||
// bytes and render the full digest as lowercase hex.
|
||||
func BlockerDigest(blockerTaskIDs []string) string {
|
||||
sorted := make([]string, len(blockerTaskIDs))
|
||||
copy(sorted, blockerTaskIDs)
|
||||
sort.Strings(sorted)
|
||||
sum := sha256.Sum256([]byte(strings.Join(sorted, ",")))
|
||||
return fmt.Sprintf("%x", sum)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package dedupkeys
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAssignmentKey(t *testing.T) {
|
||||
got := AssignmentKey("task-1", "agent-1", 3)
|
||||
want := "task_assigned:task-1:agent-1:3"
|
||||
if got != want {
|
||||
t.Fatalf("AssignmentKey() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignmentKey_VariesByGeneration(t *testing.T) {
|
||||
a := AssignmentKey("task-1", "agent-1", 1)
|
||||
b := AssignmentKey("task-1", "agent-1", 2)
|
||||
if a == b {
|
||||
t.Fatalf("expected distinct keys for distinct generations, got %q for both", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockerDigest_OrderIndependent(t *testing.T) {
|
||||
a := BlockerDigest([]string{"b1", "b2", "b3"})
|
||||
b := BlockerDigest([]string{"b3", "b1", "b2"})
|
||||
if a != b {
|
||||
t.Fatalf("BlockerDigest should be order-independent: %q != %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockerDigest_VariesBySet(t *testing.T) {
|
||||
a := BlockerDigest([]string{"b1", "b2"})
|
||||
b := BlockerDigest([]string{"b1", "b2", "b3"})
|
||||
if a == b {
|
||||
t.Fatalf("expected distinct digests for distinct sets, got %q for both", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockerDigest_DoesNotMutateInput(t *testing.T) {
|
||||
input := []string{"b3", "b1", "b2"}
|
||||
_ = BlockerDigest(input)
|
||||
if input[0] != "b3" || input[1] != "b1" || input[2] != "b2" {
|
||||
t.Fatalf("BlockerDigest mutated its input slice: %v", input)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
// QueueOutcomeNone means no enqueue was attempted, or the attempt returned
|
||||
// an error. It is the zero value, so it is what a widened signature yields
|
||||
// on any path that returns before deciding an outcome.
|
||||
//
|
||||
// It is declared identically in internal/workflow/engine/adapters.go — both
|
||||
// declarations MUST match, the same invariant QueueOutcome itself already
|
||||
// carries.
|
||||
const QueueOutcomeNone QueueOutcome = ""
|
||||
|
||||
// QueueSource distinguishes which queue implementation made a dedup
|
||||
// decision, for the "queue" label on office_run_dedup_total.
|
||||
type QueueSource string
|
||||
|
||||
const (
|
||||
// QueueSourceRuns identifies a suppression on the runs table (the
|
||||
// idx_run_idempotency windowed lookup or durable unique index).
|
||||
QueueSourceRuns QueueSource = "runs"
|
||||
// QueueSourceWakeup identifies a suppression on the wakeup-request
|
||||
// table. That table has no windowed lookup, so its kind is always
|
||||
// "durable".
|
||||
QueueSourceWakeup QueueSource = "wakeup"
|
||||
)
|
||||
|
||||
// KeylessCause discriminates why a producer enqueued a wake with no dedup
|
||||
// key: a generation that should have been available and was not
|
||||
// (KeylessCauseUnresolved), or a producer that never had an occurrence to
|
||||
// name (KeylessCauseByDesign). See
|
||||
// docs/specs/office/system-design/run-dedup-generation-01.md#unresolvable-generation.
|
||||
type KeylessCause string
|
||||
|
||||
const (
|
||||
// KeylessCauseUnresolved means the occurrence has a generation that this
|
||||
// producer simply was not handed or could not resolve.
|
||||
KeylessCauseUnresolved KeylessCause = "unresolved"
|
||||
// KeylessCauseByDesign means the producer never had an occurrence
|
||||
// identity to name — a status transition with no redelivery path, an
|
||||
// agent that expressed no dedup intent, and similar.
|
||||
KeylessCauseByDesign KeylessCause = "by_design"
|
||||
)
|
||||
|
||||
func dedupLogger() *logger.Logger {
|
||||
return logger.Default().WithFields(zap.String("component", "runs-dedup"))
|
||||
}
|
||||
|
||||
// ReportWindowedDedup records a suppression caught by the recent-duplicate
|
||||
// lookup (a windowed dedup hit): counts kind="windowed", logs at Info, and
|
||||
// returns QueueOutcomeDeduped.
|
||||
func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome {
|
||||
incRunDedup(q, reason, "windowed")
|
||||
dedupLogger().Info("run deduplicated (windowed)",
|
||||
zap.String("queue", string(q)),
|
||||
zap.String("reason", reason),
|
||||
zap.String("key", key))
|
||||
return QueueOutcomeDeduped
|
||||
}
|
||||
|
||||
// ReportInsertResult classifies the error from an insert attempt. A unique
|
||||
// violation on idx_run_idempotency counts kind="durable", logs at Warn with
|
||||
// key, reason and agent, and returns (QueueOutcomeDeduped, nil). Any other
|
||||
// non-nil error is returned UNCHANGED with QueueOutcomeNone and moves no
|
||||
// counter — a disk error is not a dedup decision. A nil error returns
|
||||
// (QueueOutcomeQueued, nil).
|
||||
func ReportInsertResult(
|
||||
q QueueSource, reason, key, agentProfileID string, err error,
|
||||
) (QueueOutcome, error) {
|
||||
if err == nil {
|
||||
return QueueOutcomeQueued, nil
|
||||
}
|
||||
if runssqlite.IsIdempotencyKeyUniqueViolation(err) {
|
||||
return ReportDurableDedup(q, reason, key, agentProfileID), nil
|
||||
}
|
||||
return QueueOutcomeNone, err
|
||||
}
|
||||
|
||||
// ReportDurableDedup records a durable conflict the caller has already
|
||||
// classified (the wakeup path's ErrWakeupIdempotencyConflict, or
|
||||
// ReportInsertResult's own classification): counts kind="durable", logs at
|
||||
// Warn, and returns QueueOutcomeDeduped.
|
||||
func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome {
|
||||
incRunDedup(q, reason, "durable")
|
||||
dedupLogger().Warn("run deduplicated (durable index)",
|
||||
zap.String("queue", string(q)),
|
||||
zap.String("reason", reason),
|
||||
zap.String("key", key),
|
||||
zap.String("agent_profile_id", agentProfileID))
|
||||
return QueueOutcomeDeduped
|
||||
}
|
||||
|
||||
// ReportKeylessEnqueue records a producer's decision to enqueue a wake with
|
||||
// no dedup key, attributed to reason and discriminated by cause so a
|
||||
// resolution failure (unresolved) is countable separately from expected,
|
||||
// by-design keyless traffic. Called by the producer at its own decision
|
||||
// site, immediately before enqueuing — the queue itself never infers a
|
||||
// cause from an empty key. detail names what failed to resolve (a fixed
|
||||
// lowercase snake_case constant, never a formatted message); it is a log
|
||||
// field only, never a counter label, and is empty for cause=by_design,
|
||||
// which is counted but never logged.
|
||||
func ReportKeylessEnqueue(reason string, cause KeylessCause, detail string) {
|
||||
incRunDedupKeyless(reason, cause)
|
||||
if cause != KeylessCauseUnresolved {
|
||||
return
|
||||
}
|
||||
dedupLogger().Info("run enqueued with no dedup key",
|
||||
zap.String("reason", reason),
|
||||
zap.String("cause", string(cause)),
|
||||
zap.String("detail", detail))
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"expvar"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// counterHasLabel reports whether the named expvar.Map has an entry whose
|
||||
// key contains every given substring. Counters are process-global expvar
|
||||
// state (no reset hook), so tests assert presence/growth rather than exact
|
||||
// values that could collide with other tests in the same binary run.
|
||||
func counterHasLabel(t *testing.T, mapName string, substrs ...string) bool {
|
||||
t.Helper()
|
||||
v := expvar.Get(mapName)
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map %q not registered", mapName)
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q is not a *expvar.Map", mapName)
|
||||
}
|
||||
found := false
|
||||
m.Do(func(kv expvar.KeyValue) {
|
||||
matches := true
|
||||
for _, s := range substrs {
|
||||
if !strings.Contains(kv.Key, s) {
|
||||
matches = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matches {
|
||||
found = true
|
||||
}
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
func TestQueueOutcomeNone_IsZeroValue(t *testing.T) {
|
||||
if runsservice.QueueOutcomeNone != "" {
|
||||
t.Fatalf("QueueOutcomeNone = %q, want empty string", runsservice.QueueOutcomeNone)
|
||||
}
|
||||
var zero runsservice.QueueOutcome
|
||||
if zero != runsservice.QueueOutcomeNone {
|
||||
t.Fatalf("zero QueueOutcome %q != QueueOutcomeNone", zero)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportWindowedDedup(t *testing.T) {
|
||||
reason := "agent-supplied-windowed-" + t.Name()
|
||||
outcome := runsservice.ReportWindowedDedup(runsservice.QueueSourceRuns, reason, "some-key")
|
||||
if outcome != runsservice.QueueOutcomeDeduped {
|
||||
t.Fatalf("outcome = %q, want deduped", outcome)
|
||||
}
|
||||
if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=windowed", "queue=runs") {
|
||||
t.Fatal("expected custom reasons to share the bounded windowed/runs entry")
|
||||
}
|
||||
if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) {
|
||||
t.Fatal("agent-supplied reason must not create its own metric series")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportInsertResult_NilError(t *testing.T) {
|
||||
outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, "r", "k", "agent", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if outcome != runsservice.QueueOutcomeQueued {
|
||||
t.Fatalf("outcome = %q, want queued", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportInsertResult_UniqueViolation(t *testing.T) {
|
||||
reason := "agent-supplied-durable-" + t.Name()
|
||||
violation := errors.New("UNIQUE constraint failed: runs.idempotency_key")
|
||||
outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", violation)
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil error on classified conflict, got %v", err)
|
||||
}
|
||||
if outcome != runsservice.QueueOutcomeDeduped {
|
||||
t.Fatalf("outcome = %q, want deduped", outcome)
|
||||
}
|
||||
if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=runs") {
|
||||
t.Fatal("expected custom reasons to share the bounded durable/runs entry")
|
||||
}
|
||||
if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) {
|
||||
t.Fatal("agent-supplied reason must not create its own metric series")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportInsertResult_OtherError_PassesThrough(t *testing.T) {
|
||||
reason := "test_other_" + t.Name()
|
||||
diskErr := errors.New("disk I/O error")
|
||||
outcome, err := runsservice.ReportInsertResult(runsservice.QueueSourceRuns, reason, "k", "agent-1", diskErr)
|
||||
if !errors.Is(err, diskErr) {
|
||||
t.Fatalf("expected the original error to pass through unchanged, got %v", err)
|
||||
}
|
||||
if outcome != runsservice.QueueOutcomeNone {
|
||||
t.Fatalf("outcome = %q, want none", outcome)
|
||||
}
|
||||
if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) {
|
||||
t.Fatal("a non-conflict error must not move the dedup counter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportDurableDedup_Wakeup(t *testing.T) {
|
||||
reason := "agent-supplied-wakeup-" + t.Name()
|
||||
outcome := runsservice.ReportDurableDedup(runsservice.QueueSourceWakeup, reason, "k", "agent-1")
|
||||
if outcome != runsservice.QueueOutcomeDeduped {
|
||||
t.Fatalf("outcome = %q, want deduped", outcome)
|
||||
}
|
||||
if !counterHasLabel(t, "office_run_dedup_total", "reason=custom", "kind=durable", "queue=wakeup") {
|
||||
t.Fatal("expected custom reasons to share the bounded durable/wakeup entry")
|
||||
}
|
||||
if counterHasLabel(t, "office_run_dedup_total", "reason="+reason) {
|
||||
t.Fatal("agent-supplied reason must not create its own metric series")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportKeylessEnqueue_CountsBothCauses(t *testing.T) {
|
||||
reasonUnresolved := "agent-supplied-keyless-unresolved-" + t.Name()
|
||||
reasonByDesign := "agent-supplied-keyless-by-design-" + t.Name()
|
||||
|
||||
runsservice.ReportKeylessEnqueue(reasonUnresolved, runsservice.KeylessCauseUnresolved, "some_detail")
|
||||
runsservice.ReportKeylessEnqueue(reasonByDesign, runsservice.KeylessCauseByDesign, "")
|
||||
|
||||
if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason=custom", "cause=unresolved") {
|
||||
t.Fatal("expected custom reasons to share the bounded unresolved entry")
|
||||
}
|
||||
if !counterHasLabel(t, "office_run_dedup_keyless_total", "reason=custom", "cause=by_design") {
|
||||
t.Fatal("expected custom reasons to share the bounded by_design entry")
|
||||
}
|
||||
if counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonUnresolved) ||
|
||||
counterHasLabel(t, "office_run_dedup_keyless_total", "reason="+reasonByDesign) {
|
||||
t.Fatal("agent-supplied reasons must not create their own keyless metric series")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const metricReasonCustom = "custom"
|
||||
|
||||
// metricReasons is the bounded set of run reasons that receive their own
|
||||
// metric series. Agent-supplied or future reasons use metricReasonCustom so
|
||||
// process-global expvar maps cannot grow once per distinct input.
|
||||
var metricReasons = map[string]struct{}{
|
||||
"task_assigned": {},
|
||||
"task_comment": {},
|
||||
"task_blockers_resolved": {},
|
||||
"task_children_completed": {},
|
||||
"approval_resolved": {},
|
||||
"task_review_requested": {},
|
||||
"task_changes_requested": {},
|
||||
"routine_trigger": {},
|
||||
"heartbeat": {},
|
||||
"budget_alert": {},
|
||||
"agent_error": {},
|
||||
"task_unblocked": {},
|
||||
"task_reopened": {},
|
||||
"task_reopened_via_comment": {},
|
||||
"task_mentioned": {},
|
||||
"stage_pending": {},
|
||||
"stage_changes_requested": {},
|
||||
"task_ready_to_close": {},
|
||||
"routine_dispatch": {},
|
||||
"routine_dispatch_cron": {},
|
||||
"routine_dispatch_event": {},
|
||||
"manual_resume_after_failure": {},
|
||||
"review_started": {},
|
||||
"approval_started": {},
|
||||
"blockers_resolved": {},
|
||||
"children_completed": {},
|
||||
}
|
||||
|
||||
// expvar maps published at package init, exposed via stdlib's /debug/vars
|
||||
// handler. Mirrors internal/office/scheduler/metrics_vars.go's "k=v;k=v"
|
||||
// label model and counters-only rule.
|
||||
var (
|
||||
runDedupTotal = expvar.NewMap("office_run_dedup_total")
|
||||
runDedupKeylessTotal = expvar.NewMap("office_run_dedup_keyless_total")
|
||||
)
|
||||
|
||||
// metricLabel builds a "k1=v1;k2=v2;..." label string for an expvar map key.
|
||||
func metricLabel(pairs ...string) string {
|
||||
if len(pairs)%2 != 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
parts = append(parts, pairs[i]+"="+pairs[i+1])
|
||||
}
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
|
||||
func incRunDedup(q QueueSource, reason, kind string) {
|
||||
runDedupTotal.Add(metricLabel("reason", metricReason(reason), "kind", kind, "queue", string(q)), 1)
|
||||
}
|
||||
|
||||
func incRunDedupKeyless(reason string, cause KeylessCause) {
|
||||
runDedupKeylessTotal.Add(metricLabel("reason", metricReason(reason), "cause", string(cause)), 1)
|
||||
}
|
||||
|
||||
func metricReason(reason string) string {
|
||||
if _, ok := metricReasons[reason]; ok {
|
||||
return reason
|
||||
}
|
||||
return metricReasonCustom
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/workflow/engine"
|
||||
|
||||
runsservice "github.com/kandev/kandev/internal/runs/service"
|
||||
)
|
||||
|
||||
// TestQueueOutcomeNone_MatchesEngineDeclaration pins the "both MUST match"
|
||||
// invariant both QueueOutcomeNone doc comments carry: runs/service and
|
||||
// workflow/engine declare the same string type and zero-value constant
|
||||
// independently, and a change to one without the other would silently break
|
||||
// callers comparing an outcome received from one package against a constant
|
||||
// imported from the other.
|
||||
func TestQueueOutcomeNone_MatchesEngineDeclaration(t *testing.T) {
|
||||
if string(runsservice.QueueOutcomeNone) != string(engine.QueueOutcomeNone) {
|
||||
t.Fatalf("runsservice.QueueOutcomeNone = %q, engine.QueueOutcomeNone = %q; both MUST match",
|
||||
runsservice.QueueOutcomeNone, engine.QueueOutcomeNone)
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -27,18 +26,6 @@ import (
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
// errIdempotencyKeyConflict signals that insertRun's CreateRun failed
|
||||
// because idx_run_idempotency rejected a duplicate idempotency_key —
|
||||
// distinct from the earlier CheckIdempotencyKey miss, which only looks
|
||||
// within IdempotencyWindowHours. Two independent producers deriving the
|
||||
// same operation id for the same event can both pass that fast-path check
|
||||
// before either commits (or the colliding row can simply be older than the
|
||||
// window); the unique index is what actually stops the second insert.
|
||||
// QueueRun treats this as a durable dedupe hit: QueueOutcomeDeduped, not an
|
||||
// error, so the losing producer's caller does not abort or log a spurious
|
||||
// failure for what is really a no-op.
|
||||
var errIdempotencyKeyConflict = errors.New("idempotency key conflict")
|
||||
|
||||
// RunQueueAdapter is the interface the workflow engine uses to enqueue
|
||||
// runs from queue_run actions. Phase 2 final's parallel agent
|
||||
// declares the same shape inside internal/workflow/engine; both
|
||||
@@ -180,34 +167,32 @@ func (s *Service) SubscribeSignal() <-chan struct{} { return s.signalCh }
|
||||
func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutcome, error) {
|
||||
agentInstanceID, err := s.resolveAgentInstance(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return QueueOutcomeNone, err
|
||||
}
|
||||
if agentInstanceID == "" {
|
||||
return "", fmt.Errorf("queue run: agent_profile_id is required")
|
||||
return QueueOutcomeNone, fmt.Errorf("queue run: agent_profile_id is required")
|
||||
}
|
||||
|
||||
if req.IdempotencyKey != "" {
|
||||
dup, err := s.repo.CheckIdempotencyKey(ctx, req.IdempotencyKey, IdempotencyWindowHours)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("idempotency check: %w", err)
|
||||
return QueueOutcomeNone, fmt.Errorf("idempotency check: %w", err)
|
||||
}
|
||||
if dup {
|
||||
s.log.Debug("run skipped (idempotent)",
|
||||
zap.String("key", req.IdempotencyKey))
|
||||
return QueueOutcomeDeduped, nil
|
||||
return ReportWindowedDedup(QueueSourceRuns, req.Reason, req.IdempotencyKey), nil
|
||||
}
|
||||
}
|
||||
|
||||
payloadMap := runPayload(req, agentInstanceID)
|
||||
payload, err := encodePayload(payloadMap)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode payload: %w", err)
|
||||
return QueueOutcomeNone, fmt.Errorf("encode payload: %w", err)
|
||||
}
|
||||
|
||||
if shouldCoalesceRun(req) {
|
||||
coalesced, err := s.repo.CoalesceRun(ctx, agentInstanceID, req.Reason, CoalesceWindowSeconds, payload)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("coalesce check: %w", err)
|
||||
return QueueOutcomeNone, fmt.Errorf("coalesce check: %w", err)
|
||||
}
|
||||
if coalesced {
|
||||
s.log.Debug("run coalesced",
|
||||
@@ -220,20 +205,17 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco
|
||||
}
|
||||
}
|
||||
|
||||
row, err := s.insertRun(ctx, agentInstanceID, req, payload)
|
||||
row, insertErr := s.insertRun(ctx, agentInstanceID, req, payload)
|
||||
// idx_run_idempotency has no time bound, so a conflict here can come
|
||||
// from a row older than IdempotencyWindowHours, not just the windowed
|
||||
// race CheckIdempotencyKey guards against above. ReportInsertResult
|
||||
// classifies that as a no-op dedupe rather than a hard error.
|
||||
outcome, err := ReportInsertResult(QueueSourceRuns, req.Reason, req.IdempotencyKey, agentInstanceID, insertErr)
|
||||
if err != nil {
|
||||
// idx_run_idempotency has no time bound, so a conflict here can
|
||||
// come from a row older than IdempotencyWindowHours, not just the
|
||||
// windowed race CheckIdempotencyKey guards against above. Either
|
||||
// way the existing row is definitionally the same operation this
|
||||
// key identifies, so treat it as a no-op dedupe rather than a hard
|
||||
// error (see errIdempotencyKeyConflict's doc comment).
|
||||
if errors.Is(err, errIdempotencyKeyConflict) {
|
||||
s.log.Debug("run skipped (idempotency index race)",
|
||||
zap.String("key", req.IdempotencyKey))
|
||||
return QueueOutcomeDeduped, nil
|
||||
}
|
||||
return "", err
|
||||
return QueueOutcomeNone, err
|
||||
}
|
||||
if outcome == QueueOutcomeDeduped {
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
s.log.Info("run queued",
|
||||
@@ -247,7 +229,10 @@ func (s *Service) QueueRun(ctx context.Context, req QueueRunRequest) (QueueOutco
|
||||
}
|
||||
|
||||
// insertRun creates the runs row and returns it. Pulled out of
|
||||
// QueueRun to keep the latter under the funlen budget.
|
||||
// QueueRun to keep the latter under the funlen budget. The returned error is
|
||||
// the raw CreateRun error (wrapped for context, never reclassified) — the
|
||||
// caller runs it through ReportInsertResult, which does its own unique-index
|
||||
// classification against the original error chain.
|
||||
func (s *Service) insertRun(
|
||||
ctx context.Context, agentInstanceID string, req QueueRunRequest, payload string,
|
||||
) (*models.Run, error) {
|
||||
@@ -267,9 +252,6 @@ func (s *Service) insertRun(
|
||||
RequestedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := s.repo.CreateRun(ctx, row); err != nil {
|
||||
if runssqlite.IsIdempotencyKeyUniqueViolation(err) {
|
||||
return nil, errIdempotencyKeyConflict
|
||||
}
|
||||
return nil, fmt.Errorf("enqueue run: %w", err)
|
||||
}
|
||||
return row, nil
|
||||
|
||||
@@ -116,6 +116,12 @@ func (r *Repository) runMigrations() error {
|
||||
if err := r.migrateTasksRemoveWorkflowFK(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Must run AFTER migrateTasksRemoveWorkflowFK: that migration recreates
|
||||
// tasks from an explicit column list. Adding this column beforehand would
|
||||
// have it silently dropped by the recreate on any database still carrying
|
||||
// the legacy FK, leaving it absent for the remainder of that boot (the
|
||||
// same hazard class as the task_sessions.name comment above).
|
||||
_ = r.migrate.Apply("tasks.assignment_generation", `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT NULL DEFAULT 0`)
|
||||
if err := r.dropRetiredSlackIntegration(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -390,6 +390,14 @@ func (r *Repository) insertTaskTx(ctx context.Context, tx *sql.Tx, task *models.
|
||||
if err := upsertRunnerInTx(ctx, tx, r.db.Rebind, task.WorkflowStepID, task.ID, task.AssigneeAgentProfileID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A task created already assigned starts at generation 1, the same
|
||||
// value UpdateTaskAssignee would commit for a first assignment - a
|
||||
// creation and a following first assignment must not both mint
|
||||
// generation 1.
|
||||
if _, err := tx.ExecContext(ctx, r.db.Rebind(
|
||||
`UPDATE tasks SET assignment_generation = 1 WHERE id = ?`), task.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return entryID, nil
|
||||
}
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild is the
|
||||
// replay regression test flagged (but not written) in the run-dedup-generation
|
||||
// build: migrateTasksRemoveWorkflowFK (base_migrations.go) recreates `tasks`
|
||||
// from an explicit 19-column list that predates assignment_generation. The
|
||||
// tasks.assignment_generation ADD COLUMN is sequenced strictly AFTER that
|
||||
// rebuild in runMigrations() precisely so a database still carrying the
|
||||
// legacy `FOREIGN KEY (workflow_id)` DDL does not have the new column
|
||||
// silently dropped for the remainder of that boot (the same hazard class as
|
||||
// the task_sessions.name comment in the same file). This had zero coverage
|
||||
// before this test: nothing exercised migrateTasksRemoveWorkflowFK's rebuild
|
||||
// path at all.
|
||||
func TestMigrateTasksRemoveWorkflowFK_AssignmentGenerationSurvivesRebuild(t *testing.T) {
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
// Seed a pre-migration `tasks` table: the legacy FK clause that triggers
|
||||
// the rebuild, the 19 columns migrateTasksRemoveWorkflowFK's SELECT list
|
||||
// expects, and no assignment_generation column at all — matching a real
|
||||
// database that predates this feature.
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT NOT NULL DEFAULT '',
|
||||
workflow_id TEXT NOT NULL DEFAULT '',
|
||||
workflow_step_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
state TEXT DEFAULT 'TODO',
|
||||
priority INTEGER DEFAULT 0,
|
||||
position INTEGER DEFAULT 0,
|
||||
wip_admitted INTEGER NOT NULL DEFAULT 1,
|
||||
queued_for_step_id TEXT NOT NULL DEFAULT '',
|
||||
queued_at TIMESTAMP,
|
||||
metadata TEXT DEFAULT '{}',
|
||||
is_ephemeral INTEGER NOT NULL DEFAULT 0,
|
||||
parent_id TEXT DEFAULT '',
|
||||
autopilot_enabled INTEGER NOT NULL DEFAULT 0,
|
||||
archived_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL,
|
||||
FOREIGN KEY (workflow_id) REFERENCES workflows(id)
|
||||
);
|
||||
INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES ('task-legacy-fk', 'ws-legacy', 'Legacy FK task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
|
||||
`); err != nil {
|
||||
t.Fatalf("seed legacy schema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo (run migrations): %v", err)
|
||||
}
|
||||
|
||||
// The rebuild must have fired: the legacy FK clause is gone.
|
||||
var tableSQL string
|
||||
if err := db.QueryRow(
|
||||
`SELECT sql FROM sqlite_master WHERE type='table' AND name='tasks'`,
|
||||
).Scan(&tableSQL); err != nil {
|
||||
t.Fatalf("read tasks schema: %v", err)
|
||||
}
|
||||
if strings.Contains(tableSQL, "FOREIGN KEY (workflow_id)") {
|
||||
t.Fatalf("tasks table still carries the legacy workflow_id FK after migration: %s", tableSQL)
|
||||
}
|
||||
|
||||
// assignment_generation must be present post-rebuild, not silently
|
||||
// dropped by the rebuild's pre-feature explicit column list.
|
||||
rows, err := db.Queryx(`PRAGMA table_info(tasks)`)
|
||||
if err != nil {
|
||||
t.Fatalf("pragma table_info: %v", err)
|
||||
}
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull, pk int
|
||||
var dflt *string
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
_ = rows.Close()
|
||||
t.Fatalf("scan pragma: %v", err)
|
||||
}
|
||||
if name == "assignment_generation" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
_ = rows.Close()
|
||||
if !found {
|
||||
t.Fatalf("assignment_generation column missing after migrating a legacy FK-bearing tasks table")
|
||||
}
|
||||
|
||||
// The pre-existing row must survive the rebuild, with the new column
|
||||
// defaulted to 0.
|
||||
var gotTitle string
|
||||
var gotGeneration int64
|
||||
if err := db.QueryRow(
|
||||
`SELECT title, assignment_generation FROM tasks WHERE id = ?`, "task-legacy-fk",
|
||||
).Scan(&gotTitle, &gotGeneration); err != nil {
|
||||
t.Fatalf("query migrated row: %v", err)
|
||||
}
|
||||
if gotTitle != "Legacy FK task" {
|
||||
t.Fatalf("title = %q, want %q (row lost during rebuild)", gotTitle, "Legacy FK task")
|
||||
}
|
||||
if gotGeneration != 0 {
|
||||
t.Fatalf("assignment_generation = %d, want 0 for a pre-existing row", gotGeneration)
|
||||
}
|
||||
}
|
||||
@@ -440,7 +440,11 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas
|
||||
task.Repositories = repos
|
||||
}
|
||||
|
||||
s.publishTaskEvent(ctx, events.TaskCreated, task, nil)
|
||||
s.publishTaskEventWithExtra(ctx, events.TaskCreated, task, nil,
|
||||
map[string]interface{}{
|
||||
"assignee_agent_profile_id": task.AssigneeAgentProfileID,
|
||||
"assignment_generation": assignmentGenerationForCreate(task),
|
||||
})
|
||||
s.pullTasksFromNewFeederWork(ctx, task.WorkflowID, task.WorkflowStepID)
|
||||
if refreshed, err := s.tasks.GetTask(ctx, task.ID); err != nil {
|
||||
s.logger.Warn("failed to refresh task after feeder pull", zap.String("task_id", task.ID), zap.Error(err))
|
||||
@@ -453,6 +457,17 @@ func (s *Service) finalizeCreatedTask(ctx context.Context, prepared *preparedTas
|
||||
return CreateTaskResult{Task: task, Outcome: CreateTaskOutcomeCreated}, nil
|
||||
}
|
||||
|
||||
// assignmentGenerationForCreate mirrors insertTaskTx's runner-row guard in
|
||||
// memory rather than re-reading the row it just wrote: a task created
|
||||
// already assigned starts at generation 1 (matching the value insertTaskTx
|
||||
// committed), everything else starts at 0 (never assigned).
|
||||
func assignmentGenerationForCreate(task *models.Task) int64 {
|
||||
if task.AssigneeAgentProfileID != "" && task.WorkflowStepID != "" {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *Service) prepareWorkspacePolicyForCreation(ctx context.Context, req *CreateTaskRequest) error {
|
||||
policy := WorkspacePolicy{}
|
||||
if req.WorkspacePolicy != nil {
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
)
|
||||
|
||||
// setupOfficeTestWithBus mirrors setupOfficeTest but keeps the MockEventBus
|
||||
// so a test can inspect the payload of a published task.created event.
|
||||
func setupOfficeTestWithBus(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repository) {
|
||||
t.Helper()
|
||||
svc, bus, repo := createTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := repo.DB().Exec(`
|
||||
CREATE TABLE IF NOT EXISTS workflow_steps (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
position INTEGER NOT NULL,
|
||||
color TEXT DEFAULT '',
|
||||
prompt TEXT DEFAULT '',
|
||||
events TEXT DEFAULT '{}',
|
||||
allow_manual_move INTEGER DEFAULT 1,
|
||||
is_start_step INTEGER DEFAULT 0,
|
||||
show_in_command_panel INTEGER DEFAULT 1,
|
||||
auto_archive_after_hours INTEGER DEFAULT 0,
|
||||
agent_profile_id TEXT DEFAULT '',
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL,
|
||||
FOREIGN KEY (workflow_id) REFERENCES workflows(id) ON DELETE CASCADE
|
||||
)`)
|
||||
if err != nil {
|
||||
t.Fatalf("create workflow_steps: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
|
||||
t.Fatalf("create workspace: %v", err)
|
||||
}
|
||||
if _, err := repo.EnsureOfficeWorkflow(ctx, "ws-1"); err != nil {
|
||||
t.Fatalf("EnsureOfficeWorkflow: %v", err)
|
||||
}
|
||||
svc.SetStartStepResolver(&dbStepResolver{repo: repo})
|
||||
svc.SetWorkspacePolicyAttacher(noOpWorkspacePolicyAttacher{})
|
||||
return svc, bus, repo
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.9: a task created already assigned publishes
|
||||
// assignment_generation 1 on its task.created event, matching the value
|
||||
// insertTaskTx committed - a creation and the first following assignment
|
||||
// must not both mint generation 1.
|
||||
func TestCreateTask_Assigned_PublishesAssignmentGenerationOne(t *testing.T) {
|
||||
svc, bus, repo := setupOfficeTestWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
result, err := svc.CreateTask(ctx, &CreateTaskRequest{
|
||||
WorkspaceID: "ws-1",
|
||||
Title: "Agent Task",
|
||||
Origin: models.TaskOriginAgentCreated,
|
||||
AssigneeAgentProfileID: "agent-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTask: %v", err)
|
||||
}
|
||||
|
||||
data := singlePublishedEventData(t, bus)
|
||||
gen, ok := data["assignment_generation"].(int64)
|
||||
if !ok || gen != 1 {
|
||||
t.Fatalf("assignment_generation = %#v, want int64(1)", data["assignment_generation"])
|
||||
}
|
||||
if got, ok := data["assignee_agent_profile_id"].(string); !ok || got != "agent-1" {
|
||||
t.Fatalf("assignee_agent_profile_id = %#v, want agent-1", data["assignee_agent_profile_id"])
|
||||
}
|
||||
|
||||
stored, err := repo.GetTask(ctx, result.Task.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTask: %v", err)
|
||||
}
|
||||
var persistedGen int64
|
||||
if err := repo.DB().QueryRowContext(ctx,
|
||||
`SELECT assignment_generation FROM tasks WHERE id = ?`, stored.ID).Scan(&persistedGen); err != nil {
|
||||
t.Fatalf("query assignment_generation: %v", err)
|
||||
}
|
||||
if persistedGen != 1 {
|
||||
t.Fatalf("persisted assignment_generation = %d, want 1", persistedGen)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-RUN-DEDUP-001.9 boundary: a task created UNASSIGNED publishes
|
||||
// assignment_generation 0, never 1 - the boundary that keeps a creation and a
|
||||
// first assignment from both minting generation 1.
|
||||
func TestCreateTask_Unassigned_PublishesAssignmentGenerationZero(t *testing.T) {
|
||||
svc, bus, repo := setupOfficeTestWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
result, err := svc.CreateTask(ctx, &CreateTaskRequest{
|
||||
WorkspaceID: "ws-1",
|
||||
Title: "Unassigned Task",
|
||||
ProjectID: "proj-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTask: %v", err)
|
||||
}
|
||||
|
||||
data := singlePublishedEventData(t, bus)
|
||||
gen, ok := data["assignment_generation"].(int64)
|
||||
if !ok || gen != 0 {
|
||||
t.Fatalf("assignment_generation = %#v, want int64(0)", data["assignment_generation"])
|
||||
}
|
||||
if got, ok := data["assignee_agent_profile_id"].(string); !ok || got != "" {
|
||||
t.Fatalf("assignee_agent_profile_id = %#v, want empty", data["assignee_agent_profile_id"])
|
||||
}
|
||||
|
||||
var persistedGen int64
|
||||
if err := repo.DB().QueryRowContext(ctx,
|
||||
`SELECT assignment_generation FROM tasks WHERE id = ?`, result.Task.ID).Scan(&persistedGen); err != nil {
|
||||
t.Fatalf("query assignment_generation: %v", err)
|
||||
}
|
||||
if persistedGen != 0 {
|
||||
t.Fatalf("persisted assignment_generation = %d, want 0", persistedGen)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,12 @@ const (
|
||||
// queued row for the same agent + reason within the coalescing
|
||||
// window, so nothing new was inserted.
|
||||
QueueOutcomeCoalesced QueueOutcome = "coalesced"
|
||||
// QueueOutcomeNone means no enqueue was attempted, or the attempt
|
||||
// returned an error. It is the zero value, so it is what a widened
|
||||
// signature yields on any path that returns before deciding an
|
||||
// outcome. Declared identically in internal/runs/service — both
|
||||
// declarations MUST match.
|
||||
QueueOutcomeNone QueueOutcome = ""
|
||||
)
|
||||
|
||||
// QueueRunRequest is the typed payload the engine hands to RunQueueAdapter.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
created: 2026-09-13
|
||||
status: completed
|
||||
requirements:
|
||||
- REQ-OFFICE-RUN-DEDUP-001
|
||||
- REQ-OFFICE-RUN-DEDUP-002
|
||||
- REQ-OFFICE-RUN-DEDUP-003
|
||||
- REQ-OFFICE-RUN-DEDUP-004
|
||||
system_design:
|
||||
- ../../specs/office/system-design/run-dedup-generation-01.md
|
||||
- ../../specs/office/system-design/run-dedup-generation-02.md
|
||||
- ../../specs/office/system-design/run-dedup-generation-03.md
|
||||
---
|
||||
|
||||
# Implementation Plan: Office Run Deduplication Generation Identity
|
||||
|
||||
## Overview
|
||||
|
||||
Give Office wake requests an occurrence identity that survives retries and
|
||||
allows later occurrences to run. Persist the task assignment generation, carry
|
||||
it through task-created events, keep producer keys convergent, and report
|
||||
deduplication outcomes with bounded metric labels.
|
||||
|
||||
## Scope
|
||||
|
||||
- Persist and increment `tasks.assignment_generation` for assignment writes.
|
||||
- Include the assignee profile and generation in task-created event data.
|
||||
- Use generation-aware keys for assignment and other durable wake occurrences.
|
||||
- Use a keyless wake when a generation cannot be resolved.
|
||||
- Keep agent-supplied metric reasons bounded to known labels or `custom`.
|
||||
- Preserve the existing queue lookup, unique indexes, coalescing window, and run
|
||||
lifecycle.
|
||||
|
||||
## Implementation Wave
|
||||
|
||||
- [x] [task-01-run-dedup-generation](task-01-run-dedup-generation.md)
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
cd apps/backend && go test ./internal/runs/service ./internal/office/service \
|
||||
./internal/office/dashboard ./internal/task/service
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
```
|
||||
|
||||
The focused backend tests cover assignment event replay, same-agent
|
||||
reassignment, bounded metric labels, and the existing deduplication paths.
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
id: "01-run-dedup-generation"
|
||||
title: "Implement Office run deduplication generation identity"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-OFFICE-RUN-DEDUP-001
|
||||
- REQ-OFFICE-RUN-DEDUP-002
|
||||
- REQ-OFFICE-RUN-DEDUP-003
|
||||
- REQ-OFFICE-RUN-DEDUP-004
|
||||
acceptance_criteria:
|
||||
- AC-OFFICE-RUN-DEDUP-001.1
|
||||
- AC-OFFICE-RUN-DEDUP-001.2
|
||||
- AC-OFFICE-RUN-DEDUP-001.3
|
||||
- AC-OFFICE-RUN-DEDUP-001.4
|
||||
- AC-OFFICE-RUN-DEDUP-001.5
|
||||
- AC-OFFICE-RUN-DEDUP-001.6
|
||||
- AC-OFFICE-RUN-DEDUP-001.7
|
||||
- AC-OFFICE-RUN-DEDUP-001.8
|
||||
- AC-OFFICE-RUN-DEDUP-001.9
|
||||
- AC-OFFICE-RUN-DEDUP-002.1
|
||||
- AC-OFFICE-RUN-DEDUP-002.2
|
||||
- AC-OFFICE-RUN-DEDUP-002.3
|
||||
- AC-OFFICE-RUN-DEDUP-002.4
|
||||
- AC-OFFICE-RUN-DEDUP-003.1
|
||||
- AC-OFFICE-RUN-DEDUP-003.2
|
||||
- AC-OFFICE-RUN-DEDUP-003.3
|
||||
- AC-OFFICE-RUN-DEDUP-003.4
|
||||
- AC-OFFICE-RUN-DEDUP-004.1
|
||||
- AC-OFFICE-RUN-DEDUP-004.2
|
||||
- AC-OFFICE-RUN-DEDUP-004.3
|
||||
- AC-OFFICE-RUN-DEDUP-004.4
|
||||
- AC-OFFICE-RUN-DEDUP-004.5
|
||||
- AC-OFFICE-RUN-DEDUP-004.6
|
||||
system_design:
|
||||
- ../../specs/office/system-design/run-dedup-generation-01.md
|
||||
- ../../specs/office/system-design/run-dedup-generation-02.md
|
||||
- ../../specs/office/system-design/run-dedup-generation-03.md
|
||||
---
|
||||
|
||||
# Task 01: Implement Office run deduplication generation identity
|
||||
|
||||
## Scope
|
||||
|
||||
- Add and preserve the task assignment generation across schema migrations.
|
||||
- Carry the immutable assignee profile ID and generation in task-created events.
|
||||
- Derive the same generation-aware assignment key in each producer.
|
||||
- Keep unresolved generations keyless and observable.
|
||||
- Bound metric label cardinality for agent-supplied reasons.
|
||||
- Avoid changes to queue coalescing and other deferred deduplication work.
|
||||
|
||||
## Acceptance
|
||||
|
||||
An assignment replay uses one durable generation key. A later assignment to the
|
||||
same agent gets a new generation and can enqueue work. A task-created event
|
||||
replayed after its run is claimed remains deduplicated. An unknown metric reason
|
||||
does not create a new permanent expvar label.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
cd apps/backend && go test ./internal/runs/service ./internal/office/service \
|
||||
./internal/office/dashboard ./internal/task/service
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
```
|
||||
|
||||
## Results
|
||||
|
||||
Implemented in the contributor change plus fixup commit
|
||||
`44a5ccc94d8d1fc212b9e839a240d1db2fe16e5f`. The fixup also carries the current
|
||||
`main` branch and resolves the PR merge conflict. Focused backend tests and
|
||||
specification lint pass.
|
||||
@@ -0,0 +1,336 @@
|
||||
---
|
||||
status: draft
|
||||
system: office
|
||||
created: 2026-09-07
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# Office Run Deduplication — Generation Identity
|
||||
|
||||
## Overview
|
||||
|
||||
Every autonomous wake in Office is enqueued through the shared runs queue with an
|
||||
`idempotency_key`. The key exists so that a *redelivery* of one occurrence (a
|
||||
replayed event, a retried handler, two producers reacting to the same fact) does
|
||||
not launch the same agent twice.
|
||||
|
||||
Several producers instead mint a key that is **permanently unique per (reason,
|
||||
task, agent)**. Such a key cannot distinguish a redelivery from a genuine repeat
|
||||
of the same kind of work, so the second legitimate occurrence is suppressed. The
|
||||
suppression is total and leaves no trace: no run row, no inbox item, no wire
|
||||
event, and a `Debug`-level log line.
|
||||
|
||||
The canonical case is assignment. `task_assigned:<task>:<agent>` is stable for
|
||||
the life of that pair, so both of these are permanent, invisible no-ops:
|
||||
|
||||
- Reassigning a task back to an agent that previously held it (A -> B -> A),
|
||||
suppressed by the 24-hour lookup.
|
||||
- Reassigning the same task to the same agent more than 24 hours later,
|
||||
suppressed by the unbounded `UNIQUE(idempotency_key)` index after the lookup
|
||||
has already passed.
|
||||
|
||||
The second case is the more damaging of the two: the fast lookup reports no
|
||||
duplicate, the producer proceeds, and the durable index rejects the insert. The
|
||||
queue reports that rejection as a successful no-op.
|
||||
|
||||
Two producers already avoid the trap by varying their key with something that
|
||||
changes per occurrence: the workflow auto-start key carries the step-transition
|
||||
row id, and the step-entry key carries the entry id and position. Re-running an
|
||||
epic by moving it through a step therefore works, while re-running it by
|
||||
re-assignment does not. This capability generalises what those two do into a
|
||||
contract that every producer must satisfy, and makes the queue's dedup decisions
|
||||
observable.
|
||||
|
||||
This capability owns the *contract on the key*, and the *observability of the
|
||||
outcome*. It does not change the 24-hour lookup, the durable unique index, the
|
||||
coalescing window, or the run lifecycle.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Occurrence:** one real-world fact that justifies waking an agent — this
|
||||
assignment, this comment, this decision, this cron fire. Two occurrences of
|
||||
the same kind on the same (task, agent) are distinct work. Two occurrences are
|
||||
**distinguishable** when some durable row the producer can read differs between
|
||||
them; where nothing does, they collapse to one, and every case that collapses is
|
||||
named under `## Out of scope`.
|
||||
- **Redelivery:** a second attempt to enqueue a wake for an occurrence that has
|
||||
already been enqueued — an event replayed, a handler retried, or a second
|
||||
producer reacting to the same fact.
|
||||
- **Dedup key:** the value persisted in `runs.idempotency_key` or
|
||||
`agent_wakeup_requests.idempotency_key`.
|
||||
- **Generation component:** the part of a dedup key that changes between two
|
||||
occurrences and stays equal across redeliveries of one occurrence.
|
||||
- **Windowed dedup hit:** the queue's recent-duplicate lookup matched an
|
||||
existing row inside its lookback window.
|
||||
- **Durable dedup hit:** the recent-duplicate lookup found nothing, and the
|
||||
unbounded unique index then rejected the insert. After this capability ships
|
||||
this means one of three things: a genuine race between two producers; a producer
|
||||
that minted a colliding key; or a **legitimate late redelivery** — one occurrence
|
||||
re-enqueued more than the lookback window after the original, which a
|
||||
level-triggered reconciler that re-derives the same key on every tick produces
|
||||
as normal operation rather than as a fault. Only the first two are anomalies.
|
||||
|
||||
## Prior art
|
||||
|
||||
### Our own recorded reasoning (wiki)
|
||||
|
||||
**Searched:** resolved a configured personal wiki vault path and collection
|
||||
name from the local `obsidian-wiki` config. The leg then **did not run**:
|
||||
neither `obsidian-wiki` nor `qmd` is on this executor's PATH, no qmd MCP server
|
||||
is exposed to this session, and the vault directory itself is unreadable here
|
||||
(a macOS privacy restriction on this process, not a permission-gate denial).
|
||||
This is a skipped step, not an empty result: the vault is configured and may
|
||||
well hold relevant prior positions that this specification therefore did not
|
||||
consult.
|
||||
|
||||
### What other products shipped (saas-kb)
|
||||
|
||||
**Searched:** nothing. The `saas-kb` MCP server and its `search_fsm_docs` tool
|
||||
are not exposed to this session; the only MCP servers available are `codex` and
|
||||
`kandev`. Skipped step, not an empty result.
|
||||
|
||||
### Prior art inside this repository
|
||||
|
||||
This leg did run, and it is the one that shaped the design.
|
||||
|
||||
- **Two producers already solve this problem**, which is why the defect is
|
||||
visible as an asymmetry rather than a total outage.
|
||||
`officeAutoStartIdempotencyKey` carries the step-transition row id, and the
|
||||
step-entry key carries the entry id and position. Re-running an epic by
|
||||
moving it through a step works; re-running it by re-assignment does not. This
|
||||
capability generalises their shape rather than inventing one.
|
||||
- **`childrenCompletedIdempotencyKey`** already argues, in a written comment,
|
||||
the exact principle this specification makes general: a key that is
|
||||
"permanently unique per (parent, agent)" would wake a parent only for its
|
||||
first delegation wave, so the key digests the child set instead. It also
|
||||
already made the judgment that the digest covers *which* children exist
|
||||
rather than each child's state, to avoid spurious wakes. That reasoning is
|
||||
adopted here, not re-derived.
|
||||
- **`task_sessions.route_generation`** is a monotonic counter already in the
|
||||
tree, bumped on a routing decision and compared for staleness. The assignment
|
||||
generation this capability needs is the same pattern, so it follows that
|
||||
precedent rather than introducing a new one.
|
||||
|
||||
**What we are doing differently:** the existing solutions are per-producer, each
|
||||
one discovered after its own bug report. This capability makes the generation
|
||||
component a property of the key contract itself (AC-OFFICE-RUN-DEDUP-001.1),
|
||||
enforced by a test over every producer, so the next producer added cannot
|
||||
reintroduce a permanent key silently. It also inverts the failure direction:
|
||||
where `officeAutoStartIdempotencyKey` falls back to a time-derived legacy key
|
||||
when its generation is missing, this contract falls back to *no key at all*,
|
||||
because a duplicate wake is recoverable and a suppressed one is not.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-OFFICE-RUN-DEDUP-001: Generation identity in every persisted dedup key
|
||||
|
||||
**Intent:** A dedup key must suppress a redelivery and must not suppress a
|
||||
repeat. A key built only from durable identifiers that never change (reason,
|
||||
task, agent) cannot do both, and today it silently chooses the wrong one.
|
||||
|
||||
**User story:** As an Office operator, I want re-assigning a task to an agent
|
||||
that already held it to actually wake that agent, so that a re-run is not a
|
||||
silent no-op.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-OFFICE-RUN-DEDUP-001.1:** When a producer enqueues a wake with a
|
||||
non-empty dedup key, that key shall contain a generation component that is
|
||||
equal for two enqueue attempts describing the same occurrence, and different
|
||||
for two enqueue attempts describing occurrences that differ in some durable row
|
||||
the producer can read. Occurrences that no durable row distinguishes collapse
|
||||
to one key; each such case shall be named under `## Out of scope`.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.2:** When a task is assigned to an agent, then to a
|
||||
different agent, then back to the first agent, the system shall wake the first
|
||||
agent for the third assignment — by inserting a run row, or, when that
|
||||
assignment lands within the coalescing window of a still-queued run for the
|
||||
same agent and task, by merging into that row. It shall not report a
|
||||
deduplicated outcome, which is the only outcome that produces no run at all.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.3:** When a task is assigned to the same agent a
|
||||
second time and the earlier assignment's run is older than the queue's
|
||||
recent-duplicate lookback window, the system shall insert a run row. That
|
||||
earlier run is necessarily outside the far shorter coalescing window, so the
|
||||
outcome here is queued and never coalesced.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.4:** When the same assignment occurrence is
|
||||
delivered to a producer more than once and the first delivery persisted a run
|
||||
row bearing its key, the system shall queue exactly one run for it. A delivery
|
||||
that coalesced persisted no such row, so a redelivery after the coalescing
|
||||
window queues a second run; see `## Out of scope`.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.5:** When a wake is triggered by an occurrence that a
|
||||
durable row identifies, its generation component shall be derived from that row
|
||||
— its primary key, or a persisted column whose value is fixed for that
|
||||
occurrence — and shall not be read from the clock at the moment the wake is
|
||||
produced. A persisted scheduled time is a stored column, not a clock read, and
|
||||
satisfies this criterion; the processing time at which a producer happens to run
|
||||
does not.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.6:** When a wake is triggered by a clock tick that no
|
||||
durable row identifies, its generation component can be derived from the tick's
|
||||
scheduled time. A tick whose due time is persisted and claimed on a durable row
|
||||
before dispatch is identified by that row, and is governed by
|
||||
AC-OFFICE-RUN-DEDUP-001.5 instead of by this criterion.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.9:** When a durable row identifies an occurrence but
|
||||
the producer would have to re-read that row after the occurrence has advanced,
|
||||
the identifying value shall be captured at the point the occurrence is committed
|
||||
or claimed and carried to the producer. A producer shall not recover a
|
||||
generation component by re-reading a row that a later occurrence has since
|
||||
moved.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.7:** When a dedup key is supplied by an agent
|
||||
through a runtime action, the system shall scope that key to the supplying
|
||||
run so a value the agent reuses across two of its own runs does not suppress
|
||||
the second.
|
||||
- **AC-OFFICE-RUN-DEDUP-001.8:** When a run row persisted before this capability
|
||||
shipped carries a key in the previous format, the system shall leave that row
|
||||
unchanged and shall not treat it as a duplicate of a key in the new format.
|
||||
|
||||
### REQ-OFFICE-RUN-DEDUP-002: One occurrence yields one key across producers
|
||||
|
||||
**Intent:** Several occurrences are observed by more than one producer. Assignment
|
||||
is observed by both the task-event subscriber and the reactivity pipeline, and
|
||||
those two producers converge on one key today. That convergence is what makes a
|
||||
reassignment wake an agent once rather than twice. Adding a generation component
|
||||
independently in each producer would break it.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-OFFICE-RUN-DEDUP-002.1:** When two producers that each derive a dedup key
|
||||
enqueue a wake for the same occurrence with the same reason, task, and agent,
|
||||
they shall derive an identical key. A producer enqueueing that occurrence with
|
||||
no key is governed by REQ-OFFICE-RUN-DEDUP-003 and does not violate this
|
||||
criterion.
|
||||
- **AC-OFFICE-RUN-DEDUP-002.2:** When an assignment is observed by both the
|
||||
task-event subscriber and the reactivity pipeline, the system shall queue
|
||||
exactly one run.
|
||||
- **AC-OFFICE-RUN-DEDUP-002.3:** When two producers attempt to enqueue the same
|
||||
key concurrently, each shall either persist a run or observe a suppression
|
||||
outcome — deduplicated, or coalesced when a coalescible run was queued for that
|
||||
agent, reason and task at its coalesce check — without returning an error to
|
||||
its caller. At most one may persist, and where such a run was queued for both,
|
||||
neither does. No outcome adds a second row bearing the key; none is a failure.
|
||||
- **AC-OFFICE-RUN-DEDUP-002.4:** When two producers attempt to enqueue the same
|
||||
key concurrently, at most one row shall ever bear that key — never two — and
|
||||
neither producer shall abort. Whether it is one row or none depends on whether
|
||||
a coalescible run was queued at each producer's coalesce check, not on which
|
||||
insert committed first; nothing may branch on which outcome a producer saw.
|
||||
|
||||
### REQ-OFFICE-RUN-DEDUP-003: Never suppress silently when the generation is unknown
|
||||
|
||||
**Intent:** A generation component can be unresolvable — a legacy event without
|
||||
the field, an occurrence row not yet committed, a read that failed. Falling back
|
||||
to a permanently-unique key in that case reintroduces exactly the defect this
|
||||
capability removes, and does so in the paths that are hardest to observe. A
|
||||
duplicate wake costs one redundant agent turn; a suppressed wake costs the work.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-OFFICE-RUN-DEDUP-003.1:** When a producer cannot resolve a generation
|
||||
component for an occurrence, it shall not enqueue the wake under a key whose
|
||||
only components are the reason, task, and agent.
|
||||
- **AC-OFFICE-RUN-DEDUP-003.2:** When a producer cannot resolve a generation
|
||||
component, it shall enqueue the wake with no dedup key, accepting a possible
|
||||
duplicate run rather than a possible suppression.
|
||||
- **AC-OFFICE-RUN-DEDUP-003.3:** When a producer enqueues a wake with no dedup
|
||||
key, the system shall record that event in telemetry, attributed to the run
|
||||
reason and discriminated by whether the key was omitted because a generation
|
||||
was unresolvable or because that reason is keyless by design, so that a
|
||||
resolution failure is countable on its own.
|
||||
- **AC-OFFICE-RUN-DEDUP-003.4:** When a wake is enqueued with an empty dedup
|
||||
key, the system shall not apply idempotency suppression to it.
|
||||
|
||||
### REQ-OFFICE-RUN-DEDUP-004: Deduplication is observable
|
||||
|
||||
**Intent:** A dedup is a decision not to do requested work. Today it is reported
|
||||
at `Debug`, is not counted, and is reported to the office producer path as an
|
||||
undifferentiated success. An operator asking "why did nothing happen when I
|
||||
reassigned this" has nothing to read.
|
||||
|
||||
**User story:** As an Office operator, I want to see that a wake was suppressed
|
||||
and why, so that a no-op is diagnosable without attaching a debugger.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-OFFICE-RUN-DEDUP-004.1:** When the queue suppresses a wake, it shall
|
||||
increment a counter exposed through the process metrics endpoint, labelled by
|
||||
run reason and by whether the suppression was a windowed or a durable dedup
|
||||
hit.
|
||||
- **AC-OFFICE-RUN-DEDUP-004.2:** When the queue suppresses a wake through the
|
||||
durable unique index after the recent-duplicate lookup reported no duplicate,
|
||||
it shall emit a structured log record at a level an operator sees by default,
|
||||
carrying the dedup key, the run reason, and the resolved agent.
|
||||
- **AC-OFFICE-RUN-DEDUP-004.3:** When the queue suppresses a wake through the
|
||||
recent-duplicate lookup, it shall emit a structured log record carrying the
|
||||
dedup key and the run reason.
|
||||
- **AC-OFFICE-RUN-DEDUP-004.4:** When a caller enqueues a wake through the
|
||||
**run queue**, the run queue shall report which of queued, deduplicated, or
|
||||
coalesced occurred, and shall report it through every run-queue enqueue
|
||||
interface Office producers use. This criterion does not reach the
|
||||
wakeup-request table, which has neither a windowed lookup nor coalescing and so
|
||||
has no third outcome to report; AC-OFFICE-RUN-DEDUP-004.6 covers it in full.
|
||||
- **AC-OFFICE-RUN-DEDUP-004.5:** When the **run queue** reports a deduplicated or
|
||||
coalesced outcome, it shall not return an error, and the caller shall not treat
|
||||
the outcome as a failure. The wakeup-request table is exempt from the first
|
||||
clause only: it reports its one suppression as a sentinel error, and a caller
|
||||
receiving that sentinel shall still not treat it as a failure.
|
||||
- **AC-OFFICE-RUN-DEDUP-004.6:** When a wakeup request is rejected by the wakeup
|
||||
table's unique index, the system shall increment the same counter family with
|
||||
the wakeup queue identified as the source.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- **Rewriting persisted keys on existing rows.** A run row is a historical
|
||||
record of a wake that happened. Backfilling it to the new key format would
|
||||
restate history and could itself collide with a key a live producer is about
|
||||
to mint. AC-OFFICE-RUN-DEDUP-001.8 makes the old rows inert instead.
|
||||
- **Changing the 24-hour recent-duplicate lookback window.** The window is a
|
||||
performance affordance on the fast path; the unbounded unique index is what
|
||||
actually enforces identity. Neither is the defect, and both are unchanged.
|
||||
- **Changing the 5-second coalescing window or which reasons coalesce.**
|
||||
Coalescing merges wakes that are already agreed to be redundant within seconds;
|
||||
it is a separate mechanism from identity-based dedup and is out of scope here.
|
||||
It is deliberately *not* a suppression: a coalesced wake merges into a run row
|
||||
that is still queued for that agent and task, carrying the newer payload, so the
|
||||
agent is woken. Only a deduplicated outcome produces no run at all. That is why
|
||||
AC-OFFICE-RUN-DEDUP-001.2 accepts a coalesced outcome and forbids only a
|
||||
deduplicated one, and why the coalescing predicate is left exactly as it stands.
|
||||
Two distinct assignment generations landing inside the same five seconds
|
||||
therefore yield one wake, not two: the operator asked twice in five seconds and
|
||||
gets one launch carrying the second request's payload. A wake that coalesces
|
||||
also leaves no row bearing its own key, so a redelivery of that occurrence
|
||||
after the window queues a second run. AC-OFFICE-RUN-DEDUP-003.2 elects a
|
||||
duplicate over a suppression, so that is the accepted direction; closing it
|
||||
would mean keying the merged row, which changes the coalescing mechanism.
|
||||
|
||||
- **Distinguishing a blocker set that resolves, un-resolves and re-resolves
|
||||
identically.** `task_blockers_resolved` identifies its occurrence by digesting
|
||||
the blocker id set, and blocker edges survive resolution — the readiness check
|
||||
reads each blocker task's state, not the edge's existence. So if every blocker
|
||||
resolves, one is reopened, and it then completes again with no blocker added or
|
||||
removed, the second wave digests to the same key as the first and is suppressed
|
||||
once past the lookback window. This is the one case
|
||||
AC-OFFICE-RUN-DEDUP-001.1 requires be named here: no durable row distinguishes
|
||||
the two waves. Both candidates were rejected — the resolving task's id is
|
||||
identical in both waves, because the readiness check fires only on the last
|
||||
blocker to complete; and a blocker task's `updated_at` is exactly the
|
||||
time-derived shape AC-OFFICE-RUN-DEDUP-001.5 disallows, besides re-introducing
|
||||
the spurious wakes `childrenCompletedIdempotencyKey` digests the child *set* to
|
||||
avoid. A durable per-wave row would close this; creating one is separate work.
|
||||
- **A user-facing surface for a suppressed wake.** Deduplication is a
|
||||
high-frequency, expected outcome on the engine redelivery paths; an inbox
|
||||
item or a task-timeline entry per occurrence would be noise that trains
|
||||
operators to ignore the surface. REQ-OFFICE-RUN-DEDUP-004 places the
|
||||
observability floor at operator telemetry and logs. If a user-visible surface
|
||||
is wanted later, the counter added here is the evidence needed to size it.
|
||||
- **Retiring the duplicate `RunReason*` constant blocks** in
|
||||
`internal/office/scheduler` and `internal/office/service`. The audit crosses
|
||||
them, but consolidating them is an unrelated refactor.
|
||||
- **The order in which several wakes from one mutation are enqueued.** A single
|
||||
task mutation can fan out to an assignee and several reviewers. This
|
||||
capability changes each wake's key, not the sequence the reactivity pipeline
|
||||
emits them in, and defines no ordering or tiebreak over that fan-out.
|
||||
- **Changing the `heartbeat` wake source's key.** `fireHeartbeat` still mints a
|
||||
live engine operation id (`heartbeat:<task>:<step>:<unix seconds>`) that the
|
||||
engine expands into a persisted key, so heartbeat *is* a producer and is
|
||||
covered by the audit. It is already generational on its tick time, which
|
||||
AC-OFFICE-RUN-DEDUP-001.6 permits, so no change is required to it. Retiring
|
||||
the source itself, and the duplicate `heartbeat` constants left behind, are
|
||||
separate work.
|
||||
@@ -0,0 +1,568 @@
|
||||
---
|
||||
status: draft
|
||||
system: office
|
||||
requirements:
|
||||
- REQ-OFFICE-RUN-DEDUP-001
|
||||
- REQ-OFFICE-RUN-DEDUP-002
|
||||
- REQ-OFFICE-RUN-DEDUP-003
|
||||
---
|
||||
|
||||
# Office Run Deduplication — Generation Identity System Design Part 1: the key contract
|
||||
|
||||
Part 1 covers what goes *into* a dedup key: the producer audit, the assignment
|
||||
generation and how it is carried, the per-producer generation sources, producer
|
||||
convergence, and the keyless path. Observability, failure and recovery, and the
|
||||
test plan are in
|
||||
[Part 2](run-dedup-generation-02.md). The full producer inventory and the exact Go
|
||||
signatures this capability adds or widens are in
|
||||
[Part 3](run-dedup-generation-03.md).
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
This design changes what goes *into* a dedup key and what the queue *reports*
|
||||
when it acts on one. It changes none of the queue's suppression mechanisms:
|
||||
`CheckIdempotencyKey` (the 24-hour recent-duplicate lookup), `idx_run_idempotency`
|
||||
and `idx_wakeup_idempotency` (the unbounded partial unique indexes), and
|
||||
`CoalesceRun` / `shouldCoalesceRun` (the 5-second merge) are all unchanged. The
|
||||
last is deliberate; see [Concurrency](#concurrency).
|
||||
|
||||
Adjacent contracts read and constrained but not owned:
|
||||
|
||||
- `internal/runs/service` — `QueueRun`, `QueueOutcome`,
|
||||
`errIdempotencyKeyConflict`.
|
||||
- `internal/workflow/engine` — `idempotencyKey` and its `EntryID` /
|
||||
`OperationID` precedence; already generational, unchanged.
|
||||
- `internal/orchestrator/event_handlers_workflow.go` —
|
||||
`officeAutoStartIdempotencyKey`; generational, unchanged apart from its legacy
|
||||
fallback.
|
||||
- `internal/task/repository/sqlite` — the `tasks` table and the
|
||||
`workflow_step_participants` runner seat.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| `REQ-OFFICE-RUN-DEDUP-001` | [Generation sources per producer](#generation-sources-per-producer), [Assignment generation](#assignment-generation) |
|
||||
| `REQ-OFFICE-RUN-DEDUP-002` | [Convergent producers](#convergent-producers) |
|
||||
| `REQ-OFFICE-RUN-DEDUP-003` | [Unresolvable generation](#unresolvable-generation), [Keyless causes per producer](run-dedup-generation-03.md#keyless-causes-per-producer) (Part 3) |
|
||||
| `REQ-OFFICE-RUN-DEDUP-004` | [Observability](run-dedup-generation-02.md#observability) (Part 2) |
|
||||
|
||||
## Producer audit
|
||||
|
||||
The full inventory — every producer that reaches a run queue, including the four
|
||||
that pass no key at all — moved to
|
||||
[Part 3](run-dedup-generation-03.md#producer-audit) when it grew a fourth table.
|
||||
Two entries are load-bearing here and are repeated rather than chased:
|
||||
`queueTaskAssignedRun` and `QueueRunCtx`'s default are the non-generational
|
||||
producers this design replaces, and `QueueRunCtx`'s default reaches every
|
||||
reactivity wake except children-completed.
|
||||
|
||||
## Assignment generation
|
||||
|
||||
Assignment is the only occurrence in the audit with no durable identity to
|
||||
borrow. `UpdateTaskAssignee` updates the runner seat in place, so
|
||||
`workflow_step_participants.id` does not change across a reassignment, and a
|
||||
plain reassignment writes no `task_step_transitions` row.
|
||||
|
||||
Add a monotonic counter to the task:
|
||||
|
||||
```text
|
||||
tasks.assignment_generation INTEGER NOT NULL DEFAULT 0
|
||||
```
|
||||
|
||||
Applied through the existing additive migration mechanism
|
||||
(`internal/task/repository/sqlite/base_migrations.go`), mirroring
|
||||
`task_sessions.route_generation`, the same pattern already in the tree.
|
||||
|
||||
### The two bump sites
|
||||
|
||||
The runner seat has **two** writers, not one, and both bump:
|
||||
|
||||
| Writer | Reached from | Bump |
|
||||
| --- | --- | --- |
|
||||
| `office/repository/sqlite/tasks.go` `UpdateTaskAssignee` | every reassignment and unassignment | increment, inside its existing transaction |
|
||||
| `task/repository/sqlite/task.go` `insertTaskTx` -> `upsertRunnerInTx` | `CreateTask` | insert `assignment_generation = 1` **under exactly the guard that writes the runner row** (`AssigneeAgentProfileID != "" && WorkflowStepID != ""`); any other create inserts `0` |
|
||||
|
||||
Both already run in a transaction. Each bumps on **every committed assignment
|
||||
write**, including one setting the agent that already held the seat. That
|
||||
unconditional rule is load-bearing: a guard of "only when the resolved runner
|
||||
changes" would leave a repeat assignment minting an identical key, the permanent
|
||||
no-op AC-001.3 forbids. Re-assigning a task to the agent that already holds it is
|
||||
a real occurrence — the operator is asking for the work again.
|
||||
|
||||
An unassignment (`assigneeID == ""`, deleting the runner row) is an assignment
|
||||
write and bumps too, so A -> unassigned -> A yields three distinct generations
|
||||
and the final assignment is not suppressed by the first.
|
||||
|
||||
The occurrence's generation is stamped into:
|
||||
|
||||
```text
|
||||
task_assigned:<taskID>:<agentProfileID>:<assignmentGeneration>
|
||||
```
|
||||
|
||||
A different shape from the pre-existing `task_assigned:<task>:<agent>` rows, so
|
||||
AC-001.8 holds without a backfill: an old row can never equal a new key. The
|
||||
counter is task-level, not per-agent — A -> B -> A produces generations 1, 2, 3,
|
||||
so the third assignment's key differs from the first's.
|
||||
|
||||
### Four writers of the same row that must NOT bump
|
||||
|
||||
`task/repository/sqlite/task.go` also calls `syncRunnerInTx` from `updateTaskTx`,
|
||||
`UpdateTaskIfWorkflowStepHasCapacity`,
|
||||
`PromoteQueuedTaskIfWorkflowStepHasCapacity` and
|
||||
`RestoreTaskMessageRollbackIfSessionState`, each passing
|
||||
`task.AssigneeAgentProfileID`. They write the same `role='runner'` row and are
|
||||
provably inert as assignments: the field is **not a stored column** — `tasks` has
|
||||
had none since ADR 0005 Wave F, every read derives it through `runnerProjection`
|
||||
from the seat — so each of these paths writes back the value it just read.
|
||||
|
||||
No production caller can put a *new* agent into the seat through these four.
|
||||
`task/service.UpdateTaskRequest` and `task/dto.UpdateTaskRequest` have no
|
||||
`AssigneeAgentProfileID`, only `AssigneeUserID`, which its own doc comment
|
||||
records as independent of the agent assignee. **Package-qualify that claim
|
||||
wherever it is cited:** `office/dashboard.UpdateTaskRequest` is a different type
|
||||
that *does* carry the field, but it routes to
|
||||
`DashboardService.SetTaskAssigneeAsAgent` -> `UpdateTaskAssignee` — bump site one
|
||||
— and never reaches `syncRunnerInTx`. Every production site supplying a new value
|
||||
reaches either that path or `CreateTaskRequest` -> `insertTaskTx`, the second
|
||||
bump site (`task/service/service_tasks.go`, `service_child_task.go` inheriting
|
||||
the parent's, `backendapp/adapters_office.go`, `mcp/handlers/handlers.go`). Every
|
||||
other non-test mention of the field is a read, a comparison, or a DTO projection.
|
||||
|
||||
Two of the four are step moves, which re-key the seat to a new `step_id`. That is
|
||||
a seat migration, not an assignment; see
|
||||
[What the counter does not cover](#what-the-counter-does-not-cover).
|
||||
|
||||
### Carrying the generation
|
||||
|
||||
**It is captured inside the assigning transaction and carried to the producers.
|
||||
A producer never re-reads it.** The `UpdateTaskAssignee` bump site reads the new
|
||||
value back inside its existing transaction, before `Commit`, and returns it; the
|
||||
caller passes it to each producer explicitly. The create-path bump site returns
|
||||
nothing and changes no signature: its generation is a constant the runner guard
|
||||
already fixes (Part 3). On the dashboard path that caller is
|
||||
`DashboardService.SetTaskAssigneeAsAgent` (`office/dashboard/service_tasks.go`).
|
||||
Note that `office/service` declares a same-named `SetTaskAssigneeAsAgent` and a
|
||||
`SetTaskAssignee` which also reach `UpdateTaskAssignee` but publish nothing and
|
||||
run no reactivity; they are not producers, may discard the value, and neither
|
||||
signature changes. Two channel-setup sites reach it as well and are likewise not
|
||||
producers.
|
||||
|
||||
`UpdateTaskAssignee`'s widened signature, all five of its non-test call sites and
|
||||
what each does with the value, and the behaviour when the read-back itself fails,
|
||||
are specified in
|
||||
[Part 3](run-dedup-generation-03.md#updatetaskassignee-and-the-bump-sites). The
|
||||
wire encoding of the event field is specified in
|
||||
[Part 3](run-dedup-generation-03.md#the-event-payload-field); `0` is a legal
|
||||
generation and is never a sentinel for absent.
|
||||
|
||||
Two carriers, both **new fields this design adds** (neither exists today):
|
||||
|
||||
- `TaskReactivityChange.AssignmentGeneration` (`office/dashboard/service.go`),
|
||||
copied by `convertChangeToMutation` (`office/scheduler/dashboard_adapter.go`)
|
||||
onto a matching `TaskMutation.AssignmentGeneration`
|
||||
(`office/scheduler/reactivity.go`). Both `*int64`; nil means "not supplied". No
|
||||
struct is needed — the mutation already carries the assignee as the bare
|
||||
`NewAssigneeID *string` and the generation follows that shape.
|
||||
- `assignment_generation` on the `task.created` / `task.updated` payload, decoded
|
||||
by `TaskUpdatedData` (`office/service/event_subscribers.go`) and published by
|
||||
`task/service/service_events.go`.
|
||||
|
||||
The field exists because the read is unsound: a producer reading
|
||||
`assignment_generation` after the commit reads *the task's current* generation,
|
||||
not *its occurrence's*. Given A -> B (1), B -> A (2), A -> B (3), a producer
|
||||
still on occurrence 1 reads 3 and mints occurrence 3's key byte for byte; one is
|
||||
then suppressed by the unique index, the direction REQ-OFFICE-RUN-DEDUP-003
|
||||
forbids. A producer handed no generation takes the keyless path of
|
||||
[Unresolvable generation](#unresolvable-generation) and does **not** read the
|
||||
task row to recover one.
|
||||
|
||||
**The one producer that already re-reads stays keyless.** `handleTaskCreated`
|
||||
calls `queueTaskAssignedRun` with `fallbackToStoredRunner=true`, so when the
|
||||
`task.created` payload omits the assignee that producer recovers it through
|
||||
`GetTaskExecutionFields` — a fresh read after the commit. Do **not** extend
|
||||
`TaskExecutionFields` with the generation; that would make the unsound read above
|
||||
a supported path. When the event carries no `assignment_generation` the producer
|
||||
goes keyless with `cause=unresolved` even though it did recover an agent id: it
|
||||
has an agent to wake and no occurrence identity to name.
|
||||
|
||||
### The reactivity gate
|
||||
|
||||
A new key alone is not sufficient. `ApplyTaskMutation`
|
||||
(`office/scheduler/reactivity.go`) reaches `reactToAssigneeChange` only when the
|
||||
agent differs — `change.NewAssigneeID != nil && *change.NewAssigneeID !=
|
||||
task.AssigneeAgentProfileID` — and `DashboardReactivityAdapter` substitutes the
|
||||
*pre-update* assignee into the snapshot. On a repeat assignment to the agent
|
||||
already holding the seat both sides are equal, so **no `task_assigned` wake is
|
||||
produced at all** and AC-001.3 fails before any key is built.
|
||||
|
||||
The gate drops the equality test and fires whenever `change.NewAssigneeID` is
|
||||
non-nil. That is safe by the same call-site argument the bump sites use: of the
|
||||
three `TaskReactivityChange` constructors in `office/dashboard/service_tasks.go`,
|
||||
only `runReactivityForAssigneeChange` sets `NewAssigneeID`; the status and
|
||||
comment constructors leave it nil, so a non-nil value already means "this
|
||||
mutation is an assignment". An unassignment (non-nil but `""`) needs no new
|
||||
guard: the pipeline's `queue` closure already returns early on an empty agent id.
|
||||
|
||||
**The interrupt needs a guard of its own, and does not have one today.**
|
||||
`reactToAssigneeChange` also sets `res.InterruptSessionID`, hard-cancelling the
|
||||
seat's previous session — and it does so on `task.AssigneeAgentProfileID != ""`
|
||||
alone, never comparing `newAssigneeID`. That was safe only because the caller's
|
||||
equality gate never delivered it a same-agent repeat. Removing that gate exposes
|
||||
it, and because the adapter substitutes the pre-update assignee, on an A -> A
|
||||
repeat the previous assignee is non-empty and the interrupt would cancel the
|
||||
agent's own in-flight run. Move the comparison inside: set `InterruptSessionID`
|
||||
only when `task.AssigneeAgentProfileID != ""` **and** `newAssigneeID !=
|
||||
task.AssigneeAgentProfileID`. Only the wake becomes unconditional — this
|
||||
capability changes dedup identity, not session lifecycle.
|
||||
|
||||
### What the counter does not cover
|
||||
|
||||
Three cases a builder would otherwise guess at. Ordering note:
|
||||
`assignment_generation` is compared only for equality, never `>`/`<`. Nothing
|
||||
reads it as a sequence, so no ordering or tiebreak over generations is defined.
|
||||
|
||||
**Two concurrent reassignments of one task.** Each bump site increments inside
|
||||
its own transaction and writes are serialised, so the commits take distinct
|
||||
generations N and N+1 — two keys, two runs, one per assignment. No extra locking.
|
||||
|
||||
**A producer handed no generation.** Because the value is carried, not read, the
|
||||
only "stale value" case left is *absent* value — an event minted before this
|
||||
shipped, or a nil `AssignmentGeneration`. It goes keyless.
|
||||
|
||||
**A step move.** It re-keys the seat row to the new `step_id`, and because
|
||||
`runnerProjection` falls back to `workflow_steps.agent_profile_id` it can both
|
||||
change the resolved runner with no participant write and materialise a per-task
|
||||
row where none existed. Neither is an assignment: the occurrence is already
|
||||
covered by `officeAutoStartIdempotencyKey`'s `stepTransitionID`. Do not bump to
|
||||
compensate — that would re-wake an agent auto-start has already woken.
|
||||
|
||||
## Generation sources per producer
|
||||
|
||||
The replacement for each non-generational producer.
|
||||
|
||||
| Reason | New key | Generation source |
|
||||
| --- | --- | --- |
|
||||
| `task_assigned` | `task_assigned:<task>:<agent>:<assignmentGeneration>` | `tasks.assignment_generation` |
|
||||
| `task_comment` (reactivity, assignee) | `task_comment:<commentID>:<agentID>` | comment row id, plus the recipient |
|
||||
| `task_mentioned` | `task_mentioned:<commentID>:<agentID>` | comment row id, plus the recipient |
|
||||
| `task_blockers_resolved` | `task_blockers_resolved:<blockedTask>:<agent>:<sha256 of blocker ids>` | blocker-set digest |
|
||||
| `blockers_resolved` operation id | `blockers_resolved:<blockedTask>:<sha256 of blocker ids>` | same digest, same sort rule |
|
||||
| `task_unblocked` | none (keyless) | no durable occurrence row |
|
||||
| `task_reopened` | none (keyless) | no durable occurrence row |
|
||||
| `task_reopened_via_comment` | `task_reopened_via_comment:<commentID>:<agent>` | comment row id |
|
||||
| `task_review_requested` | none (keyless) | no durable occurrence row |
|
||||
|
||||
`task_mentioned` must carry the recipient: the mention wake and the assignee
|
||||
comment wake describe the same comment but address different agents, and
|
||||
including the agent keeps the fan-out independent when a comment mentions several
|
||||
agents and one is later re-resolved. `task_comment` carries it for the same
|
||||
reason — see [Convergent producers](#convergent-producers).
|
||||
|
||||
**AC-001.8 for the reshaped comment keys.** Only `task_assigned` gains a segment.
|
||||
`task_comment`, `task_mentioned` and `task_reopened_via_comment` keep three
|
||||
segments and merely swap the task id for a comment id, so the "different shape"
|
||||
argument above does not carry over; all three hold instead because comment ids
|
||||
and task ids come from disjoint uuid-keyed tables, so
|
||||
`<reason>:<taskID>:<agentID>` can never equal `<reason>:<commentID>:<agentID>`
|
||||
for a real pair. `task_comment`'s superseded form is the
|
||||
`{reason}:{taskID}:{agentID}` default this design removes from `QueueRunCtx`.
|
||||
|
||||
`task_blockers_resolved` uses a set digest rather than the resolving blocker's
|
||||
id, matching `childrenCompletedIdempotencyKey`'s already-argued shape: a wave is
|
||||
identified by which blockers were in play, so a second wave that adds blockers
|
||||
gets a distinct key. A wave re-resolving the *same* set digests identically and
|
||||
is suppressed — the one collapse the requirements name under `## Out of scope`.
|
||||
No producer may improvise around it.
|
||||
|
||||
Both blocker producers must sort the ids **lexicographically by
|
||||
`blocker_task_id`** before digesting, and must not inherit `ListTaskBlockers`'s
|
||||
ordering. That query is `ORDER BY created_at`, which is not unique: two blockers
|
||||
added in one timestamp tick have undefined relative order, so two reads of one
|
||||
unchanged set can digest differently and mint two keys for one occurrence.
|
||||
`childrenCompletedIdempotencyKey` is safe only because `ListChildStates` orders
|
||||
by `id`; the blocker path must impose its own.
|
||||
|
||||
Sorting is necessary but not sufficient: the two blocker producers live in
|
||||
different packages and AC-002.1 requires them to derive an identical key, so they
|
||||
call **one shared builder** rather than each implementing the same convention.
|
||||
The delimiter, the digest encoding and the empty-set rule are binding and are
|
||||
specified in
|
||||
[Part 3](run-dedup-generation-03.md#the-shared-key-builders).
|
||||
|
||||
Three status-driven reasons are keyless. `ApplyTaskMutation` is called
|
||||
synchronously after the mutation commits, never from the event bus, so a status
|
||||
transition has no redelivery path and the key guarded nothing. Two guards
|
||||
suffice: the pipeline's per-mutation `seen` map (`{agentID}:{taskID}:{reason}`)
|
||||
stops a double-queue inside one mutation, and the 5-second coalescing window
|
||||
absorbs a rapid repeat across mutations. A status-transition ledger is not
|
||||
justified by the defect, and AC-003.2 elects a duplicate over a suppression.
|
||||
|
||||
`QueueRunCtx`'s `idempotencyKey == ""` default is **removed**, not replaced. Its
|
||||
existence is what silently gave every future caller a permanent key. After this
|
||||
change a `RunContext` with no `IdempotencyKey` is enqueued keyless, the safe
|
||||
direction. Producers that want dedup set the field explicitly, and the
|
||||
`RunContext.IdempotencyKey` doc comment records that empty now means "no dedup"
|
||||
rather than "derive one for me".
|
||||
|
||||
### Agent-supplied keys
|
||||
|
||||
`SpawnAgentRun` prefixes a **non-empty** agent value with the calling run's id:
|
||||
|
||||
```text
|
||||
agent:<callerRunID>:<agent-supplied key>
|
||||
```
|
||||
|
||||
A retry of the same run reuses the run id and still dedupes, which is the case
|
||||
the agent is protecting against; a later run gets a different prefix and is not
|
||||
suppressed. With no caller run id the request is enqueued keyless.
|
||||
|
||||
An **empty** `input.IdempotencyKey` is not prefixed — it stays empty and goes
|
||||
keyless. Prefixing it would collapse every no-dedup-intent `SpawnAgentRun` inside
|
||||
one run onto the single key `agent:<callerRunID>:`, suppressing the second such
|
||||
call: the run-scoped form of the exact defect this capability removes.
|
||||
|
||||
### Routine keys
|
||||
|
||||
Every routine fire commits a `RoutineRun` row (`dispatchRoutineRun` ->
|
||||
`CreateRoutineRun`) before `materialiseLightweightRoutineRun` builds the key, so a
|
||||
durable row is always available. But **that row identifies the dispatch attempt,
|
||||
not always the occurrence**, and which is which depends on the source. The
|
||||
discriminator is `RoutineRun.Source` (`shared.RoutineSourceCron` = `"cron"`,
|
||||
`"manual"`, `"webhook"`), which `buildRoutineIdempotencyKey`'s current
|
||||
`(routineID, triggerID string, startedAt *time.Time)` signature does not receive
|
||||
and must gain.
|
||||
|
||||
| `run.Source` | New key | Generation source |
|
||||
| --- | --- | --- |
|
||||
| `cron` | `routine:<routineID>:<triggerID>:tick:<claimed tick, Unix seconds>` | the scheduled tick claimed off `office_routine_triggers.next_run_at` |
|
||||
| `manual`, `webhook` | `routine:<routineID>:run:<routineRunID>` | `RoutineRun.ID` |
|
||||
|
||||
**Why cron does not use `RoutineRun.ID`.** `processCronTrigger` opens with
|
||||
`ClaimTrigger`, a compare-and-swap (`SET next_run_at = NULL WHERE id = ? AND
|
||||
next_run_at = ?`). Exactly one caller wins a given scheduled tick and the loser
|
||||
returns before any row is written, so there is one `RoutineRun` per *slot*: the
|
||||
row is the attempt, the slot is the occurrence. Keying on `run.ID` would give a
|
||||
redelivery of one slot a fresh key, the direction AC-001.4 forbids. Catch-up does
|
||||
not change this - `computeRoutineMissed` collapses N missed ticks into **one**
|
||||
fire carrying `missedTicks`, so a catch-up is still one occurrence.
|
||||
|
||||
**Why manual and webhook do use `RoutineRun.ID`.** Neither claims a slot. Two
|
||||
manual fires are two distinct intents, an operator asking twice, and
|
||||
`RoutineRun.ID` is their only durable distinguishing identity. This is also the
|
||||
collision the audit table names: both carry `triggerID == ""`, so today they
|
||||
collapse onto one `routine:<id>:<minute>` key.
|
||||
|
||||
The consequence, stated rather than left to be inferred: a **retried** manual or
|
||||
webhook request commits a second `RoutineRun` and therefore wakes twice, because
|
||||
nothing upstream of `CreateRoutineRun` deduplicates the request itself. That is
|
||||
the direction AC-003.2 elects, a duplicate wake over a suppressed one, and
|
||||
narrowing it would need a request-level identity this capability does not
|
||||
introduce. Two concurrent manual fires behave the same way: two rows, two keys,
|
||||
two wakes, no locking. Cron is the only source where the CAS makes redelivery
|
||||
converge.
|
||||
|
||||
**The claimed tick must be CARRIED, not re-read (AC-001.9).** `processCronTrigger`
|
||||
calls `UpdateTriggerNextRun(advanceTo)` *before* dispatching, so by the time the
|
||||
key is built the trigger row's `next_run_at` already names the **next** slot;
|
||||
only the in-memory `*trigger.NextRunAt` still holds the claimed value. Thread it
|
||||
from `processCronTrigger` through `DispatchRoutineRunWithMissed` ->
|
||||
`dispatchRoutineRun` -> `materialiseRoutineRun` ->
|
||||
`materialiseLightweightRoutineRun`. A producer that re-reads the trigger row
|
||||
mints the next occurrence's key, the same unsound-read failure
|
||||
[Carrying the generation](#carrying-the-generation) rules out for assignment.
|
||||
|
||||
`trigger` reaches `dispatchRoutineRun` and stops there, so the claimed tick
|
||||
travels the last two hops as its own `*time.Time` parameter, exactly as
|
||||
`missedTicks` already does. `materialiseRoutineRun` branches on `tmpl.Title`;
|
||||
`materialiseHeavyRoutineRun` ignores the parameter, the treatment `missedTicks`
|
||||
already receives on that branch, because the heavy path creates a real task and
|
||||
builds no wakeup key.
|
||||
|
||||
**Nil and boundary.** `processCronTrigger` returns early when
|
||||
`trigger.NextRunAt == nil`, so a live cron dispatch always has a claimed tick. The
|
||||
exported `DispatchRoutineRunWithMissed` can still be called with
|
||||
`source == "cron"` and a nil trigger or nil claimed tick; that enqueues
|
||||
**keyless** with `cause=unresolved` (see
|
||||
[Unresolvable generation](#unresolvable-generation)) rather than falling back to a
|
||||
minute. Unix seconds suffice because cron granularity is one minute at finest, so
|
||||
two distinct slots of one trigger can never share a value.
|
||||
|
||||
A `RoutineRun.Source` that is neither `cron` nor `manual`/`webhook` matches no
|
||||
branch of the table above. It enqueues **keyless** with `cause=unresolved`, the
|
||||
same direction as a cron dispatch with no claimed tick: an unrecognised source is
|
||||
a source whose occurrence this design cannot name, and inventing a key for it
|
||||
would be guessing at an identity rather than reading one.
|
||||
|
||||
`unix_minute` is removed from both branches; no fire keeps a processing-clock
|
||||
key.
|
||||
|
||||
## Convergent producers
|
||||
|
||||
Two producers observe an assignment on the paths this section governs:
|
||||
`queueTaskAssignedRun`
|
||||
(`office/service/event_subscribers.go`, from `task.created` / `task.updated`) and
|
||||
`reactToAssigneeChange` (`office/scheduler/reactivity.go`, through
|
||||
`ApplyTaskMutation`). They converge on one key today by accident of format —
|
||||
`fmt.Sprintf("task_assigned:%s:%s", …)` and `QueueRunCtx`'s
|
||||
`fmt.Sprintf("%s:%s:%s", reason, taskID, agentID)` produce the same string — and
|
||||
that convergence must survive (AC-002.2).
|
||||
|
||||
**They are driven by different entry paths, and no decision here may assume one
|
||||
call sequence reaches both.** Verified:
|
||||
|
||||
| Entry path | Publishes / calls | Producer that fires |
|
||||
| --- | --- | --- |
|
||||
| Office dashboard `SetTaskAssigneeAsAgent` | `publishTaskUpdated` -> `events.OfficeTaskUpdated`, then `runReactivityForAssigneeChange` | reactivity **only** |
|
||||
| Task creation with an assignee | `events.TaskCreated` | `queueTaskAssignedRun` **only** |
|
||||
| Task service update | `events.TaskUpdated` | `queueTaskAssignedRun`, but see below |
|
||||
|
||||
`handleTaskUpdated` subscribes to `events.TaskUpdated`; `publishTaskUpdated`
|
||||
emits `events.OfficeTaskUpdated`. Different subjects, so the dashboard assignment
|
||||
path does **not** reach `queueTaskAssignedRun`.
|
||||
|
||||
**The `task.updated` row cannot carry an assignment *change*.** As
|
||||
[Four writers](#four-writers-of-the-same-row-that-must-not-bump) establishes, no
|
||||
production update path reaching `syncRunnerInTx` can put a new agent into the
|
||||
seat. So `queueTaskAssignedRun`'s live assignment occurrence is task
|
||||
**creation**; `handleTaskUpdated` stays subscribed as a redelivery and defensive
|
||||
path, and if it fires without a carried generation it goes keyless with
|
||||
`cause=unresolved`. Nothing here may assume a task-service update produces an
|
||||
assignment occurrence.
|
||||
|
||||
**A third producer exists on the onboarding path and does not converge.**
|
||||
`office/onboarding` creates a task with an assignee and enqueues its own
|
||||
`task_assigned` run, while the same creation's `task.created` event drives
|
||||
`queueTaskAssignedRun`. The onboarding producer is never handed the generation,
|
||||
so it goes keyless rather than minting a divergent key. AC-002.1 binds only
|
||||
producers that each derive a key, so a keyless third producer owes no
|
||||
convergence; see
|
||||
[Part 3](run-dedup-generation-03.md#onboarding-is-a-third-task_assigned-producer).
|
||||
|
||||
AC-002.2 therefore requires something narrower than "both always fire": *when one
|
||||
assignment occurrence is observed by both producers — a redelivery, or a future
|
||||
path that drives both — they must derive the same key.* Carrying the generation
|
||||
delivers that: both stamp the value the assigning transaction committed rather
|
||||
than each re-deriving one. Part 2's test case drives both producers directly
|
||||
rather than through a dashboard call, because on that path only one fires.
|
||||
|
||||
To make the convergence structural rather than coincidental, both producers call
|
||||
one exported builder instead of formatting the string themselves. Place it beside
|
||||
`commentkeys` as a sibling package so the office service, the office scheduler
|
||||
and the orchestrator all reach it without an import cycle;
|
||||
`officeAutoStartIdempotencyKey` keeps its own shape, a step-driven auto-start
|
||||
being a different occurrence from a reassignment.
|
||||
|
||||
**The comment wakes do not converge, and do not need to.** `queueCommentRun`
|
||||
passes `commentkeys.TaskComment(commentID)` to `dispatchEngineTrigger` as an
|
||||
*operation id*, which the engine expands to
|
||||
`<opKey>:<step>:<task>:<agent>:<digest>` before persisting, so the shapes can
|
||||
never be equal. They are already mutually exclusive by construction:
|
||||
`runReactivityForComment` (`office/dashboard/service_tasks.go` — **not**
|
||||
`handleCommentCreated`, which never touches the field) sets
|
||||
`SkipAssigneeCommentWake` from `dispatchCommentEngineTrigger`'s return, and the
|
||||
adapter carries it across as `SkipAssigneeWake`, so when the engine handled the
|
||||
comment the reactivity assignee wake is skipped. One comment, one assignee wake,
|
||||
enforced by a branch rather than a key collision. The engine is unchanged.
|
||||
|
||||
Consequently `task_comment` **does** carry the recipient. The occurrence is
|
||||
"(this comment, this recipient)", not "this comment": one comment fans out to an
|
||||
assignee and every mentioned agent, and a bare `task_comment:<commentID>` would
|
||||
collide two recipients' wakes and suppress one. The `task_comment:` prefix is
|
||||
preserved, so `shouldCoalesceRun`'s prefix test still excludes these from
|
||||
coalescing, and `commentkeys` already parses salted keys (`CommentIDFromKey` cuts
|
||||
at the first colon), so the extra segment needs no parser change.
|
||||
|
||||
### Concurrency
|
||||
|
||||
Two producers racing on one key resolve without locking, and this design adds
|
||||
none. Both call `CheckIdempotencyKey` and may see no duplicate; both proceed;
|
||||
`idx_run_idempotency` admits one; the loser's `CreateRun` returns a unique
|
||||
violation, mapped to `errIdempotencyKeyConflict` and reported as
|
||||
`QueueOutcomeDeduped` with a nil error.
|
||||
|
||||
That last step is today true **only** in `runs/service`. `queueRunInline` and
|
||||
`SchedulerService.QueueRun` return the raw violation as an error, so a race on
|
||||
the reactivity path surfaces as a failure rather than a dedupe. Both gain the
|
||||
mapping via the shared helper (see
|
||||
[Counters](run-dedup-generation-02.md#counters)); without it AC-002.3 does not
|
||||
hold on the path this capability fixes.
|
||||
|
||||
**The loser can observe either suppression outcome, and AC-002.3 accepts both.**
|
||||
The queue checks the key, *then* coalesces, *then* inserts. A loser whose lookup
|
||||
ran after the winner's insert sees `Deduped`; one whose lookup ran before that
|
||||
insert and whose coalesce check ran after it sees `Coalesced`, because the
|
||||
winner's row is queued for the same agent, reason and task inside the 5-second
|
||||
window. Both leave exactly one run row and neither returns an error. A test for
|
||||
AC-002.3 must accept either and must not pin `Deduped`. Note also that
|
||||
`SchedulerService.QueueRun` coalesces unconditionally where `runs/service` first
|
||||
consults `shouldCoalesceRun`; neither is changed, and this outcome contract is
|
||||
written to hold for both.
|
||||
|
||||
Neither caller aborts, and the observable result is the same whichever insert
|
||||
committed first (AC-002.4): at most one row ever bears that key, and none does
|
||||
when a coalescible run was queued for that agent, reason and task at each
|
||||
producer's coalesce check, because a coalesced enqueue never reaches
|
||||
`insertRun`. See
|
||||
[Key durability](run-dedup-generation-02.md#key-durability), which owns that
|
||||
boundary. No ordering or tiebreak is defined: the losing path has no output to
|
||||
order. The run `id` is a fresh UUID per attempt, so nothing may depend on which
|
||||
uuid won.
|
||||
|
||||
## Unresolvable generation
|
||||
|
||||
A producer that cannot resolve its generation component enqueues with an empty
|
||||
key. It does **not** fall back to `<reason>:<task>:<agent>` (AC-003.1), because
|
||||
that is the defect.
|
||||
|
||||
Concretely: an event carrying no `assignment_generation`, or a nil
|
||||
`AssignmentGeneration` on a mutation; `officeAutoStartIdempotencyKey`'s zero
|
||||
`stepTransitionID`; `SpawnAgentRun` with no caller run id; a reactivity comment
|
||||
wake whose `MutationComment` is nil; and a `source == "cron"` routine dispatch
|
||||
reaching [Routine keys](#routine-keys) with no claimed tick carried.
|
||||
|
||||
`SpawnAgentRun` with an **empty agent-supplied key** is deliberately absent from
|
||||
that list: nothing failed to resolve, the agent expressed no dedup intent, so it
|
||||
is `by_design`. Part 3's
|
||||
[keyless causes](run-dedup-generation-03.md#keyless-causes-per-producer) assigns
|
||||
every producer its side and agrees.
|
||||
|
||||
An empty key already means "no dedup" throughout the queue: `QueueRun` skips
|
||||
`CheckIdempotencyKey`, `insertRun` writes a NULL `idempotency_key`, and both
|
||||
unique indexes are partial and ignore NULL (AC-003.4). Coalescing still applies —
|
||||
`shouldCoalesceRun` only excludes the `task_comment:` prefix.
|
||||
|
||||
**Where the keyless counter is incremented, and how `cause` reaches it.** The
|
||||
cause is a property of the producer's *decision*, not of the request: by the time
|
||||
an enqueue reaches a queue, "I never needed a key" and "I tried to build one and
|
||||
failed" are both an empty string, and `RunContext` carries nothing separating
|
||||
them. So the queue does **not** report keyless enqueues and nothing infers a cause
|
||||
from an empty key. Instead `internal/runs/service` exports a reporter beside the
|
||||
suppression helper of [Counters](run-dedup-generation-02.md#counters):
|
||||
|
||||
```text
|
||||
runsservice.ReportKeylessEnqueue(reason string, cause KeylessCause, detail string)
|
||||
```
|
||||
|
||||
with `KeylessCauseUnresolved` and `KeylessCauseByDesign`. Each producer calls it
|
||||
where it decides to go keyless, naming its own cause, immediately before
|
||||
enqueuing. `detail` names **what failed to resolve** — the short constant
|
||||
[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns each
|
||||
producer — so the `Info` record [Part 2](run-dedup-generation-02.md#logs)
|
||||
requires for `cause=unresolved` separates two producers that go keyless under one
|
||||
reason. It is `""` for `cause=by_design`, which is not logged. Counters stay declared once in `internal/runs/service` — no parallel
|
||||
map — and the import direction already holds: `runs/service` imports
|
||||
`office/models` only, so `office/service`, `office/scheduler` and
|
||||
`office/routines` can all call it without a cycle.
|
||||
|
||||
`cause=unresolved` is every site enumerated in *this* section. `cause=by_design`
|
||||
is the three status reasons and any other producer that never had an occurrence to
|
||||
name. **The earlier rule — that any `RunContext` setting no key is `by_design` —
|
||||
is withdrawn**: with `QueueRunCtx`'s default removed a producer that failed to
|
||||
resolve also arrives with no key, so that rule counted precisely the failures
|
||||
AC-003.3 exists to isolate as normal traffic. A producer reaching a queue keyless
|
||||
without having called the reporter is a bug; the key-format table test in
|
||||
[Part 2](run-dedup-generation-02.md#testing) is where it is caught.
|
||||
|
||||
The counters themselves are defined in
|
||||
[Part 2](run-dedup-generation-02.md#counters).
|
||||
@@ -0,0 +1,398 @@
|
||||
---
|
||||
status: draft
|
||||
system: office
|
||||
requirements:
|
||||
- REQ-OFFICE-RUN-DEDUP-001
|
||||
- REQ-OFFICE-RUN-DEDUP-002
|
||||
- REQ-OFFICE-RUN-DEDUP-003
|
||||
- REQ-OFFICE-RUN-DEDUP-004
|
||||
---
|
||||
|
||||
# Office Run Deduplication — Generation Identity System Design Part 2: observability and verification
|
||||
|
||||
Part 2 covers what the queue *reports* when it acts on a dedup key
|
||||
(`REQ-OFFICE-RUN-DEDUP-004`), the upgrade and failure behaviour, and the test
|
||||
plan for all four requirements. The key contract itself — the producer audit,
|
||||
the assignment generation, the per-producer sources, convergence and the keyless
|
||||
path — is in [Part 1](run-dedup-generation-01.md). The full producer inventory
|
||||
and the exact Go signatures named below are in
|
||||
[Part 3](run-dedup-generation-03.md).
|
||||
|
||||
## Observability
|
||||
|
||||
### Counters
|
||||
|
||||
New expvar maps, following `internal/office/scheduler/metrics_vars.go`'s
|
||||
`k=v;k=v` label model and the counters-only rule stated there.
|
||||
|
||||
**Three queue implementations make a dedup decision, not one, and all three are
|
||||
in scope:**
|
||||
|
||||
| Queue | Reached from | Today |
|
||||
| --- | --- | --- |
|
||||
| `runs/service.Service.QueueRun` | `office/service.Service.QueueRun` when a runs service is wired | checks the key; maps the unique violation to `Deduped`; logs `Debug` |
|
||||
| `office/service.Service.queueRunInline` | same, when no runs service is wired (older tests, transitional deployments) | checks the key; returns the raw `CreateRun` error; logs `Debug` |
|
||||
| `office/scheduler.SchedulerService.QueueRun` | `QueueRunCtx`, and so every reactivity wake | checks the key; returns the raw `CreateRun` error; logs `Debug` |
|
||||
|
||||
The third most needs instrumenting: `ApplyTaskMutation` -> `QueueRunCtx` ->
|
||||
`SchedulerService.QueueRun` is the path a reassignment travels and is the *least*
|
||||
observable today. Instrumenting only `internal/runs/service` would leave
|
||||
AC-004.1 / .2 / .3 unmet on the flow the card reports.
|
||||
|
||||
To keep one behaviour rather than three, the *reporting* is one exported helper
|
||||
in `internal/runs/service`: it classifies a suppression as `windowed` or
|
||||
`durable`, increments the counter, emits the log, and returns the `QueueOutcome`.
|
||||
All three queues call it. Counters are declared once there; nothing declares a
|
||||
parallel map. Its name, its two entry points, and what it does with an error that
|
||||
is *not* a unique violation are specified in
|
||||
[Part 3](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome).
|
||||
|
||||
The keyless counter is **not** driven from the queues. It has its own exported
|
||||
reporter in the same package, `ReportKeylessEnqueue(reason, cause, detail)`, called by the
|
||||
producer at its own decision site, because only the producer knows whether the key
|
||||
was never needed or could not be resolved — see
|
||||
[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation).
|
||||
|
||||
| Name | Labels | Incremented when |
|
||||
| --- | --- | --- |
|
||||
| `office_run_dedup_total` | `reason`, `kind` (`windowed` or `durable`), `queue` (`runs` or `wakeup`) | the queue suppresses a wake |
|
||||
| `office_run_dedup_keyless_total` | `reason`, `cause` (`unresolved` or `by_design`) | a producer calls `ReportKeylessEnqueue` before enqueuing with no dedup key |
|
||||
|
||||
`kind` earns the metric: `windowed` is the expected, high-frequency outcome on
|
||||
the engine redelivery paths, while `durable` means the 24-hour lookup found
|
||||
nothing and the unbounded index still rejected the insert. After this ships that
|
||||
is one of three things — a genuine race, a producer minting a colliding key, or a
|
||||
**legitimate late redelivery** of one occurrence past the lookback window. The
|
||||
third is not a fault and is not rare: `ParentWakeReconciler` is a level-triggered
|
||||
backstop that re-derives the identical `wakeOperationID` from current task state
|
||||
on every tick, with no age bound, so a parent stuck for more than a day produces a
|
||||
durable hit as normal operation. `Warn` is still the right level — a durable hit
|
||||
always means the fast path missed something an operator may want to see — but
|
||||
neither the log text nor any alert built on this counter may call a durable hit an
|
||||
anomaly on its own. One undifferentiated counter would bury the first two.
|
||||
`cause` earns its place the same way: `by_design` is high-frequency and expected,
|
||||
`unresolved` is the generation that should have been carried and was not
|
||||
(AC-003.3), and by `reason` alone the two are indistinguishable. Metric reasons
|
||||
use the finite set of known run reasons. Any agent-supplied or future reason is
|
||||
reported as `custom`, so process-global expvar maps cannot grow once per input.
|
||||
Cardinality is therefore the known reason set plus one custom bucket times a
|
||||
small constant, the same order as `routing_*`.
|
||||
|
||||
`office_run_dedup_total{queue="wakeup"}` increments where `CreateWakeupRequest`
|
||||
returns `ErrWakeupIdempotencyConflict` (AC-004.6); that path has no windowed
|
||||
lookup, so its `kind` is always `durable`. Its `reason` label is
|
||||
`WakeupRequest.Reason`, the field already on the row being inserted, not the
|
||||
coarser `Source`, which would collapse every wakeup onto one series.
|
||||
|
||||
### Logs
|
||||
|
||||
- Windowed hit: `Info`, carrying key and reason. Currently `Debug`.
|
||||
- Durable hit: `Warn`, carrying key, reason, and resolved agent. Currently
|
||||
`Debug`, on both the `runs` and wakeup paths.
|
||||
- Keyless enqueue with `cause=unresolved`: `Info`, carrying `reason` and the
|
||||
`detail` constant naming what failed to resolve — `ReportKeylessEnqueue`'s
|
||||
third argument, whose per-producer values
|
||||
[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) fixes. Without
|
||||
it the four producers that go keyless under `reason=task_assigned` are
|
||||
indistinguishable in the log, which is what AC-003.3 exists to prevent.
|
||||
`detail` is a log field only, never a counter label. A `cause=by_design`
|
||||
keyless enqueue is counted but **not** logged — it is the normal path for
|
||||
several reasons and would be pure noise.
|
||||
|
||||
### Outcome propagation
|
||||
|
||||
`runs/service.QueueRun` already returns `QueueOutcome`. Office's own enqueue
|
||||
interfaces discard it: `office/shared.RunQueuer.QueueRun`,
|
||||
`office/service.Service.QueueRun`, `office/scheduler.SchedulerService.QueueRun`
|
||||
and `QueueRunCtx`, plus the duplicate declarations in `office/runtime/actions.go`
|
||||
and `office/approvals/service.go`, all return only `error`. Widen them to
|
||||
`(runsservice.QueueOutcome, error)` (AC-004.4).
|
||||
|
||||
The three queues under [Counters](#counters) produce that outcome, and
|
||||
`office/service.Service.QueueRun` forwards whichever of its two it used.
|
||||
`queueRunInline` and `SchedulerService.QueueRun` return no `QueueOutcome` today
|
||||
and map no unique violation; both gain the mapping from the shared helper, so a
|
||||
durable conflict on the reactivity path becomes `Deduped` with a nil error
|
||||
instead of today's `"reactivity run failed"` `Error` log (AC-002.3).
|
||||
|
||||
**The wakeup enqueue is outside this widening**, and AC-004.4 now says so in its
|
||||
own text rather than leaving it to be inferred. `CreateWakeupRequest`
|
||||
(`office/repository/sqlite/wakeup_requests.go`, interface in
|
||||
`office/routines/service.go`) keeps its `error` signature: no coalescing and no
|
||||
windowed lookup, so there is no third outcome to report, and its one suppression
|
||||
is the sentinel `ErrWakeupIdempotencyConflict`, which AC-004.5 exempts by name.
|
||||
The exemption covers the signature only — a caller receiving that sentinel must
|
||||
still not treat it as a failure, which the one live caller already does by
|
||||
logging and continuing.
|
||||
|
||||
**Sizing.** This widening is the highest-diff item here — `.QueueRun(` appears
|
||||
~112 times across 33 files in `internal/office`, ~9 of them production call
|
||||
sites. Size it as its own task rather than folding it into "observability".
|
||||
|
||||
**A path that enqueues nothing reports `QueueOutcomeNone`.** `QueueOutcome`'s
|
||||
zero value is the empty string, which is none of `queued`, `deduped` or
|
||||
`coalesced`. Part 3 gives it a name and a meaning
|
||||
([the no-op outcome](run-dedup-generation-03.md#the-suppression-reporter-and-the-no-op-outcome)).
|
||||
This is additive and changes no acceptance criterion: AC-004.4 obliges the queue
|
||||
to report one of three outcomes *when a caller enqueues a wake*, and a widened
|
||||
signature returning a non-nil error never meets that condition. The reactivity
|
||||
pipeline's empty-agent-id guard is **not** an instance of it: that guard lives in
|
||||
the `queue` closure and returns *before* calling `QueueRunCtx`, so it yields no
|
||||
outcome at all rather than `QueueOutcomeNone`. `QueueRunCtx` itself has no such
|
||||
early return.
|
||||
|
||||
Callers that have nothing to decide assign the outcome to `_`. No caller may
|
||||
treat `Deduped` or `Coalesced` as an error (AC-004.5); the reactivity pipeline's
|
||||
`queue` closure in particular must keep appending to `res.Runs` only for
|
||||
`QueueOutcomeQueued`, so `ApplyTaskMutationResult.Runs` stops reporting
|
||||
suppressed wakes as queued ones.
|
||||
|
||||
## Key durability
|
||||
|
||||
A dedup key is durable only for an enqueue that reached `insertRun`. All three
|
||||
queues run `CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, and
|
||||
`CoalesceRun` (`internal/runs/repository/sqlite/runs.go`) selects its neighbour
|
||||
on `agent_profile_id`, `reason`, `status='queued'`, `requested_at` inside the
|
||||
window, — for `task_assigned` only — the payload's `task_id`, **and
|
||||
`(idempotency_key IS NULL OR idempotency_key NOT LIKE 'task_comment:%')`**.
|
||||
|
||||
That last predicate is the one to be exact about, because two different keys are
|
||||
in play and only one of them is read:
|
||||
|
||||
- **The incoming enqueue's key is neither read nor written here.** The `UPDATE`
|
||||
sets `coalesced_count` and `payload` only, so the merged row keeps its own key
|
||||
and the incoming key is persisted nowhere. That is what makes a coalesced
|
||||
enqueue **persist no row bearing its own key**; neither unique index ever sees
|
||||
it. Whether the incoming enqueue may coalesce at all is decided earlier and
|
||||
elsewhere, by `shouldCoalesceRun` (`internal/runs/service`), which excludes
|
||||
only a `task_comment:`-prefixed key.
|
||||
- **The predicate reads the candidate *neighbour's* key**, keeping a
|
||||
`task_comment:`-keyed row from absorbing another reason's wake. A neighbour is
|
||||
therefore matchable when its key is NULL **or** any non-`task_comment:` key,
|
||||
including a `task_assigned:<task>:<agent>:<generation>` row. A test that needs
|
||||
a matchable neighbour must not construct a `task_comment:`-keyed one — it would
|
||||
never match, and the test would pass while asserting nothing.
|
||||
|
||||
Two consequences the contract states rather than leaving to be discovered:
|
||||
|
||||
- Two producers racing on one key may **both** coalesce into a queued neighbour,
|
||||
leaving **zero** rows bearing that key. If the neighbour is claimed between
|
||||
their two coalesce checks, the later one inserts and exactly one row bears it.
|
||||
AC-002.4 is therefore written as *at most one, never two* rather than *exactly
|
||||
one*: that is the only invariant true in every interleaving, and a test must
|
||||
not pin the count to one.
|
||||
- An occurrence that coalesced and is then redelivered *after* the coalescing
|
||||
window has no durable key to match, and queues a second run (AC-001.4). That
|
||||
duplicate is accepted, not closed: AC-003.2 elects a duplicate over a
|
||||
suppression, and keying the merged row would change `CoalesceRun`, which
|
||||
[Part 1](run-dedup-generation-01.md#purpose-and-boundaries) holds unchanged.
|
||||
|
||||
This is a property of the pre-existing coalescing mechanism, not of the
|
||||
generation component — but the generation component makes it *reachable* for
|
||||
`task_assigned`, which is why it is stated here. Before this capability two
|
||||
assignments inside one window shared a permanent key, so the second was stopped
|
||||
by `CheckIdempotencyKey` and never reached `CoalesceRun`. With distinct
|
||||
generations the windowed lookup misses and the coalesce arm is live.
|
||||
|
||||
## Failure and recovery
|
||||
|
||||
- A producer handed no `assignment_generation` goes keyless, never permanent:
|
||||
the wake happens, dedup for that one enqueue does not. The only degraded case.
|
||||
- A counter increment is best-effort and never fails an enqueue.
|
||||
- The migration is additive with a `0` default and runs at startup, so both bump
|
||||
sites always have the column. Every task starts at `0` and the first assignment
|
||||
after upgrade commits `1`, so no task's first new-format key is `...:0`.
|
||||
- **A second migration rebuilds `tasks` and must carry the column through it.**
|
||||
`taskPriorityMigrationStatements`
|
||||
(`office/repository/sqlite/base_migrations.go`) recreates `tasks` from an
|
||||
**explicit column list** to change `priority` from INTEGER to TEXT. That list
|
||||
would silently drop `assignment_generation`, so the column joins it the same
|
||||
way eight other columns already do in that file — **its established idiom for
|
||||
this exact hazard, followed rather than replaced**:
|
||||
1. a defensive `ALTER TABLE tasks ADD COLUMN assignment_generation INTEGER NOT
|
||||
NULL DEFAULT 0` before the recreate, with the error swallowed like its
|
||||
neighbours (`archived_by_cascade_id`, `wip_admitted`, `external_id` and the
|
||||
rest), so the `SELECT` can reference the column on a legacy fixture that
|
||||
predates it;
|
||||
2. the column declared in the `tasks_priority_new` definition;
|
||||
3. `COALESCE(assignment_generation,0)` in the recreate `SELECT`, so values are
|
||||
**preserved**, not reset.
|
||||
Copying rather than defaulting is what makes this order-independent: it holds
|
||||
whether or not the task repository's own migration ran first, so nothing rests
|
||||
on an argument about which repository initialises when.
|
||||
Left undone the defect is not a hard failure, which is why it is easy to miss.
|
||||
The task repository initialises first (`backendapp/storage.go`), so the column
|
||||
is added, then dropped by the rebuild, absent for the remainder of that boot,
|
||||
then re-added at `0` on the next one — `db.MigrateLogger.Apply`
|
||||
(`internal/db/migratelog.go`) is not ledger-backed, so the ALTER re-runs. A
|
||||
task previously at generation 3 re-mints `...:1` and collides with its own
|
||||
historical key on the unbounded index: this capability's own defect,
|
||||
resurrected.
|
||||
- Events in flight across the upgrade carry no `assignment_generation`. Their
|
||||
producer goes keyless with `cause=unresolved` rather than minting a
|
||||
legacy-shaped key: one event's lifetime, in the direction AC-003.2 elects.
|
||||
- No runtime feature flag. A key-format change plus telemetry has no
|
||||
partially-migrated state to guard — a mixed fleet writing old and new formats
|
||||
does not collide, the property AC-001.8 relies on.
|
||||
|
||||
## Testing
|
||||
|
||||
Backend `*_test.go` beside each source. Behaviours with no equivalent test today:
|
||||
|
||||
- A -> B -> A reassignment wakes the first agent again on the third assignment
|
||||
(AC-001.2): three distinct keys, and the third enqueue reports queued or
|
||||
coalesced, never deduped. Assert the **outcome**, not a row count — a test
|
||||
driving all three inside the 5-second window legitimately gets two rows.
|
||||
- A repeat assignment whose prior run's `requested_at` predates the window queues
|
||||
a run (AC-001.3), driven by writing an aged row. That row is also outside the
|
||||
coalescing window, so this case asserts `Queued` exactly.
|
||||
- Both producers, driven directly with the same task, agent and carried
|
||||
generation, derive one key and queue one run (AC-002.1 / .2). At the producers,
|
||||
not through a dashboard call — only one fires there.
|
||||
- A repeat assignment to the **same** agent passes the relaxed reactivity gate,
|
||||
bumps, and wakes the agent (AC-001.3), including when the prior run is aged
|
||||
past the window; and it does **not** set `InterruptSessionID`. A reassignment
|
||||
to a *different* agent still does — both branches of the guard added in
|
||||
[The reactivity gate](run-dedup-generation-01.md#the-reactivity-gate).
|
||||
- A non-assignment task update (title, priority) reaches `syncRunnerInTx` but
|
||||
does not bump and queues no `task_assigned` run — the inert-writer claim in
|
||||
[Four writers](run-dedup-generation-01.md#four-writers-of-the-same-row-that-must-not-bump).
|
||||
- A task created with an assignee whose `task.created` event carries
|
||||
`assignment_generation` commits generation `1` and queues one run; the same
|
||||
creation with the field absent goes keyless with `cause=unresolved` and still
|
||||
queues, exercising `fallbackToStoredRunner`.
|
||||
- **A task created UNASSIGNED publishes `assignment_generation` `0`**, carried on
|
||||
`publishTaskEventWithExtra`'s `extra` map, and queues no `task_assigned` run.
|
||||
Assigning it afterwards commits `1` and queues one. This pins the boundary that
|
||||
keeps a creation and a first assignment from both minting `...:1`, and it is
|
||||
the only assertion that catches a create path that hardcodes `1`.
|
||||
- A producer handed a generation does not read the task row for one: drive
|
||||
occurrence 1's producer after occurrence 3 has committed and assert it still
|
||||
mints occurrence 1's key.
|
||||
- A durable conflict on the **reactivity** path (`SchedulerService.QueueRun`, and
|
||||
the same for `queueRunInline`) returns `Deduped` with a nil error and does not
|
||||
log `"reactivity run failed"` (AC-002.3).
|
||||
- A key-format table test over **every row of the
|
||||
[producer audit](run-dedup-generation-03.md#producer-audit)**, with the
|
||||
assertion chosen by what the row produces. **Two assertions, not one:**
|
||||
- A row that mints a key asserts its output has a segment that varies across
|
||||
two constructed occurrences (AC-001.1), so a new producer reintroducing a
|
||||
permanent key fails a test rather than shipping.
|
||||
- A row that is **keyless by contract** has no key and therefore no segment,
|
||||
so the varying-segment assertion is unsatisfiable and is not applied to it.
|
||||
It asserts instead that the enqueue carries an **empty** key and that the
|
||||
producer reported the `cause` and `detail`
|
||||
[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer) assigns it.
|
||||
These are the three permanently-keyless rows of
|
||||
[Part 3](run-dedup-generation-03.md#keyless-today--must-report-a-cause) —
|
||||
the fourth, `agent_error`, becomes generational here and takes the first
|
||||
assertion — plus the three status-driven reactivity reasons.
|
||||
The same-set blocker re-resolution named in the requirements' `## Out of scope`
|
||||
is the one row that mints a key and is asserted *stable* rather than varying,
|
||||
pinning the collapse rather than letting it drift.
|
||||
- Durable-index conflict increments `office_run_dedup_total` with `kind=durable`
|
||||
and returns `Deduped` with a nil error.
|
||||
- An unresolvable generation enqueues keyless with
|
||||
`office_run_dedup_keyless_total{cause=unresolved}`; a status-driven reason
|
||||
enqueues keyless with `cause=by_design` and emits no log record.
|
||||
- `SpawnAgentRun` with an empty agent key and a live caller run id enqueues
|
||||
keyless, and two such calls in one run are **not deduplicated**. Assert the
|
||||
outcome, not a row count: an empty key passes `shouldCoalesceRun`, and the
|
||||
first call's own row — which persisted a NULL key — satisfies the
|
||||
`idempotency_key IS NULL` arm of the neighbour predicate in
|
||||
[Key durability](#key-durability), whose task-scoping clause applies only to
|
||||
`task_assigned`, so two calls to one agent with one reason inside the 5-second
|
||||
window legitimately coalesce. Drive them with distinct reasons, or
|
||||
outside the window, to assert `Queued` twice.
|
||||
- Two recipients' wakes for one comment (assignee plus a mentioned agent) mint
|
||||
distinct keys and both queue.
|
||||
- **AC-001.7's run-id prefix, on the non-empty agent key** — the case the
|
||||
empty-key bullet above does not reach. Two `SpawnAgentRun` calls passing the
|
||||
*same* literal key inside one caller run dedupe; the same literal key from a
|
||||
*second* caller run queues, because the prefix differs. A test that drives only
|
||||
the empty-key path leaves the prefix logic unpinned.
|
||||
- **The wakeup queue increments its own counter (AC-004.6).** A
|
||||
`CreateWakeupRequest` rejected with `ErrWakeupIdempotencyConflict` moves
|
||||
`office_run_dedup_total{queue="wakeup",kind="durable"}` **through
|
||||
`ReportDurableDedup`, not `ReportInsertResult`** — the caller does its own
|
||||
`errors.Is` on the sentinel, so assert the counter moves without
|
||||
`runs/service` ever being handed that error. Labelled by
|
||||
`WakeupRequest.Reason` and not by `Source`, and the caller does not treat the
|
||||
sentinel as a failure. Nothing else in this plan exercises that file, so
|
||||
without this the label can be unwired or wired to the wrong `queue` value with
|
||||
the suite green.
|
||||
- **The three direct keyless producers each report their assigned cause** — the
|
||||
onboarding wake and `handleTaskCreated`'s fallback as `unresolved`, the
|
||||
recovery sweep as `by_design` — per
|
||||
[Part 3](run-dedup-generation-03.md#keyless-causes-per-producer). The recovery
|
||||
sweep additionally asserts it does **not** mint the assignment key: drive a
|
||||
sweep over a task whose assignment run already persisted
|
||||
`task_assigned:<task>:<agent>:<generation>` and assert the sweep still queues,
|
||||
rather than being suppressed by the durable index.
|
||||
- **`agent_error` and `manual_resume_after_failure` are generational** on the
|
||||
failed run id (`manual_resume_after_failure` falls back to the task id when
|
||||
the failed run id is empty). Two distinct run failures for one agent mint
|
||||
different keys and both queue; a redelivery of one failure escalation is
|
||||
suppressed. There is no two-CEO case to assert for `agent_error`: a
|
||||
workspace admits at most one CEO (`ErrAgentCEOAlreadyExists`) and
|
||||
`queueCEOAgentError` escalates to `ceos[0]` alone.
|
||||
- **Both blocker producers derive a byte-identical digest** for one blocker set,
|
||||
driven through the shared builder from both packages, including a set whose ids
|
||||
were returned in a different order. An empty blocker set enqueues nothing and
|
||||
moves no keyless counter.
|
||||
- **A path that enqueues nothing returns `QueueOutcomeNone`**, not
|
||||
`QueueOutcomeQueued`: a widened signature returning an error. Assert
|
||||
`ApplyTaskMutationResult.Runs` gains no entry. The `queue` closure's
|
||||
empty-agent-id guard is a **separate** case with a different assertion — it
|
||||
reaches no queue at all, so assert `res.Runs` gains no entry and that
|
||||
`QueueRunCtx` was never called, not that it returned an outcome.
|
||||
- **`QueueOutcomeNone` is declared identically in both packages.** A compile-time
|
||||
or table assertion that `runsservice.QueueOutcomeNone` and
|
||||
`engine.QueueOutcomeNone` hold the same value, pinning the "both MUST match"
|
||||
invariant those two declarations already carry.
|
||||
- **`ReportInsertResult` passes a non-conflict error through unchanged**, with
|
||||
`QueueOutcomeNone` and no counter movement — a disk error is not a dedup
|
||||
decision and must not be counted or swallowed as one.
|
||||
- **The `tasks` rebuild preserves the column and its values.** Drive
|
||||
`taskPriorityMigrationStatements` over a legacy fixture whose `tasks.priority`
|
||||
is still INTEGER, seeded with one row at `assignment_generation` 3 and one at
|
||||
`0`, and assert both survive the recreate unchanged. Also drive it over a
|
||||
fixture predating the column entirely and assert the defensive ALTER makes the
|
||||
`SELECT` succeed at `0`. Without this the loss is invisible: the column is
|
||||
re-added on the next boot, so nothing fails and only the resurrected key
|
||||
collision would ever show it.
|
||||
- Routine keys, per source: two cron fires of one trigger on **different claimed
|
||||
ticks** mint different keys; one slot re-dispatched with the same claimed tick
|
||||
mints the identical key; two **manual** fires of one routine inside one minute
|
||||
mint different keys (today they collide); and a `source == "cron"` dispatch with
|
||||
no claimed tick carried enqueues keyless with `cause=unresolved`. The cron case
|
||||
is driven by supplying the claimed tick directly, not by advancing a clock.
|
||||
- A cron producer does not recover the tick by re-reading the trigger row
|
||||
(AC-001.9): drive a dispatch after `UpdateTriggerNextRun` has advanced
|
||||
`next_run_at` and assert the key still names the claimed slot, not the next one.
|
||||
- `ReportKeylessEnqueue` is what moves `office_run_dedup_keyless_total`: a
|
||||
producer that goes keyless increments its own `cause` and logs its own
|
||||
`detail` constant — assert two producers going keyless under
|
||||
`reason=task_assigned` are distinguishable in the log record, which is the
|
||||
whole reason the argument exists; and an enqueue with an
|
||||
empty key that did **not** call the reporter moves no counter — the assertion
|
||||
that keeps the withdrawn "empty key means `by_design`" inference from returning.
|
||||
- An occurrence that coalesced into a queued neighbour and is then redelivered
|
||||
after the coalescing window queues a **second** run (AC-001.4, see
|
||||
[Key durability](#key-durability)): enqueue generation N, enqueue generation
|
||||
N+1 inside the window and assert `Coalesced`, age the merged row past the
|
||||
window, then re-enqueue generation N+1's key and assert `Queued`. This pins the
|
||||
accepted duplicate — a test asserting one run here is asserting the wrong
|
||||
contract.
|
||||
- Two producers enqueuing one key concurrently **with** a coalescible run already
|
||||
queued for that agent, reason and task: both observe a suppression, neither
|
||||
errors, and no row bears the new key (AC-002.3, AC-002.4). The no-neighbour
|
||||
arrangement is the separate case the same ACs bound, and needs its own test.
|
||||
- A durable-index conflict produced by re-deriving one occurrence's key past the
|
||||
lookback window (the `ParentWakeReconciler` shape) increments
|
||||
`office_run_dedup_total{kind=durable}`, returns `Deduped` with a nil error, and
|
||||
is not reported as a failure anywhere.
|
||||
|
||||
No Playwright coverage: the observable surfaces are `/debug/vars` and backend
|
||||
logs, and the one user-visible consequence (a run row appearing for a repeat
|
||||
assignment) is asserted at the queue in Go.
|
||||
@@ -0,0 +1,504 @@
|
||||
---
|
||||
status: draft
|
||||
system: office
|
||||
requirements:
|
||||
- REQ-OFFICE-RUN-DEDUP-001
|
||||
- REQ-OFFICE-RUN-DEDUP-002
|
||||
- REQ-OFFICE-RUN-DEDUP-003
|
||||
- REQ-OFFICE-RUN-DEDUP-004
|
||||
created: 2026-09-07
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# Office Run Deduplication — Generation Identity System Design Part 3: producer inventory and API contracts
|
||||
|
||||
Part 3 owns two things Parts 1 and 2 refer to but do not spell out: the complete
|
||||
inventory of every producer that reaches a run queue, and the exact Go contracts
|
||||
this capability adds or widens. The conceptual key contract is in
|
||||
[Part 1](run-dedup-generation-01.md); observability, key durability and the test
|
||||
plan are in [Part 2](run-dedup-generation-02.md).
|
||||
|
||||
Everything here is a contract a builder would otherwise have to invent. Where a
|
||||
name or a byte encoding is given, it is binding — two producers that must agree
|
||||
(AC-OFFICE-RUN-DEDUP-002.1) cannot agree on a convention nobody wrote down.
|
||||
|
||||
## Producer audit
|
||||
|
||||
Every site that reaches a run queue. "Generational" means the key already varies
|
||||
with the occurrence. The fourth table is the one Parts 1 and 2 previously
|
||||
omitted: producers that pass no key at all, which
|
||||
AC-OFFICE-RUN-DEDUP-003.3 nonetheless obliges to report a cause.
|
||||
|
||||
### Already generational — no change
|
||||
|
||||
| Producer | Key shape | Generation component |
|
||||
| --- | --- | --- |
|
||||
| `office/service` `queueCommentRun` | op id `task_comment:<commentID>`, engine-expanded on persist | comment row id |
|
||||
| `office/service` `handleApprovalResolved` | `approval_resolved:<approvalID>` | approval row id |
|
||||
| `office/approvals/service.go` `queueApprovalRun` | `approval:<approvalID>` | approval row id |
|
||||
| `office/dashboard/decisions.go` `decisionRunIdempotencyKey` | `decision:<decisionID>` | decision row id |
|
||||
| `office/scheduler` `childrenCompletedIdempotencyKey` | `task_children_completed:<parent>:<agent>:<sha256 of child ids>` | child-set digest |
|
||||
| `workflow/engine/phase2_callbacks.go` `idempotencyKey` | `<entryID or operationID>:<step>:<task>:<agent>:<digest>` | step-entry / operation id |
|
||||
| `orchestrator` `officeAutoStartIdempotencyKey` | `task_assigned:<task>:<agent>:<step>:<stepTransitionID>` | step-transition row id |
|
||||
| `orchestrator` step-entry operation id | `step_entry:<entryID>:<position>` | entry id and position |
|
||||
| `scheduler/cron/heartbeat.go` operation id | `heartbeat:<task>:<step>:<unix seconds>` | tick time (AC-001.6) |
|
||||
| `office/service/scheduler_wake_reconciler.go` `wakeOperationID` | `task_children_completed:<parent>:<child-id digest>` op id, shared with `queueChildrenCompletedRun` | child-set digest |
|
||||
|
||||
`officeAutoStartIdempotencyKey`'s `legacy:<task.UpdatedAt>` branch fires when
|
||||
`stepTransitionID` is zero. It is time-derived, which AC-001.5 disallows for an
|
||||
occurrence that has a durable row. Replace it with the keyless path of
|
||||
[Unresolvable generation](run-dedup-generation-01.md#unresolvable-generation); do
|
||||
not otherwise touch this producer.
|
||||
|
||||
### Not generational — must change
|
||||
|
||||
| Producer | Key today | Failure |
|
||||
| --- | --- | --- |
|
||||
| `office/service` `queueTaskAssignedRun` | `task_assigned:<task>:<agent>` | permanent per pair |
|
||||
| `office/scheduler/run.go` `QueueRunCtx` default | `<reason>:<task>:<agent>` | permanent per (reason, task, agent) |
|
||||
| `office/service` blocker-resolved dispatch | op id `blockers_resolved:<blockedTask>` | permanent per blocked task |
|
||||
|
||||
`QueueRunCtx`'s default is the larger of the two: every `RunContext` that does
|
||||
not set `IdempotencyKey` inherits it — every reactivity wake except
|
||||
children-completed.
|
||||
|
||||
### Unconstrained — must be scoped
|
||||
|
||||
| Producer | Key today | Failure |
|
||||
| --- | --- | --- |
|
||||
| `office/runtime/actions.go` `SpawnAgentRun` | verbatim `input.IdempotencyKey` | an agent reusing a literal permanently suppresses its own later wakes |
|
||||
| `office/routines/service.go` `buildRoutineIdempotencyKey` | `routine:<id>[:<trigger>]:<unix_minute>` | the minute comes from the **processing** clock (`run.StartedAt`, set to `time.Now()` inside `dispatchRoutineRun`), not from the occurrence. A cron fire's occurrence is its claimed scheduled tick; a manual or webhook fire's is `RoutineRun.ID`. Both are durable and neither is what the key uses. The reachable collision today is two manual fires of one routine inside one minute — they share `triggerID == ""` and the same minute |
|
||||
|
||||
### Keyless today — must report a cause
|
||||
|
||||
These four call a run queue **directly** with an empty key. None of them passes
|
||||
through `QueueRunCtx`, so the blanket entry in the second table does not reach
|
||||
them, and none was previously enumerated anywhere in this design. Each is
|
||||
covered by [Keyless causes per producer](#keyless-causes-per-producer) below, and
|
||||
each is a row the key-format table test of
|
||||
[Part 2](run-dedup-generation-02.md#testing) must iterate — under that test's
|
||||
keyless assertion (empty key, plus the reported `cause` and `detail`), never its
|
||||
varying-segment one, which a producer with no key cannot satisfy.
|
||||
|
||||
| Producer | Reason | Key today | Disposition |
|
||||
| --- | --- | --- | --- |
|
||||
| `office/onboarding/service.go` `maybeCreateOnboardingTask` | `task_assigned` | `""` | stays keyless, `cause=unresolved` |
|
||||
| `office/service/scheduler_recovery.go` recovery sweep | `task_assigned` | `""` | stays keyless, `cause=by_design` |
|
||||
| `office/service/retry.go` CEO error escalation | `agent_error` | `""` | **becomes generational** on the failed run id |
|
||||
| `office/service/failure.go` `requeueRunForTask` | `manual_resume_after_failure` | `""` | **becomes generational** on the failed run id (task id if empty) |
|
||||
|
||||
`office/onboarding` also declares its own `runReasonTaskAssigned` constant, a
|
||||
third copy of the same string. Consolidating it is out of scope for the same
|
||||
reason the requirements already give for the `office/scheduler` and
|
||||
`office/service` blocks.
|
||||
|
||||
### Writers that reach no queue at all
|
||||
|
||||
`UpdateTaskAssignee` has five non-test call sites and only one of them is a
|
||||
producer. The other four build no key and wake nobody, so they never appear in
|
||||
the tables above; they are enumerated in
|
||||
[UpdateTaskAssignee and the bump sites](#updatetaskassignee-and-the-bump-sites)
|
||||
because they still take the widened signature.
|
||||
|
||||
## Keyless causes per producer
|
||||
|
||||
AC-OFFICE-RUN-DEDUP-003.3 splits keyless enqueues into `unresolved` — a
|
||||
generation that should have been available and was not — and `by_design` — a
|
||||
producer that never had an occurrence to name. The split only pays for itself if
|
||||
each producer is assigned a side deliberately, so each is assigned one here.
|
||||
|
||||
| Producer | Reason | `cause` | Why |
|
||||
| --- | --- | --- | --- |
|
||||
| reactivity status wakes | `task_unblocked`, `task_reopened`, `task_review_requested` | `by_design` | a status transition has no redelivery path; see Part 1 |
|
||||
| `office/service/scheduler_recovery.go` | `task_assigned` | `by_design` | see [The recovery sweep](#the-recovery-sweep-must-not-borrow-the-assignment-generation) |
|
||||
| `office/onboarding/service.go` | `task_assigned` | `unresolved` | the occurrence **does** have a generation; this producer simply is not handed it |
|
||||
| `handleTaskCreated` with `fallbackToStoredRunner` | `task_assigned` | `unresolved` | Part 1 |
|
||||
| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `task_assigned` | `unresolved` | Part 1 |
|
||||
| `SpawnAgentRun`, no caller run id | agent-supplied | `unresolved` | Part 1 |
|
||||
| `SpawnAgentRun`, empty agent key | agent-supplied | `by_design` | the agent expressed no dedup intent; nothing failed to resolve |
|
||||
| routine dispatch, `cron` with no claimed tick | routine reasons | `unresolved` | Part 1 |
|
||||
| routine dispatch, unrecognised `Source` | routine reasons | `unresolved` | Part 1 |
|
||||
| reactivity comment wake, nil `MutationComment` | `task_comment` | `unresolved` | Part 1 |
|
||||
|
||||
**The `detail` argument, per producer.** Part 1's
|
||||
[`ReportKeylessEnqueue`](run-dedup-generation-01.md#unresolvable-generation)
|
||||
takes `detail` so the `cause=unresolved` log record names *what* failed to
|
||||
resolve, which `reason` alone cannot: four distinct producers above go keyless
|
||||
under `reason=task_assigned`. Each `unresolved` row reports a fixed lowercase
|
||||
snake_case constant naming the producer's own failure, not a formatted message:
|
||||
|
||||
| Producer | `detail` |
|
||||
| --- | --- |
|
||||
| `office/onboarding/service.go` | `onboarding_no_generation` |
|
||||
| `handleTaskCreated` with `fallbackToStoredRunner` | `event_missing_generation` |
|
||||
| `officeAutoStartIdempotencyKey`, zero `stepTransitionID` | `zero_step_transition` |
|
||||
| `SpawnAgentRun`, no caller run id | `no_caller_run` |
|
||||
| routine dispatch, `cron` with no claimed tick | `cron_no_claimed_tick` |
|
||||
| routine dispatch, unrecognised `Source` | `unrecognised_routine_source` |
|
||||
| reactivity comment wake, nil `MutationComment` | `nil_mutation_comment` |
|
||||
|
||||
Every `by_design` row passes `""`: that cause is counted and never logged, so it
|
||||
has nothing to distinguish. `detail` is a log field only and **not** a counter
|
||||
label — these seven values would multiply the keyless counter's cardinality for
|
||||
no operational question that `cause` and `reason` do not already answer.
|
||||
|
||||
### The recovery sweep must not borrow the assignment generation
|
||||
|
||||
`office/service/scheduler_recovery.go` re-queues tasks it finds unstarted. It is
|
||||
level-triggered from current task state, the same shape as `ParentWakeReconciler`,
|
||||
and by the time it runs, `tasks.assignment_generation` for that task is readable.
|
||||
Reading it is nonetheless **forbidden**, and not only because AC-001.9 rules out
|
||||
recovering a generation by re-reading a row.
|
||||
|
||||
The decisive argument is that it would defeat the sweep. The sweep exists to
|
||||
recover a task whose assignment wake was lost or whose run never started. If it
|
||||
minted `task_assigned:<task>:<agent>:<generation>` from the current row, that is
|
||||
byte-for-byte the key the original assignment already persisted on the run that
|
||||
failed. `idx_run_idempotency` is unbounded, so the recovery enqueue would be
|
||||
rejected as a durable dedup hit and the sweep would recover nothing — it would be
|
||||
a permanent no-op with a counter, which is the defect this whole capability
|
||||
exists to remove, reintroduced in the one path meant to be the backstop.
|
||||
|
||||
So the sweep enqueues keyless with `cause=by_design`. Its occurrence is "this
|
||||
task was still unstarted at this sweep", which no durable row records. The cost
|
||||
is a possible duplicate wake if the sweep runs twice before a run is claimed;
|
||||
AC-003.2 elects exactly that trade. The 5-second coalescing window absorbs the
|
||||
common case.
|
||||
|
||||
### Onboarding is a third `task_assigned` producer
|
||||
|
||||
`maybeCreateOnboardingTask` calls `CreateOfficeTask` -> `taskservice.CreateTask`
|
||||
with `AssigneeAgentProfileID` set, then enqueues a `task_assigned` run itself.
|
||||
Because that creation publishes `events.TaskCreated`, `queueTaskAssignedRun` also
|
||||
observes the same occurrence and — after this capability — mints the keyed
|
||||
`task_assigned:<task>:<agent>:1`.
|
||||
|
||||
Part 1's [Convergent producers](run-dedup-generation-01.md#convergent-producers)
|
||||
says two producers observe an assignment. For a creation occurrence reached
|
||||
through onboarding there are **three**, and the third cannot converge with the
|
||||
other two: `CreateOfficeTask` returns only the task id, so onboarding never
|
||||
receives the generation the creating transaction committed. Widening that
|
||||
adapter is not undertaken here.
|
||||
|
||||
Onboarding therefore enqueues keyless with `cause=unresolved` — `unresolved`
|
||||
rather than `by_design` precisely because the occurrence *does* have a
|
||||
generation and this producer merely is not handed it, which is the distinction
|
||||
AC-003.3 exists to make countable. This does not violate AC-002.1: that criterion
|
||||
binds two producers that **each derive a key**, and a producer enqueueing with no
|
||||
key is governed by REQ-003 instead. The scoping is in the criterion's own text,
|
||||
so the onboarding pair needs no exemption of its own. The consequence is stated rather than left to
|
||||
be discovered: onboarding's wake and `queueTaskAssignedRun`'s wake are two
|
||||
enqueues for one occurrence, and only one carries a key. Both fire on one call
|
||||
path within milliseconds, for one agent, reason and task, so `CoalesceRun` merges
|
||||
them into a single queued run in the ordinary case. If they are ever separated by
|
||||
more than the coalescing window, the operator gets a duplicate onboarding wake.
|
||||
That is AC-003.2's elected direction, and the counter makes it visible.
|
||||
|
||||
### `agent_error` becomes generational
|
||||
|
||||
`office/service/retry.go` escalates a run failure to the workspace CEO. Its
|
||||
payload already carries `run_id`, so the occurrence has a durable identity and
|
||||
the audit's own rule applies: a producer with an occurrence to name must name it.
|
||||
|
||||
```text
|
||||
agent_error:<failedRunID>:<ceoAgentProfileID>
|
||||
```
|
||||
|
||||
The CEO agent id is included to keep the key's shape uniform with the other
|
||||
recipient-addressed keys (`task_comment`, `task_mentioned`), which is the whole
|
||||
of its justification. **It is not doing collision-avoidance work**, and a builder
|
||||
must not go looking for the fan-out that would make it necessary: a workspace has
|
||||
at most one CEO — `office/agents/service.go` and `office/service/agents.go` both
|
||||
reject a second with `ErrAgentCEOAlreadyExists` — and `queueCEOAgentError`
|
||||
(`office/service/retry.go`) escalates to `ceos[0]` alone, with no fan-out.
|
||||
`agent_error:<failedRunID>` would be equally correct today; the salt is retained
|
||||
because it costs nothing and survives a CEO seat being replaced.
|
||||
|
||||
A redelivery of one failure escalation is suppressed; a second, distinct failure
|
||||
of the same agent produces a different `run_id` and wakes again.
|
||||
|
||||
This producer moves out of the keyless path entirely and takes no `cause`. AC-001.8
|
||||
needs no argument here: every `agent_error` row persisted before this capability
|
||||
carries a NULL `idempotency_key`, and both unique indexes are partial and ignore
|
||||
NULL, so an old row can never collide with the new key.
|
||||
|
||||
## Go API contracts
|
||||
|
||||
### `UpdateTaskAssignee` and the bump sites
|
||||
|
||||
`office/repository/sqlite/tasks.go` and the interface declaration in
|
||||
`office/dashboard/service.go` both widen to return the generation the
|
||||
transaction committed:
|
||||
|
||||
```go
|
||||
UpdateTaskAssignee(ctx context.Context, taskID, assigneeID string) (int64, error)
|
||||
```
|
||||
|
||||
The returned value is read back **inside the existing transaction, after the
|
||||
UPDATE and before `Commit`**. It is the generation of *this* assignment
|
||||
occurrence, which is the only value a producer may key on.
|
||||
|
||||
All five non-test call sites, and what each does with the value:
|
||||
|
||||
| Call site | Role | Disposition |
|
||||
| --- | --- | --- |
|
||||
| `office/dashboard/service_tasks.go` (reached from `DashboardService.SetTaskAssigneeAsAgent`) | **producer** | carries the value into `TaskReactivityChange.AssignmentGeneration` |
|
||||
| `office/service/task_assignee.go` `SetTaskAssignee` | non-producer | discards it (`_`); its own signature is unchanged |
|
||||
| `office/service/task_assignee.go` `SetTaskAssigneeAsAgent` | non-producer | discards it (`_`); its own signature is unchanged |
|
||||
| `office/service/channels.go` channel setup | non-producer | discards it (`_`) |
|
||||
| `office/channels/service.go` channel setup | non-producer | discards it (`_`) |
|
||||
|
||||
**The channel-task flow is explicitly out of scope as a producer, and in scope as
|
||||
a bump.** Both channel sites assign a long-lived channel task's runner seat
|
||||
through `UpdateTaskAssignee`, so both bump the counter — that is correct, a
|
||||
channel task's runner genuinely is being assigned. Neither publishes an
|
||||
assignment event nor runs the reactivity pipeline, so neither builds a key and
|
||||
neither wakes anybody. They take the widened signature and discard the value. No
|
||||
`ReportKeylessEnqueue` call is made, because no enqueue happens.
|
||||
|
||||
**A read-back failure fails the assignment.** The read is a `SELECT` of one
|
||||
column on the row just written, in the same transaction, so it can only fail if
|
||||
that transaction is already unusable. When it does fail, the transaction rolls
|
||||
back and `UpdateTaskAssignee` returns the error with a zero generation: the
|
||||
assignment does **not** commit. Committing an assignment whose generation could
|
||||
not be read would produce an occurrence that no producer can ever name, which is
|
||||
strictly worse than a failed call the caller can retry.
|
||||
|
||||
The create-path bump site, `task/repository/sqlite/task.go` `insertTaskTx` ->
|
||||
`upsertRunnerInTx`, inserts the literal `1` under the guard that writes the
|
||||
runner row — `task.AssigneeAgentProfileID != "" && task.WorkflowStepID != ""` —
|
||||
so it needs no read-back and neither signature changes.
|
||||
|
||||
**What carries the generation onto `task.created`.** Not a post-commit re-read;
|
||||
AC-001.9 forbids it and it is not needed, because the value is a constant the
|
||||
guard above already determines:
|
||||
|
||||
- the guard fired -> a runner row committed at generation `1`;
|
||||
- the guard did not fire -> no runner row, and the task keeps the column default,
|
||||
generation `0`.
|
||||
|
||||
`CreateTask` (`task/service/service_tasks.go`) holds the same two fields the
|
||||
guard reads, on the very `*models.Task` it handed the transaction, so it
|
||||
re-evaluates that condition without a query and publishes the result through
|
||||
`publishTaskEventWithExtra`
|
||||
(`task/service/service_events.go`), whose `extra map[string]interface{}` is the
|
||||
carrier. The create site calls `publishTaskEvent` today and moves to the
|
||||
`WithExtra` form; `publishTaskEvent` already delegates to it, so no new plumbing
|
||||
is introduced.
|
||||
|
||||
**A task created unassigned publishes generation `0`, not `1`.** `0` means "never
|
||||
assigned" and can never name an assignment occurrence, so no `task_assigned`
|
||||
producer keys on it — a creation that assigned nobody produces no assignment
|
||||
occurrence to wake for, and the task's first real assignment goes through
|
||||
`UpdateTaskAssignee` and commits `1`. Publishing `1` for an unassigned creation
|
||||
would let one task mint `task_assigned:<task>:<agent>:1` twice — once at a
|
||||
creation that assigned nobody, once at its genuine first assignment — recreating
|
||||
the durable collision this capability exists to remove.
|
||||
|
||||
This satisfies AC-001.9 rather than bending it. The prohibition is on recovering
|
||||
a generation by re-reading a row a later occurrence has since moved; nothing is
|
||||
re-read here. The value is derived from in-memory fields of the struct the
|
||||
creating transaction was given, so it cannot observe a later occurrence, and for
|
||||
a creation it is a constant either way.
|
||||
|
||||
### The event payload field
|
||||
|
||||
`assignment_generation` on the `task.created` / `task.updated` payload is a
|
||||
**nullable** JSON number, decoded into the `*int64` that Part 1 specifies:
|
||||
|
||||
- absent, or JSON `null` -> `nil` -> the producer goes keyless with
|
||||
`cause=unresolved`;
|
||||
- a JSON number -> that generation, used verbatim.
|
||||
|
||||
`0` on the wire is a literal generation zero and **is not** a sentinel for
|
||||
"absent". A task at generation `0` has never been assigned, so it can never be
|
||||
the subject of a `task_assigned` occurrence; but a decoder that mapped `0` to
|
||||
"absent" would silently convert a malformed payload into a keyless enqueue, and
|
||||
one that mapped "absent" to `0` would mint `...:0` keys that collide across every
|
||||
unassigned task. Neither is permitted: the field is nullable on the wire and
|
||||
optional in the decoder, and the two states stay distinct.
|
||||
|
||||
### The shared key builders
|
||||
|
||||
Two exported builders live beside `internal/runs/commentkeys`, reachable from
|
||||
`office/service`, `office/scheduler` and `orchestrator` without an import cycle.
|
||||
|
||||
**The assignment key.** Both `task_assigned` producers call it rather than
|
||||
formatting the string themselves, which is what makes their convergence
|
||||
structural instead of coincidental (AC-002.1).
|
||||
|
||||
**The blocker-set digest.** Both blocker producers — `task_blockers_resolved` in
|
||||
`office/scheduler` and the `blockers_resolved` operation id in `office/service` —
|
||||
call **one shared builder**. They live in different packages, and
|
||||
AC-002.1 requires them to derive an identical key for one occurrence, so a
|
||||
convention each implements separately is not sufficient: two packages choosing
|
||||
`,` and `:` as a delimiter produce different digests for the same blocker set and
|
||||
fail invisibly, presenting as a durable dedup miss rather than as an error.
|
||||
|
||||
The encoding is binding, and is the encoding
|
||||
`childrenCompletedIdempotencyKey` already uses, so the two agree by construction:
|
||||
|
||||
1. take the `blocker_task_id` of each blocker in the wave;
|
||||
2. sort ascending by byte value (**not** by `ListTaskBlockers`'s `created_at`,
|
||||
which is not unique);
|
||||
3. join with a single `,` (U+002C);
|
||||
4. SHA-256 over the UTF-8 bytes of that string;
|
||||
5. render the full 32-byte digest as lowercase hex (Go `%x`), never truncated.
|
||||
|
||||
No de-duplication step is needed before sorting: `task_blockers` is
|
||||
`PRIMARY KEY (task_id, blocker_task_id)`, so one task cannot list the same
|
||||
blocker twice and two reads of an unchanged set always digest the same input.
|
||||
|
||||
**An empty blocker set is not an occurrence and is not enqueued.** A wave with no
|
||||
blockers has nothing to have resolved, and the alternative is worse than a
|
||||
missing wake: the digest of the empty string is a constant, so an empty-set key
|
||||
would be `task_blockers_resolved:<task>:<agent>:<one fixed digest>` — permanently
|
||||
unique per (task, agent), which is precisely the defect this capability removes.
|
||||
A producer that finds an empty set returns without enqueuing and without calling
|
||||
`ReportKeylessEnqueue`, because no enqueue was attempted.
|
||||
|
||||
**The set is digested at the readiness read, and is never re-read to build the
|
||||
key.** Each blocker producer already reads the blocker rows to decide whether the
|
||||
task is ready; that read is the snapshot the digest uses. This is the
|
||||
capture-and-carry AC-001.9 imposes on the assignment generation and the cron
|
||||
claimed tick, applied to a set rather than a scalar, and it is what stops a
|
||||
producer digesting a set that a later wave has already moved.
|
||||
|
||||
- `office/service` `resolveAndWakeIfUnblocked` (`event_subscribers.go`) already
|
||||
holds the slice it read from `ListTaskBlockers`, and digests that slice.
|
||||
- `office/scheduler` `allBlockersResolvedExcept` (`reactivity.go`) reads the same
|
||||
rows and currently **discards** them, returning only a boolean. It returns the
|
||||
slice alongside that boolean, and `cascadeBlockersResolved` passes it to the
|
||||
shared builder rather than issuing a second read.
|
||||
|
||||
Two producers observing one occurrence therefore digest what each saw at its own
|
||||
readiness decision, and the consequence is stated rather than left to be
|
||||
inferred. A blocker edge **added** between the two reads makes the later producer
|
||||
find the task not ready at all, so it enqueues nothing. An edge **removed**
|
||||
between them makes it digest a smaller set and mint a second key, which is a
|
||||
duplicate wake — the direction AC-003.2 elects. A set that differs between the
|
||||
two reads is a different wave, so AC-002.1's "same occurrence" premise does not
|
||||
hold across it and no convergence is owed. Nothing here may lock the blocker
|
||||
table or re-read to reconcile the two.
|
||||
|
||||
**A failed blocker read is not an empty set, and the two must not collapse.** An
|
||||
empty set returns without enqueuing because there was no occurrence; a failed
|
||||
read means the producer does not know whether there was one. On a read error the
|
||||
producer propagates the error and enqueues nothing — it does **not** fall through
|
||||
to the empty-set arm, does not enqueue keyless, and does not call
|
||||
`ReportKeylessEnqueue`, because no enqueue was attempted in either case.
|
||||
|
||||
It must nonetheless be visible, and on one of the two paths it is not today.
|
||||
`resolveAndWakeIfUnblocked` returns the error and `queueBlockersResolvedRuns`
|
||||
logs it at `Error` with the blocked task id. `cascadeBlockersResolved` instead
|
||||
drops it with a bare `continue` on `err != nil || !ready`, recording nothing, so
|
||||
a wake lost to a read failure is indistinguishable from a task that was
|
||||
legitimately still blocked. That arm gains an `Error` log naming the blocked task
|
||||
and the error. This is the only behaviour change this section makes to the
|
||||
readiness check itself; the check's own logic is unchanged.
|
||||
|
||||
### The suppression reporter and the no-op outcome
|
||||
|
||||
Part 2 requires one shared helper so three queue implementations report one
|
||||
behaviour. It is exported from `internal/runs/service`, beside
|
||||
`ReportKeylessEnqueue`, as three functions rather than one — the suppressions
|
||||
are reached at different points and carry different evidence, and one of them is
|
||||
classified by its caller rather than here:
|
||||
|
||||
```go
|
||||
type QueueSource string
|
||||
|
||||
const (
|
||||
QueueSourceRuns QueueSource = "runs"
|
||||
QueueSourceWakeup QueueSource = "wakeup"
|
||||
)
|
||||
|
||||
// The recent-duplicate lookup matched. Counts kind="windowed", logs at Info,
|
||||
// and returns QueueOutcomeDeduped.
|
||||
func ReportWindowedDedup(q QueueSource, reason, key string) QueueOutcome
|
||||
|
||||
// Classifies the error from an insert. A unique violation on idempotency_key
|
||||
// counts kind="durable", logs at Warn with key, reason and agent, and returns
|
||||
// (QueueOutcomeDeduped, nil). Any other non-nil error is returned UNCHANGED
|
||||
// with QueueOutcomeNone and moves no counter. A nil error returns
|
||||
// (QueueOutcomeQueued, nil).
|
||||
func ReportInsertResult(
|
||||
q QueueSource, reason, key, agentProfileID string, err error,
|
||||
) (QueueOutcome, error)
|
||||
|
||||
// A durable conflict the CALLER has already classified. Counts kind="durable",
|
||||
// logs at Warn, and returns QueueOutcomeDeduped. ReportInsertResult delegates
|
||||
// here once its own classification succeeds.
|
||||
func ReportDurableDedup(q QueueSource, reason, key, agentProfileID string) QueueOutcome
|
||||
```
|
||||
|
||||
`ReportInsertResult` is the single place the *runs* unique violation is
|
||||
recognised and mapped, which is what makes AC-002.3 hold on `queueRunInline` and
|
||||
`SchedulerService.QueueRun` as well as in `runs/service`. Neither function ever
|
||||
returns `QueueOutcomeCoalesced`: all three queues order
|
||||
`CheckIdempotencyKey` -> `CoalesceRun` -> `insertRun`, so an enqueue that
|
||||
coalesced returns from the coalesce arm and never reaches an insert to classify. A non-conflict error is
|
||||
deliberately passed through untouched: a disk error is not a dedup decision and
|
||||
must not be counted as one, nor swallowed into a successful-looking outcome.
|
||||
|
||||
**The wakeup path classifies its own conflict and calls `ReportDurableDedup`**,
|
||||
not `ReportInsertResult` (AC-004.6). `ReportInsertResult` recognises a conflict
|
||||
with `runssqlite.IsIdempotencyKeyUniqueViolation`
|
||||
(`internal/runs/repository/sqlite`), which matches `idx_run_idempotency` on the
|
||||
`runs` table and is already the classifier `runs/service` uses. It does **not**
|
||||
recognise `ErrWakeupIdempotencyConflict`
|
||||
(`office/repository/sqlite/wakeup_requests.go`) — a wrapped sentinel raised by a
|
||||
different table's index, in the office tree — and it must not be taught to:
|
||||
importing that sentinel would reverse the import direction this design pins
|
||||
(`runs/service` imports only `office/models` from the office tree).
|
||||
|
||||
So the classification stays where the sentinel already is. The wakeup caller
|
||||
lives in the office tree, already receives the error, does its own
|
||||
`errors.Is(err, ErrWakeupIdempotencyConflict)`, and calls `ReportDurableDedup`
|
||||
with `QueueSourceWakeup`. Its `kind` is always `durable`: that table has no
|
||||
windowed lookup. This is reporting only; per AC-004.4 and AC-004.5 the wakeup
|
||||
signature itself is not widened and keeps returning that sentinel, and the caller
|
||||
keeps treating it as a suppression rather than a failure.
|
||||
|
||||
**The no-op outcome.** `QueueOutcome` is a string type whose zero value is `""`,
|
||||
which is not one of `queued`, `deduped` or `coalesced`. That zero value is given
|
||||
a name and a meaning rather than being left for a builder to rediscover:
|
||||
|
||||
```go
|
||||
// QueueOutcomeNone means no enqueue was attempted, or the attempt returned an
|
||||
// error. It is the zero value, so it is what a widened signature yields on any
|
||||
// path that returns before deciding an outcome.
|
||||
const QueueOutcomeNone QueueOutcome = ""
|
||||
```
|
||||
|
||||
**It is declared in both copies of the type.** `QueueOutcome` exists twice —
|
||||
`internal/runs/service/service.go` and `internal/workflow/engine/adapters.go` —
|
||||
and each declaration carries the doc invariant "both MUST match".
|
||||
`QueueOutcomeNone` is therefore added to **both**, with the same value and the
|
||||
same doc comment. Part 1 lists `internal/workflow/engine` as already generational
|
||||
and unchanged; that is a statement about its *keys*, and adding this constant is
|
||||
the only edit this capability makes to that package. Adding it to one side alone
|
||||
would break an invariant the code asserts in its own comments.
|
||||
|
||||
This is additive and changes no acceptance criterion. AC-004.4's obligation to
|
||||
report queued, deduplicated or coalesced is conditioned on a caller enqueuing a
|
||||
wake; a path that enqueues nothing never meets that condition. The paths that
|
||||
return `QueueOutcomeNone` are the widened signatures returning a non-nil error,
|
||||
including `ReportInsertResult`'s non-conflict arm.
|
||||
|
||||
**`QueueRunCtx` is not one of them.** It has no early return on an empty resolved
|
||||
agent id. That guard sits one level up, in the reactivity pipeline's `queue`
|
||||
closure (`office/scheduler/reactivity.go`), which returns *before* calling
|
||||
`QueueRunCtx` at all — so an empty agent id yields no outcome to report rather
|
||||
than `QueueOutcomeNone`, and `ApplyTaskMutationResult.Runs` gains nothing because
|
||||
no enqueue was attempted. Part 1's
|
||||
[reactivity gate](run-dedup-generation-01.md#the-reactivity-gate) states this
|
||||
correctly and is the reference.
|
||||
|
||||
No caller may treat `QueueOutcomeNone` as success. In particular the reactivity
|
||||
pipeline's `queue` closure appends to `ApplyTaskMutationResult.Runs` only for
|
||||
`QueueOutcomeQueued`, so neither a suppression nor a no-op is reported as a
|
||||
queued run. Returning `QueueOutcomeQueued` from a path that inserted nothing
|
||||
would put a phantom run in that slice, which is the specific mis-report Part 2
|
||||
added the `Queued`-only rule to prevent.
|
||||
Reference in New Issue
Block a user