diff --git a/apps/backend/internal/backendapp/adapters_office.go b/apps/backend/internal/backendapp/adapters_office.go index b4e1669a45..87041e86e5 100644 --- a/apps/backend/internal/backendapp/adapters_office.go +++ b/apps/backend/internal/backendapp/adapters_office.go @@ -260,6 +260,7 @@ func (a *routineWakeupAdapter) CreateWakeupRequest( Reason: req.Reason, Payload: req.Payload, RequestedAt: req.RequestedAt, + CausationID: req.CausationID, } if req.IdempotencyKey != "" { row.IdempotencyKey = sql.NullString{String: req.IdempotencyKey, Valid: true} diff --git a/apps/backend/internal/backendapp/cron.go b/apps/backend/internal/backendapp/cron.go index 3be3f44ad0..8cab00a3ec 100644 --- a/apps/backend/internal/backendapp/cron.go +++ b/apps/backend/internal/backendapp/cron.go @@ -12,6 +12,7 @@ import ( officemodels "github.com/kandev/kandev/internal/office/models" officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" officeroutines "github.com/kandev/kandev/internal/office/routines" + officeservice "github.com/kandev/kandev/internal/office/service" schedulercron "github.com/kandev/kandev/internal/scheduler/cron" tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" workflowrepo "github.com/kandev/kandev/internal/workflow/repository" @@ -48,6 +49,10 @@ func startCronScheduler( routines := schedulercron.NewRoutinesHandler(routineTicker, nil, log) loop := schedulercron.NewLoop(schedulercron.DefaultTickInterval, log, heartbeat, budget, routines, officeRecovery, parentWakeReconciler) + // AC-OFFICE-LOOP-LIVENESS-003.6: published once, at boot, so a flat + // office_loop_cron_tick_total is distinguishable from a restarted + // process versus a genuinely stopped loop. + officeservice.RecordLoopProcessStarted(time.Now().UTC().Format(time.RFC3339)) loop.Start(ctx) log.Info("phase 5 cron loop started", zap.Duration("interval", schedulercron.DefaultTickInterval)) diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 2ccf00347a..5961116984 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -106,6 +106,7 @@ import ( officeskills "github.com/kandev/kandev/internal/office/skills" officewakeup "github.com/kandev/kandev/internal/office/wakeup" orchexecutor "github.com/kandev/kandev/internal/orchestrator/executor" + v1 "github.com/kandev/kandev/pkg/api/v1" // Runs queue (Phase 3 of task-model-unification) runsscheduler "github.com/kandev/kandev/internal/runs/scheduler" @@ -1474,6 +1475,10 @@ func wireOfficeSvcsDependencies( orchestratorSvc.SetOfficeTaskStatusUpdater(services.OfficeSvcs.Dashboard) // Wire the office service as the retry canceller for task reassignment. services.OfficeSvcs.Dashboard.SetRetryCanceller(services.Office) + // Wire the office service as the terminal-shape recorder so a + // dashboard-driven displaced-run cancellation still counts toward + // office_loop_terminal_total. + services.OfficeSvcs.Dashboard.SetTerminalShapeRecorder(services.Office) // Wire the office service as the task canceller for status→cancelled hard-cancels. services.OfficeSvcs.Dashboard.SetTaskCanceller(services.Office) // Route the Office "No parent" mutation through the canonical task detach @@ -1573,6 +1578,13 @@ type schedulerTaskStarterAdapter struct { orch *orchestrator.Service } +// The production routed-launch adapter is required to satisfy +// TaskStarterWithSession — no silent fallback to a session-less launch +// (AC-OFFICE-LOOP-LIVENESS-002.7). A compile-time assertion pins this +// harder than a runtime wiring test: dropping the method fails the +// build, not just a test run. +var _ officescheduler.TaskStarterWithSession = (*schedulerTaskStarterAdapter)(nil) + // StartTask starts a task on the orchestrator with the given launch parameters. func (a *schedulerTaskStarterAdapter) StartTask( ctx context.Context, @@ -1594,6 +1606,32 @@ func (a *schedulerTaskStarterAdapter) StartTaskWithRoute( launch officescheduler.LaunchContext, route officescheduler.RouteOverride, ) error { + _, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route) + return err +} + +// StartTaskWithRouteReturningSession implements +// officescheduler.TaskStarterWithSession so the routed launch path can +// persist the session id it started (AC-OFFICE-LOOP-LIVENESS-002.7). +func (a *schedulerTaskStarterAdapter) StartTaskWithRouteReturningSession( + ctx context.Context, + taskID, agentProfileID string, + launch officescheduler.LaunchContext, + route officescheduler.RouteOverride, +) (string, error) { + execution, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route) + if err != nil || execution == nil { + return "", err + } + return execution.SessionID, nil +} + +func (a *schedulerTaskStarterAdapter) startTaskWithRoute( + ctx context.Context, + taskID, agentProfileID string, + launch officescheduler.LaunchContext, + route officescheduler.RouteOverride, +) (*orchexecutor.TaskExecution, error) { return a.orch.StartTaskWithRoute(ctx, taskID, agentProfileID, orchexecutor.LaunchContext{ ExecutorID: launch.ExecutorID, @@ -2069,19 +2107,101 @@ func backfillAgentDefaultSkills( } } -// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in the -// officeservice.TaskStarterWithLaunchContextFunc adapter. Extracted from -// initOfficeServices to keep that function under the funlen cap. +// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in an +// adapter that also implements officeservice.TaskStarterWithLaunchContextSession, +// so the direct (non-routed) launch path forwards per-run skill additions and +// persists the session id the orchestrator's *executor.TaskExecution already +// carries (AC-OFFICE-LOOP-LIVENESS-002.7) instead of discarding it. Extracted +// from initOfficeServices to keep that function under the funlen cap. func newOfficeTaskStarter(orchestratorSvc *orchestrator.Service) officeservice.TaskStarter { - return officeservice.TaskStarterWithLaunchContextFunc( - func(ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext) error { - _, err := orchestratorSvc.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID, - launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt, - launch.WorkflowStepID, launch.PlanMode, false, launch.Attachments, launch.Env, - launch.AdditionalSkillSlugs) - return err - }, - ) + return &officeOrchestratorTaskStarter{orch: orchestratorSvc} +} + +// officeOrchestratorTaskStarter satisfies officeservice.TaskStarter, +// TaskStarterWithEnv, TaskStarterWithLaunchContext, TaskStarterWithSession, +// and TaskStarterWithLaunchContextSession against the orchestrator service's +// direct launch path. +type officeOrchestratorTaskStarter struct { + orch *orchestrator.Service +} + +// The production direct-launch adapter is required to satisfy +// TaskStarterWithLaunchContextSession for the same reason as the routed +// adapter above (AC-OFFICE-LOOP-LIVENESS-002.7): the session id must be +// recorded unconditionally, whether or not the launch also carries skills. +var _ officeservice.TaskStarterWithLaunchContextSession = (*officeOrchestratorTaskStarter)(nil) + +// StartTask implements officeservice.TaskStarter. +func (a *officeOrchestratorTaskStarter) StartTask( + ctx context.Context, taskID, agentProfileID, executorID, + executorProfileID string, priority string, prompt, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, +) error { + _, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID, + executorProfileID, priority, prompt, workflowStepID, planMode, attachments, nil, nil) + return err +} + +// StartTaskWithEnv implements officeservice.TaskStarterWithEnv. +func (a *officeOrchestratorTaskStarter) StartTaskWithEnv( + ctx context.Context, taskID, agentProfileID, executorID, + executorProfileID string, priority string, prompt, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, env map[string]string, +) error { + _, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID, + executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil) + return err +} + +// StartTaskWithEnvReturningSession implements +// officeservice.TaskStarterWithSession. +func (a *officeOrchestratorTaskStarter) StartTaskWithEnvReturningSession( + ctx context.Context, taskID, agentProfileID, executorID, + executorProfileID string, priority string, prompt, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, env map[string]string, +) (string, error) { + execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID, + executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil) + if err != nil || execution == nil { + return "", err + } + return execution.SessionID, nil +} + +// StartTaskWithLaunchContext implements officeservice.TaskStarterWithLaunchContext. +func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContext( + ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext, +) error { + _, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, + launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt, + launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env, + launch.AdditionalSkillSlugs) + return err +} + +// StartTaskWithLaunchContextReturningSession implements +// officeservice.TaskStarterWithLaunchContextSession. +func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContextReturningSession( + ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext, +) (string, error) { + execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, + launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt, + launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env, + launch.AdditionalSkillSlugs) + if err != nil || execution == nil { + return "", err + } + return execution.SessionID, nil +} + +func (a *officeOrchestratorTaskStarter) startTaskWithEnvAndSkills( + ctx context.Context, taskID, agentProfileID, executorID, + executorProfileID string, priority string, prompt, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, env map[string]string, additionalSkillSlugs []string, +) (*orchexecutor.TaskExecution, error) { + return a.orch.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID, + executorID, executorProfileID, priority, prompt, + workflowStepID, planMode, false, attachments, env, additionalSkillSlugs) } // newAgentAuth wraps officeagents.NewAgentAuth with a dev-mode warning when diff --git a/apps/backend/internal/db/sqlguard/exemptions.json b/apps/backend/internal/db/sqlguard/exemptions.json index d2473597fc..9bff08d2e8 100644 --- a/apps/backend/internal/db/sqlguard/exemptions.json +++ b/apps/backend/internal/db/sqlguard/exemptions.json @@ -68,6 +68,8 @@ {"file": "internal/office/repository/sqlite/dashboard.go", "symbol": "QueryRunActivity", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"}, {"file": "internal/office/repository/sqlite/labels.go", "symbol": "GetOrCreateLabel", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"}, {"file": "internal/office/repository/sqlite/labels.go", "symbol": "AddLabelToTask", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"}, + {"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "CountEligibleTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"}, + {"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "ListOverdueOrStrandedTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"}, {"file": "internal/office/repository/sqlite/tasks.go", "symbol": "hasFTSTable", "rule": "sqlite-catalog", "reason": "SQLite-only FTS inspection"}, {"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTask", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"}, {"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTaskForRun", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"}, diff --git a/apps/backend/internal/office/dashboard/handler.go b/apps/backend/internal/office/dashboard/handler.go index 580ce62d4e..200113d112 100644 --- a/apps/backend/internal/office/dashboard/handler.go +++ b/apps/backend/internal/office/dashboard/handler.go @@ -44,6 +44,7 @@ type Handler struct { gitMgr *configloader.GitManager runDetail RunDetailRepo agentSummary AgentSummaryRepository + loopHealth LoopHealthRepo handoff *taskservice.HandoffService guard ActiveSourceChecker logger *logger.Logger @@ -76,6 +77,9 @@ func NewHandler(svc *DashboardService, labelRepo labelFetcher, gitMgr *configloa if r, ok := labelRepo.(AgentSummaryRepository); ok { h.agentSummary = r } + if r, ok := labelRepo.(LoopHealthRepo); ok { + h.loopHealth = r + } return h } @@ -127,6 +131,8 @@ func RegisterRoutes(api *gin.RouterGroup, svc *DashboardService, labelRepo label api.GET("/workspaces/:wsId/routing/preview", h.getWorkspaceRoutingPreview) api.GET("/runs/:id/attempts", h.listRunAttempts) api.GET("/agents/:id/route", h.getAgentRoute) + + registerLoopHealthRoutes(api, h) } // -- Dashboard -- diff --git a/apps/backend/internal/office/dashboard/handler_loop_health.go b/apps/backend/internal/office/dashboard/handler_loop_health.go new file mode 100644 index 0000000000..39073595a8 --- /dev/null +++ b/apps/backend/internal/office/dashboard/handler_loop_health.go @@ -0,0 +1,64 @@ +package dashboard + +import ( + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/office/service" +) + +// registerLoopHealthRoutes mounts the two REQ-OFFICE-LOOP-LIVENESS +// reads on the Office route group. Both inherit AgentAuthMiddleware +// and officeWorkspaceScopeMiddleware from the group they're mounted +// on; :wsId is their only id, so neither needs a +// officeParamScopeResolvers entry. +func registerLoopHealthRoutes(api *gin.RouterGroup, h *Handler) { + api.GET("/workspaces/:wsId/loop-health", h.getLoopHealth) + api.GET("/workspaces/:wsId/loop-counters", h.getLoopCounters) +} + +// getLoopHealth serves GET /workspaces/:wsId/loop-health +// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once here and +// threaded through every predicate (AC-004.14). Any unreadable input +// fails the whole read with 503 rather than a partial verdict +// (AC-004.10), incrementing office_loop_liveness_degraded_total by +// the failing input's reason. +func (h *Handler) getLoopHealth(c *gin.Context) { + if h.loopHealth == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: "loop health not configured"}) + return + } + wsID := c.Param("wsId") + now := time.Now().UTC() + + resp, err := EvaluateLoopHealth(c.Request.Context(), h.loopHealth, wsID, now) + if err != nil { + h.respondLoopHealthError(c, wsID, err) + return + } + c.JSON(http.StatusOK, resp) +} + +func (h *Handler) respondLoopHealthError(c *gin.Context, workspaceID string, err error) { + var degraded *LoopHealthDegradedError + reason := ReasonRunReadFailed + if errors.As(err, °raded) { + reason = degraded.Reason + } + service.IncLoopLivenessDegraded(workspaceID, reason) + c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: err.Error(), "reason": reason}) +} + +// getLoopCounters serves GET /workspaces/:wsId/loop-counters +// (REQ-OFFICE-LOOP-LIVENESS-003). Reads process-wide expvar state, so +// it never fails or 503s — an empty workspace simply reads all zeros. +func (h *Handler) getLoopCounters(c *gin.Context) { + wsID := c.Param("wsId") + c.JSON(http.StatusOK, ReadLoopCounters(wsID)) +} + +// LoopHealthRepoImpl returns the wired loop-health repo (may be nil). +// Exposed for tests that need to confirm the wiring. +func (h *Handler) LoopHealthRepoImpl() LoopHealthRepo { return h.loopHealth } diff --git a/apps/backend/internal/office/dashboard/loop_counters.go b/apps/backend/internal/office/dashboard/loop_counters.go new file mode 100644 index 0000000000..0ec252199f --- /dev/null +++ b/apps/backend/internal/office/dashboard/loop_counters.go @@ -0,0 +1,161 @@ +package dashboard + +import ( + "expvar" + "strings" + + "github.com/kandev/kandev/internal/office/service" +) + +// loopCounterMapNames lists every office_loop_* expvar.Map counter +// name (REQ-OFFICE-LOOP-LIVENESS-003, "Counters" table). Zero-filled +// at counter-name granularity (AC-003.8's malformed-key note and the +// design's "Zero-filling is at counter-name granularity" rule): a +// name absent from a reader's response would be indistinguishable +// from "never observed". +var loopCounterMapNames = []string{ + "office_loop_trigger_claimed_total", + "office_loop_routine_run_total", + "office_loop_wakeup_created_total", + "office_loop_run_claimed_total", + "office_loop_launch_total", + "office_loop_launch_without_session_total", + "office_loop_session_persist_failed_total", + "office_loop_terminal_total", + "office_loop_last_run_at_write_failed_total", + "office_loop_liveness_degraded_total", +} + +// LoopCountersResponse is the GET .../loop-counters payload. workspace +// carries every counter above labelled to the requesting workspace +// (secondary labels like source/disposition/shape/reason preserved, +// but only as observed); process carries the cron-tick counter, the +// two published instants, and the _unattributed totals — identical +// for every reader, no workspace's data (AC-003.5, AC-003.8). +type LoopCountersResponse struct { + Workspace map[string]map[string]int64 `json:"workspace"` + Process LoopCountersProcessDTO `json:"process"` +} + +// LoopCountersProcessDTO is the process-scoped block of the counter +// read: never filtered by workspace, identical for every caller. +type LoopCountersProcessDTO struct { + CronTickTotal int64 `json:"cron_tick_total"` + CronTickAt string `json:"cron_tick_at"` + ProcessStartedAt string `json:"process_started_at"` + Unattributed map[string]map[string]int64 `json:"unattributed"` + Malformed map[string]map[string]int64 `json:"malformed"` +} + +// ReadLoopCounters parses every office_loop_* expvar.Map back into a +// workspace-filtered, zero-filled response (AC-003.4, AC-003.5, +// AC-003.7). A key that fails to parse into the expected "k=v;k=v" +// label shape lands in the process block's malformed bucket rather +// than being dropped or attributed to any workspace, the same +// treatment AC-003.8 gives an unresolved workspace. +func ReadLoopCounters(workspaceID string) LoopCountersResponse { + resp := LoopCountersResponse{ + Workspace: make(map[string]map[string]int64, len(loopCounterMapNames)), + Process: LoopCountersProcessDTO{ + Unattributed: make(map[string]map[string]int64), + Malformed: make(map[string]map[string]int64), + }, + } + for _, name := range loopCounterMapNames { + resp.Workspace[name] = map[string]int64{} + } + + if v := expvar.Get("office_loop_cron_tick_total"); v != nil { + if m, ok := v.(*expvar.Map); ok { + resp.Process.CronTickTotal = expvarMapTotal(m) + } + } + if v := expvar.Get("office_loop_cron_tick_at"); v != nil { + resp.Process.CronTickAt = v.String() + resp.Process.CronTickAt = strings.Trim(resp.Process.CronTickAt, `"`) + } + if v := expvar.Get("office_loop_process_started_at"); v != nil { + resp.Process.ProcessStartedAt = strings.Trim(v.String(), `"`) + } + + for _, name := range loopCounterMapNames { + v := expvar.Get(name) + if v == nil { + continue + } + m, ok := v.(*expvar.Map) + if !ok { + continue + } + partitionLoopCounterMap(&resp, name, m, workspaceID) + } + return resp +} + +// partitionLoopCounterMap walks one counter's labelled entries and +// routes each into the workspace block, the process unattributed +// bucket, or the process malformed bucket. +func partitionLoopCounterMap(resp *LoopCountersResponse, mapName string, m *expvar.Map, workspaceID string) { + m.Do(func(kv expvar.KeyValue) { + iv, ok := kv.Value.(*expvar.Int) + if !ok { + return + } + labels, ok := parseLoopCounterKey(kv.Key) + if !ok { + addLoopCounterBucket(resp.Process.Malformed, mapName, kv.Key, iv.Value()) + return + } + ws, hasWorkspace := labels["workspace"] + switch { + case hasWorkspace && ws == workspaceID: + resp.Workspace[mapName][kv.Key] = iv.Value() + case hasWorkspace && ws == service.LoopUnattributedWorkspace: + addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value()) + case hasWorkspace: + // A different workspace's value — filtered out entirely + // (AC-003.5): not returned in any form to this reader. + default: + // No workspace label at all (process-only counter reached + // via this generic loop, or a future label-less counter) — + // treat as unattributed rather than silently dropped. + addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value()) + } + }) +} + +func addLoopCounterBucket(bucket map[string]map[string]int64, mapName, key string, value int64) { + if bucket[mapName] == nil { + bucket[mapName] = map[string]int64{} + } + bucket[mapName][key] = value +} + +// parseLoopCounterKey parses the "k1=v1;k2=v2" expvar map key back +// into labels. Total parse (AC-003.8's argument extended to this +// read): any segment that doesn't split into exactly one "=" fails +// the whole key rather than silently dropping that label. +func parseLoopCounterKey(key string) (map[string]string, bool) { + if key == "" { + return map[string]string{}, true + } + labels := make(map[string]string) + for _, part := range strings.Split(key, ";") { + kv := strings.SplitN(part, "=", 2) + if len(kv) != 2 || kv[0] == "" { + return nil, false + } + labels[kv[0]] = kv[1] + } + return labels, true +} + +func expvarMapTotal(m *expvar.Map) int64 { + var total int64 + m.Do(func(kv expvar.KeyValue) { + if iv, ok := kv.Value.(*expvar.Int); ok { + total += iv.Value() + } + }) + return total +} diff --git a/apps/backend/internal/office/dashboard/loop_counters_test.go b/apps/backend/internal/office/dashboard/loop_counters_test.go new file mode 100644 index 0000000000..de4ad46c1f --- /dev/null +++ b/apps/backend/internal/office/dashboard/loop_counters_test.go @@ -0,0 +1,75 @@ +package dashboard + +import ( + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/service" +) + +func TestReadLoopCounters_FiltersToRequestingWorkspace(t *testing.T) { + wsA := "ws-counters-a" + wsB := "ws-counters-b" + service.IncLoopRunClaimed(wsA) + service.IncLoopRunClaimed(wsB) + + resp := ReadLoopCounters(wsA) + + got := resp.Workspace["office_loop_run_claimed_total"] + if _, ok := got[service.LoopMetricLabel("workspace", wsB)]; ok { + t.Errorf("workspace block contains another workspace's key: %+v", got) + } + if v, ok := got[service.LoopMetricLabel("workspace", wsA)]; !ok || v < 1 { + t.Errorf("workspace block missing own key, got %+v", got) + } +} + +func TestReadLoopCounters_UnobservedCounterIsZeroFilledNotOmitted(t *testing.T) { + resp := ReadLoopCounters("ws-counters-never-seen") + + for _, name := range loopCounterMapNames { + if _, ok := resp.Workspace[name]; !ok { + t.Errorf("counter %q missing from workspace block, want zero-filled empty map", name) + } + } +} + +func TestReadLoopCounters_UnattributedAndProcessInstantsPresent(t *testing.T) { + service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace) + service.IncLoopCronTick("2026-05-01T12:00:00Z") + service.RecordLoopProcessStarted("2026-05-01T11:00:00Z") + + resp := ReadLoopCounters("ws-counters-some-other-workspace") + + unattr := resp.Process.Unattributed["office_loop_trigger_claimed_total"] + if _, ok := unattr[service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; !ok { + t.Errorf("unattributed bucket missing entry, got %+v", unattr) + } + if _, ok := resp.Workspace["office_loop_trigger_claimed_total"][service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; ok { + t.Errorf("unattributed key leaked into workspace block: %+v", resp.Workspace["office_loop_trigger_claimed_total"]) + } + if resp.Process.CronTickTotal < 1 { + t.Errorf("cron tick total = %d, want >= 1", resp.Process.CronTickTotal) + } + if resp.Process.CronTickAt == "" { + t.Error("cron tick at is empty, want the published instant") + } + if resp.Process.ProcessStartedAt == "" { + t.Error("process started at is empty, want the published instant") + } +} + +func TestReadLoopCounters_MalformedKeyCountsIntoMalformedBucket(t *testing.T) { + m := expvar.Get("office_loop_run_claimed_total").(*expvar.Map) + m.Add("not-a-valid-label", 1) + + resp := ReadLoopCounters("ws-counters-malformed-probe") + + malformed := resp.Process.Malformed["office_loop_run_claimed_total"] + if v, ok := malformed["not-a-valid-label"]; !ok || v < 1 { + t.Errorf("malformed bucket missing key, got %+v", malformed) + } + if _, ok := resp.Workspace["office_loop_run_claimed_total"]["not-a-valid-label"]; ok { + t.Error("malformed key leaked into workspace block") + } +} diff --git a/apps/backend/internal/office/dashboard/loop_health.go b/apps/backend/internal/office/dashboard/loop_health.go new file mode 100644 index 0000000000..e7cc0a9041 --- /dev/null +++ b/apps/backend/internal/office/dashboard/loop_health.go @@ -0,0 +1,347 @@ +package dashboard + +import ( + "context" + "time" + + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/service" +) + +// Thresholds (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.8 — echoed verbatim +// in every response so a caller never has to hardcode them). +const ( + officeLoopTriggerOverdueGrace = 3 * time.Minute + officeLoopTriggerStrandedGrace = 5 * time.Minute + officeLoopRunQueuedGrace = 2 * time.Minute + officeLoopRunClaimedGrace = 10 * time.Minute + officeLoopEvaluationWindow = 24 * time.Hour + officeLoopEvidenceCap = 50 +) + +// Verdict values, closed set (AC-004.1), precedence order +// unknown -> dead -> degraded -> not_armed -> healthy (AC-004.2). +const ( + VerdictUnknown = "unknown" + VerdictDead = "dead" + VerdictDegraded = "degraded" + VerdictNotArmed = "not_armed" + VerdictHealthy = "healthy" +) + +// Degraded reasons (AC-004.10), the enumerated set +// office_loop_liveness_degraded_total{reason} is labelled with. Per +// OPERATOR DECISION F31, the silent-success evidence read and the +// terminal-shape count read both map to ReasonTerminalReadFailed — +// both ultimately answer "what shape did this run end in", so a +// caller diagnosing a 503 reads one reason for either query rather +// than a distinction that doesn't help them act. +const ( + ReasonActivationReadFailed = "activation_read_failed" + ReasonTriggerReadFailed = "trigger_read_failed" + ReasonRunReadFailed = "run_read_failed" + ReasonTerminalReadFailed = "terminal_read_failed" +) + +// LoopHealthDegradedError names which input could not be read +// (AC-004.10). The HTTP layer maps this to a 503 and increments +// office_loop_liveness_degraded_total{reason}. +type LoopHealthDegradedError struct { + Reason string + Err error +} + +func (e *LoopHealthDegradedError) Error() string { + return "loop health degraded (" + e.Reason + "): " + e.Err.Error() +} + +func (e *LoopHealthDegradedError) Unwrap() error { return e.Err } + +// LoopHealthRepo is the persistence surface EvaluateLoopHealth needs. +// Implemented by *sqlite.Repository. +type LoopHealthRepo interface { + LoopLivenessActivationChecked() (at time.Time, published bool, err error) + CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error) + ListOverdueOrStrandedTriggers( + ctx context.Context, workspaceID string, now time.Time, + overdueGrace, strandedGrace time.Duration, limit int, + ) ([]sqlite.TriggerHealthRow, int, error) + ListStuckRuns( + ctx context.Context, workspaceID string, now time.Time, + queuedGrace, claimedGrace time.Duration, limit int, + ) ([]sqlite.StuckRunRow, int, error) + ListSilentSuccesses( + ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int, + ) ([]sqlite.SilentSuccessRow, int, error) + ListTerminalRunsInWindow( + ctx context.Context, workspaceID string, windowStart time.Time, + ) ([]sqlite.TerminalRunShapeInputRow, error) +} + +// TriggerEvidenceDTO is one overdue or stranded trigger row (AC-004.7). +type TriggerEvidenceDTO struct { + TriggerID string `json:"trigger_id"` + RoutineID string `json:"routine_id"` + RoutineName string `json:"routine_name"` + Condition string `json:"condition"` + NextRunAt *time.Time `json:"next_run_at,omitempty"` + LastFiredAt *time.Time `json:"last_fired_at,omitempty"` +} + +// StuckRunEvidenceDTO is one claimed-stuck or queued-stuck run row. +type StuckRunEvidenceDTO struct { + RunID string `json:"run_id"` + AgentProfileID string `json:"agent_profile_id"` + Status string `json:"status"` + Condition string `json:"condition"` + StuckSince *time.Time `json:"stuck_since,omitempty"` +} + +// SilentSuccessEvidenceDTO is one finished/processed/sessionless run. +type SilentSuccessEvidenceDTO struct { + RunID string `json:"run_id"` + RequestedAt time.Time `json:"requested_at"` + FinishedAt *time.Time `json:"finished_at,omitempty"` +} + +// EvidenceListDTO wraps a capped list with its untruncated total +// (AC-004.9): "capped" is true exactly when total > len(rows). +type triggerEvidenceListDTO struct { + Rows []TriggerEvidenceDTO `json:"rows"` + Total int `json:"total"` + Cap int `json:"cap"` + Capped bool `json:"capped"` +} + +type stuckRunEvidenceListDTO struct { + Rows []StuckRunEvidenceDTO `json:"rows"` + Total int `json:"total"` + Cap int `json:"cap"` + Capped bool `json:"capped"` +} + +type silentSuccessEvidenceListDTO struct { + Rows []SilentSuccessEvidenceDTO `json:"rows"` + Total int `json:"total"` + Cap int `json:"cap"` + Capped bool `json:"capped"` +} + +// LoopHealthThresholdsDTO echoes every threshold and the evaluation +// window used to reach the verdict (AC-004.8), in seconds so the +// response carries no ambiguous duration string. +type LoopHealthThresholdsDTO struct { + TriggerOverdueGraceSeconds int `json:"trigger_overdue_grace_seconds"` + TriggerStrandedGraceSeconds int `json:"trigger_stranded_grace_seconds"` + RunQueuedGraceSeconds int `json:"run_queued_grace_seconds"` + RunClaimedGraceSeconds int `json:"run_claimed_grace_seconds"` + EvaluationWindowSeconds int `json:"evaluation_window_seconds"` + EvidenceCap int `json:"evidence_cap"` +} + +// LoopHealthResponse is the GET .../loop-health payload. +type LoopHealthResponse struct { + Verdict string `json:"verdict"` + EvaluatedAt time.Time `json:"evaluated_at"` + Thresholds LoopHealthThresholdsDTO `json:"thresholds"` + EligibleTriggerCount int `json:"eligible_trigger_count"` + Triggers triggerEvidenceListDTO `json:"triggers"` + StuckRuns stuckRunEvidenceListDTO `json:"stuck_runs"` + SilentSuccesses silentSuccessEvidenceListDTO `json:"silent_successes"` + TerminalShapeCounts map[string]int `json:"terminal_shape_counts"` + EvaluationWindowStart time.Time `json:"evaluation_window_start"` +} + +// allTerminalShapes lists every TerminalShape so the response +// zero-fills unobserved shapes rather than omitting their key, +// mirroring the counter read's zero-fill-by-name rule. +var allTerminalShapes = []service.TerminalShape{ + service.ShapePreActivation, + service.ShapeLaunchedCompleted, + service.ShapeLaunchedFailed, + service.ShapeSilentSuccess, + service.ShapeUnlaunchedSkipped, + service.ShapeUnlaunchedFailed, + service.ShapeUnclassified, +} + +// EvaluateLoopHealth answers "is the loop alive" for one workspace +// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once by the caller +// and passed to every predicate so the trigger, run and terminal reads +// cannot disagree about the instant they were evaluated against +// (AC-004.14). Side-effect free (AC-004.11): every call is a read. +// +// On any unreadable input, returns a *LoopHealthDegradedError naming +// the failing input rather than a partial verdict (AC-004.10). +func EvaluateLoopHealth( + ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time, +) (*LoopHealthResponse, error) { + thresholds := LoopHealthThresholdsDTO{ + TriggerOverdueGraceSeconds: int(officeLoopTriggerOverdueGrace.Seconds()), + TriggerStrandedGraceSeconds: int(officeLoopTriggerStrandedGrace.Seconds()), + RunQueuedGraceSeconds: int(officeLoopRunQueuedGrace.Seconds()), + RunClaimedGraceSeconds: int(officeLoopRunClaimedGrace.Seconds()), + EvaluationWindowSeconds: int(officeLoopEvaluationWindow.Seconds()), + EvidenceCap: officeLoopEvidenceCap, + } + windowStart := now.Add(-officeLoopEvaluationWindow) + + activationAt, published, err := repo.LoopLivenessActivationChecked() + if err != nil { + return nil, &LoopHealthDegradedError{Reason: ReasonActivationReadFailed, Err: err} + } + if !published { + return &LoopHealthResponse{ + Verdict: VerdictUnknown, + EvaluatedAt: now, + Thresholds: thresholds, + Triggers: triggerEvidenceListDTO{Rows: []TriggerEvidenceDTO{}, Cap: officeLoopEvidenceCap}, + StuckRuns: stuckRunEvidenceListDTO{Rows: []StuckRunEvidenceDTO{}, Cap: officeLoopEvidenceCap}, + SilentSuccesses: silentSuccessEvidenceListDTO{Rows: []SilentSuccessEvidenceDTO{}, Cap: officeLoopEvidenceCap}, + TerminalShapeCounts: zeroFilledShapeCounts(), + EvaluationWindowStart: windowStart, + }, nil + } + + triggerEvidence, eligibleCount, err := evaluateLoopHealthTriggers(ctx, repo, workspaceID, now) + if err != nil { + return nil, err + } + stuckEvidence, err := evaluateLoopHealthStuckRuns(ctx, repo, workspaceID, now) + if err != nil { + return nil, err + } + silentEvidence, err := evaluateLoopHealthSilentSuccesses(ctx, repo, workspaceID, windowStart, activationAt) + if err != nil { + return nil, err + } + shapeCounts, err := evaluateLoopHealthTerminalShapes(ctx, repo, workspaceID, windowStart, activationAt, published) + if err != nil { + return nil, err + } + + verdict := decideLoopHealthVerdict(triggerEvidence.Total, eligibleCount, stuckEvidence.Total, silentEvidence.Total) + + return &LoopHealthResponse{ + Verdict: verdict, + EvaluatedAt: now, + Thresholds: thresholds, + EligibleTriggerCount: eligibleCount, + Triggers: triggerEvidence, + StuckRuns: stuckEvidence, + SilentSuccesses: silentEvidence, + TerminalShapeCounts: shapeCounts, + EvaluationWindowStart: windowStart, + }, nil +} + +// decideLoopHealthVerdict applies the AC-004.2 precedence +// unknown -> dead -> degraded -> not_armed -> healthy. unknown is +// decided by the caller before evidence is even read, so this +// function only sees the four remaining branches. +func decideLoopHealthVerdict(triggerTotal, eligibleCount, stuckTotal, silentTotal int) string { + switch { + case triggerTotal > 0: + return VerdictDead + case stuckTotal > 0 || silentTotal > 0: + return VerdictDegraded + case eligibleCount == 0: + return VerdictNotArmed + default: + return VerdictHealthy + } +} + +func evaluateLoopHealthTriggers( + ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time, +) (triggerEvidenceListDTO, int, error) { + eligibleCount, err := repo.CountEligibleTriggers(ctx, workspaceID) + if err != nil { + return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err} + } + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, workspaceID, now, officeLoopTriggerOverdueGrace, officeLoopTriggerStrandedGrace, officeLoopEvidenceCap, + ) + if err != nil { + return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err} + } + dtos := make([]TriggerEvidenceDTO, 0, len(rows)) + for _, r := range rows { + dtos = append(dtos, TriggerEvidenceDTO{ + TriggerID: r.TriggerID, + RoutineID: r.RoutineID, + RoutineName: r.RoutineName, + Condition: r.Condition, + NextRunAt: r.NextRunAt, + LastFiredAt: r.LastFiredAt, + }) + } + return triggerEvidenceListDTO{ + Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos), + }, eligibleCount, nil +} + +func evaluateLoopHealthStuckRuns( + ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time, +) (stuckRunEvidenceListDTO, error) { + rows, total, err := repo.ListStuckRuns( + ctx, workspaceID, now, officeLoopRunQueuedGrace, officeLoopRunClaimedGrace, officeLoopEvidenceCap, + ) + if err != nil { + return stuckRunEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonRunReadFailed, Err: err} + } + dtos := make([]StuckRunEvidenceDTO, 0, len(rows)) + for _, r := range rows { + dtos = append(dtos, StuckRunEvidenceDTO{ + RunID: r.RunID, AgentProfileID: r.AgentProfileID, Status: r.Status, + Condition: r.Condition, StuckSince: r.StuckSince, + }) + } + return stuckRunEvidenceListDTO{ + Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos), + }, nil +} + +// evaluateLoopHealthSilentSuccesses reports ReasonTerminalReadFailed +// on failure, not ReasonRunReadFailed — OPERATOR DECISION F31. +func evaluateLoopHealthSilentSuccesses( + ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart, activationAt time.Time, +) (silentSuccessEvidenceListDTO, error) { + rows, total, err := repo.ListSilentSuccesses(ctx, workspaceID, windowStart, activationAt, officeLoopEvidenceCap) + if err != nil { + return silentSuccessEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err} + } + dtos := make([]SilentSuccessEvidenceDTO, 0, len(rows)) + for _, r := range rows { + dtos = append(dtos, SilentSuccessEvidenceDTO{ + RunID: r.RunID, RequestedAt: r.RequestedAt, FinishedAt: r.FinishedAt, + }) + } + return silentSuccessEvidenceListDTO{ + Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos), + }, nil +} + +func evaluateLoopHealthTerminalShapes( + ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart time.Time, + activationAt time.Time, published bool, +) (map[string]int, error) { + rows, err := repo.ListTerminalRunsInWindow(ctx, workspaceID, windowStart) + if err != nil { + return nil, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err} + } + counts := zeroFilledShapeCounts() + for _, r := range rows { + shape := service.ClassifyTerminalRun(r.Status, r.Outcome, r.SessionID, r.RequestedAt, activationAt, published) + counts[string(shape)]++ + } + return counts, nil +} + +func zeroFilledShapeCounts() map[string]int { + counts := make(map[string]int, len(allTerminalShapes)) + for _, s := range allTerminalShapes { + counts[string(s)] = 0 + } + return counts +} diff --git a/apps/backend/internal/office/dashboard/loop_health_route_test.go b/apps/backend/internal/office/dashboard/loop_health_route_test.go new file mode 100644 index 0000000000..895d8c8b56 --- /dev/null +++ b/apps/backend/internal/office/dashboard/loop_health_route_test.go @@ -0,0 +1,103 @@ +package dashboard_test + +import ( + "encoding/json" + "expvar" + "net/http" + "net/http/httptest" + "testing" + + "github.com/kandev/kandev/internal/office/service" +) + +// degradedExpvarInt reads one office_loop_liveness_degraded_total cell +// by its full label key, mirroring the terminalShapeExpvarInt helper +// pattern used for the same expvar.Map shape elsewhere in this suite. +func degradedExpvarInt(t *testing.T, key string) int64 { + t.Helper() + v := expvar.Get("office_loop_liveness_degraded_total") + if v == nil { + t.Fatal("expvar map office_loop_liveness_degraded_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatal("office_loop_liveness_degraded_total is not a Map") + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("office_loop_liveness_degraded_total[%q] is not an Int", key) + } + return iv.Value() +} + +// AC-003.4: the loop-health and loop-counters routes are reachable +// with no dev-mode/pprof env var set — newTestDeps's router mounts +// them with no such gate, matching production registration. +func TestGetLoopHealth_ReachableWithoutDevMode(t *testing.T) { + deps := newTestDeps(t) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-health", nil) + w := httptest.NewRecorder() + deps.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } +} + +func TestGetLoopCounters_ReachableWithoutDevMode(t *testing.T) { + deps := newTestDeps(t) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-counters", nil) + w := httptest.NewRecorder() + deps.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } +} + +// AC-004.10: a read failure through the real, database-backed +// LoopHealthRepo (not the pure-Go fake used by loop_health_test.go) +// must reach the HTTP layer as a 503 carrying a reason field, and +// must increment office_loop_liveness_degraded_total by that reason +// — exercising the full handler wiring, not just EvaluateLoopHealth +// in isolation. +func TestGetLoopHealth_ReadFailureRespondsServiceUnavailableAndIncrementsCounter(t *testing.T) { + deps := newTestDeps(t) + wsID := "ws-loop-health-db-closed" + key := service.LoopMetricLabel("workspace", wsID, "reason", "activation_read_failed") + before := degradedExpvarInt(t, key) + + if err := deps.db.Close(); err != nil { + t.Fatalf("close db: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/"+wsID+"/loop-health", nil) + w := httptest.NewRecorder() + deps.router.ServeHTTP(w, req) + + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503: %s", w.Code, w.Body.String()) + } + + var body map[string]any + if err := json.NewDecoder(w.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body["reason"] != "activation_read_failed" { + t.Errorf("reason = %v, want activation_read_failed", body["reason"]) + } + if _, ok := body["error"]; !ok { + t.Errorf("body = %+v, want an error field", body) + } + + after := degradedExpvarInt(t, key) + if after != before+1 { + t.Errorf("degraded counter delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/dashboard/loop_health_test.go b/apps/backend/internal/office/dashboard/loop_health_test.go new file mode 100644 index 0000000000..8f49e8b6cc --- /dev/null +++ b/apps/backend/internal/office/dashboard/loop_health_test.go @@ -0,0 +1,390 @@ +package dashboard + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/repository/sqlite" +) + +// fakeLoopHealthRepo is a hand-rolled LoopHealthRepo for exercising +// EvaluateLoopHealth's verdict precedence and failure-reason mapping +// without a database. +type fakeLoopHealthRepo struct { + activationAt time.Time + activationPublished bool + activationErr error + + eligibleCount int + eligibleErr error + + triggerRows []sqlite.TriggerHealthRow + triggerTotal int + triggerErr error + + stuckRows []sqlite.StuckRunRow + stuckTotal int + stuckErr error + + silentRows []sqlite.SilentSuccessRow + silentTotal int + silentErr error + + terminalRows []sqlite.TerminalRunShapeInputRow + terminalErr error +} + +func (f *fakeLoopHealthRepo) LoopLivenessActivationChecked() (time.Time, bool, error) { + return f.activationAt, f.activationPublished, f.activationErr +} + +func (f *fakeLoopHealthRepo) CountEligibleTriggers(context.Context, string) (int, error) { + return f.eligibleCount, f.eligibleErr +} + +func (f *fakeLoopHealthRepo) ListOverdueOrStrandedTriggers( + context.Context, string, time.Time, time.Duration, time.Duration, int, +) ([]sqlite.TriggerHealthRow, int, error) { + return f.triggerRows, f.triggerTotal, f.triggerErr +} + +func (f *fakeLoopHealthRepo) ListStuckRuns( + context.Context, string, time.Time, time.Duration, time.Duration, int, +) ([]sqlite.StuckRunRow, int, error) { + return f.stuckRows, f.stuckTotal, f.stuckErr +} + +func (f *fakeLoopHealthRepo) ListSilentSuccesses( + context.Context, string, time.Time, time.Time, int, +) ([]sqlite.SilentSuccessRow, int, error) { + return f.silentRows, f.silentTotal, f.silentErr +} + +func (f *fakeLoopHealthRepo) ListTerminalRunsInWindow( + context.Context, string, time.Time, +) ([]sqlite.TerminalRunShapeInputRow, error) { + return f.terminalRows, f.terminalErr +} + +func baseFakeRepo(now time.Time) *fakeLoopHealthRepo { + return &fakeLoopHealthRepo{ + activationAt: now.Add(-48 * time.Hour), + activationPublished: true, + eligibleCount: 1, + } +} + +func TestEvaluateLoopHealth_UnpublishedActivationIsUnknown(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.activationPublished = false + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictUnknown { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown) + } + if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 { + t.Errorf("expected empty evidence lists when unknown, got %+v", resp) + } + for shape, count := range resp.TerminalShapeCounts { + if count != 0 { + t.Errorf("shape %q = %d, want 0 (zero-filled)", shape, count) + } + } +} + +func TestEvaluateLoopHealth_OverdueTriggerIsDead(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}} + repo.triggerTotal = 1 + // Even with a stuck run also present, dead outranks degraded. + repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1"}} + repo.stuckTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictDead { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDead) + } +} + +func TestEvaluateLoopHealth_StuckRunIsDegraded(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}} + repo.stuckTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictDegraded { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded) + } +} + +func TestEvaluateLoopHealth_SilentSuccessIsDegraded(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.silentRows = []sqlite.SilentSuccessRow{{RunID: "r1", RequestedAt: now.Add(-time.Hour)}} + repo.silentTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictDegraded { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded) + } +} + +func TestEvaluateLoopHealth_NoEligibleTriggerIsNotArmed(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.eligibleCount = 0 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-empty", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictNotArmed { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictNotArmed) + } + if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 { + t.Errorf("expected empty evidence for a workspace with nothing wrong, got %+v", resp) + } +} + +func TestEvaluateLoopHealth_NothingWrongIsHealthy(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictHealthy { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictHealthy) + } +} + +func TestEvaluateLoopHealth_EchoesThresholds(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + want := LoopHealthThresholdsDTO{ + TriggerOverdueGraceSeconds: 180, + TriggerStrandedGraceSeconds: 300, + RunQueuedGraceSeconds: 120, + RunClaimedGraceSeconds: 600, + EvaluationWindowSeconds: 86400, + EvidenceCap: 50, + } + if resp.Thresholds != want { + t.Errorf("thresholds = %+v, want %+v", resp.Thresholds, want) + } + wantWindowStart := now.Add(-24 * time.Hour) + if !resp.EvaluationWindowStart.Equal(wantWindowStart) { + t.Errorf("window start = %v, want %v", resp.EvaluationWindowStart, wantWindowStart) + } +} + +func TestEvaluateLoopHealth_ClassifiesTerminalRunsIntoShapeCounts(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + processed := "processed" + repo.terminalRows = []sqlite.TerminalRunShapeInputRow{ + {Status: "finished", Outcome: &processed, SessionID: "s1", RequestedAt: now.Add(-time.Hour)}, + {Status: "finished", Outcome: &processed, SessionID: "", RequestedAt: now.Add(-time.Hour)}, + } + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.TerminalShapeCounts["launched_completed"] != 1 { + t.Errorf("launched_completed = %d, want 1", resp.TerminalShapeCounts["launched_completed"]) + } + if resp.TerminalShapeCounts["silent_success"] != 1 { + t.Errorf("silent_success = %d, want 1", resp.TerminalShapeCounts["silent_success"]) + } +} + +func TestEvaluateLoopHealth_ActivationReadFailureDegrades(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.activationErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonActivationReadFailed) +} + +func TestEvaluateLoopHealth_TriggerReadFailureDegrades(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.triggerErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonTriggerReadFailed) +} + +func TestEvaluateLoopHealth_EligibleTriggerCountReadFailureDegrades(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.eligibleErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonTriggerReadFailed) +} + +func TestEvaluateLoopHealth_StuckRunReadFailureDegrades(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.stuckErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonRunReadFailed) +} + +// OPERATOR DECISION F31: both the silent-success read and the +// terminal-shape read map to ReasonTerminalReadFailed. +func TestEvaluateLoopHealth_SilentSuccessReadFailureDegradesWithTerminalReason(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.silentErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonTerminalReadFailed) +} + +func TestEvaluateLoopHealth_TerminalShapeReadFailureDegradesWithTerminalReason(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.terminalErr = errors.New("boom") + + _, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + assertDegradedReason(t, err, ReasonTerminalReadFailed) +} + +// AC-004.2: unknown outranks dead. An unpublished activation must win +// even when overdue-trigger evidence (the dead condition) is also +// present in the same read. +func TestEvaluateLoopHealth_UnknownOutranksDead(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.activationPublished = false + repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}} + repo.triggerTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictUnknown { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown) + } +} + +// AC-004.2: degraded outranks not_armed. Zero eligible triggers (the +// not_armed condition) must not suppress a stuck run already in +// flight from before the workspace went unarmed. +func TestEvaluateLoopHealth_DegradedOutranksNotArmed(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + repo.eligibleCount = 0 + repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}} + repo.stuckTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if resp.Verdict != VerdictDegraded { + t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded) + } +} + +// AC-004.7: evidence rows carry the offending row's identifying data, +// not just a nonzero count that flips the verdict. +func TestEvaluateLoopHealth_EvidenceRowsArePopulated(t *testing.T) { + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + repo := baseFakeRepo(now) + nextRunAt := now.Add(time.Hour) + repo.triggerRows = []sqlite.TriggerHealthRow{{ + TriggerID: "trig-1", RoutineID: "rout-1", RoutineName: "Nightly sync", + Condition: "overdue", NextRunAt: &nextRunAt, + }} + repo.triggerTotal = 1 + stuckSince := now.Add(-20 * time.Minute) + repo.stuckRows = []sqlite.StuckRunRow{{ + RunID: "run-1", AgentProfileID: "agent-1", Status: "claimed", + Condition: "claimed_stuck", StuckSince: &stuckSince, + }} + repo.stuckTotal = 1 + repo.silentRows = []sqlite.SilentSuccessRow{{ + RunID: "run-2", RequestedAt: now.Add(-time.Hour), + }} + repo.silentTotal = 1 + + resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + + if len(resp.Triggers.Rows) != 1 { + t.Fatalf("trigger rows = %d, want 1", len(resp.Triggers.Rows)) + } + gotTrigger := resp.Triggers.Rows[0] + if gotTrigger.TriggerID != "trig-1" || gotTrigger.RoutineID != "rout-1" || + gotTrigger.RoutineName != "Nightly sync" || gotTrigger.Condition != "overdue" { + t.Errorf("trigger row = %+v, want trig-1/rout-1/Nightly sync/overdue", gotTrigger) + } + if gotTrigger.NextRunAt == nil || !gotTrigger.NextRunAt.Equal(nextRunAt) { + t.Errorf("trigger row next_run_at = %v, want %v", gotTrigger.NextRunAt, nextRunAt) + } + + if len(resp.StuckRuns.Rows) != 1 { + t.Fatalf("stuck run rows = %d, want 1", len(resp.StuckRuns.Rows)) + } + gotStuck := resp.StuckRuns.Rows[0] + if gotStuck.RunID != "run-1" || gotStuck.AgentProfileID != "agent-1" || + gotStuck.Status != "claimed" || gotStuck.Condition != "claimed_stuck" { + t.Errorf("stuck run row = %+v, want run-1/agent-1/claimed/claimed_stuck", gotStuck) + } + if gotStuck.StuckSince == nil || !gotStuck.StuckSince.Equal(stuckSince) { + t.Errorf("stuck run row stuck_since = %v, want %v", gotStuck.StuckSince, stuckSince) + } + + if len(resp.SilentSuccesses.Rows) != 1 { + t.Fatalf("silent success rows = %d, want 1", len(resp.SilentSuccesses.Rows)) + } + if got := resp.SilentSuccesses.Rows[0]; got.RunID != "run-2" { + t.Errorf("silent success row = %+v, want run-2", got) + } +} + +func assertDegradedReason(t *testing.T, err error, wantReason string) { + t.Helper() + if err == nil { + t.Fatal("expected an error, got nil") + } + var degraded *LoopHealthDegradedError + if !errors.As(err, °raded) { + t.Fatalf("error = %v, want *LoopHealthDegradedError", err) + } + if degraded.Reason != wantReason { + t.Errorf("reason = %q, want %q", degraded.Reason, wantReason) + } +} diff --git a/apps/backend/internal/office/dashboard/service.go b/apps/backend/internal/office/dashboard/service.go index dfb0351835..1e2a3bdcea 100644 --- a/apps/backend/internal/office/dashboard/service.go +++ b/apps/backend/internal/office/dashboard/service.go @@ -12,6 +12,7 @@ import ( "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/routing" "github.com/kandev/kandev/internal/office/shared" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" taskmodels "github.com/kandev/kandev/internal/task/models" workflowmodels "github.com/kandev/kandev/internal/workflow/models" v1 "github.com/kandev/kandev/pkg/api/v1" @@ -99,8 +100,12 @@ type Repository interface { GetTaskWorkflowStepID(ctx context.Context, taskID string) (string, error) // CancelDisplacedParticipantRun cancels the run(s) the step-entry // fan-out queued for agentProfileID at (taskID, stepID). Used after a - // claim displaces an agent from a role. - CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error) + // claim displaces an agent from a role. Returns the rows actually + // cancelled so the caller can record each one's loop-liveness + // terminal shape. + CancelDisplacedParticipantRun( + ctx context.Context, taskID, stepID, agentProfileID string, + ) ([]runssqlite.CancelledRun, error) IsTaskWorkflowStepTerminal(ctx context.Context, taskID string) (terminal, hasStep bool, err error) } @@ -152,6 +157,17 @@ type RetryCanceller interface { CancelPendingRetriesForTask(ctx context.Context, taskID string) error } +// TerminalShapeRecorder classifies and counts a loop-liveness terminal +// shape (office_loop_terminal_total, REQ-OFFICE-LOOP-LIVENESS-005) for +// runs the dashboard package cancelled directly. Mirrors the RetryCanceller +// / TaskCanceller pattern above: the classification logic (activation +// instant, workspace resolution) is stateful and already lives once on +// *office/service.Service, so this seam reuses it instead of a second, +// divergence-prone copy in dashboard. +type TerminalShapeRecorder interface { + RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun) +} + // ProjectBudgetEvaluator evaluates project-scoped budget policies for a // destination project. Wired to costs.CostService.EvaluateProjectBudget so // reassigning a task into a project re-checks that project's policies — @@ -376,34 +392,35 @@ type WorkspaceSettings struct { // DashboardService provides dashboard, inbox, activity, run, and task-search business logic. type DashboardService struct { - repo Repository - logger *logger.Logger - activity shared.ActivityLogger - agents shared.AgentReader - costs shared.CostChecker - permissions shared.PermissionLister // optional; nil means no permission items in inbox - settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable - governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable - eb bus.EventBus // optional; nil means no events are published - retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign - taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled - taskDetacher TaskDetacher // optional; canonical empty-parent mutation - taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated - sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal - reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations - engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine - approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs - skillLister SkillLister // optional; nil means skill_count is always 0 - routineLister RoutineLister // optional; nil means routine_count is always 0 - failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries - failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources - markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint - decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints - routingProvider RoutingProvider // optional; nil disables /routing endpoints (503) - attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses - runResolver RunResolver // optional; nil means status-change activity rows have no run_id - assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization - projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies + repo Repository + logger *logger.Logger + activity shared.ActivityLogger + agents shared.AgentReader + costs shared.CostChecker + permissions shared.PermissionLister // optional; nil means no permission items in inbox + settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable + governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable + eb bus.EventBus // optional; nil means no events are published + retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign + terminalShapeRecorder TerminalShapeRecorder // optional; nil means displaced-run cancellations aren't counted in office_loop_terminal_total + taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled + taskDetacher TaskDetacher // optional; canonical empty-parent mutation + taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated + sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal + reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations + engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine + approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs + skillLister SkillLister // optional; nil means skill_count is always 0 + routineLister RoutineLister // optional; nil means routine_count is always 0 + failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries + failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources + markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint + decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints + routingProvider RoutingProvider // optional; nil disables /routing endpoints (503) + attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses + runResolver RunResolver // optional; nil means status-change activity rows have no run_id + assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization + projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies // officeSessionIdentity gates RecordAgentDecision's use of the caller's // own session id. Defaults false (zero value); set via // SetOfficeSessionIdentity, wired from features.officeSessionIdentity. @@ -516,6 +533,13 @@ func (s *DashboardService) SetRetryCanceller(c RetryCanceller) { s.retryCanceller = c } +// SetTerminalShapeRecorder wires the seam used to count a loop-liveness +// terminal shape for a run this package cancelled directly (a displaced +// participant's queued run). +func (s *DashboardService) SetTerminalShapeRecorder(r TerminalShapeRecorder) { + s.terminalShapeRecorder = r +} + // SetRunResolver wires the seam used to attribute status-change activity // rows back to the office run that produced them. Optional; when unset, // rows are logged with an empty run_id. diff --git a/apps/backend/internal/office/dashboard/service_tasks.go b/apps/backend/internal/office/dashboard/service_tasks.go index 4cb7c32749..5b821ec342 100644 --- a/apps/backend/internal/office/dashboard/service_tasks.go +++ b/apps/backend/internal/office/dashboard/service_tasks.go @@ -501,12 +501,17 @@ func (s *DashboardService) cancelDisplacedRun(ctx context.Context, taskID, stepI if displacedAgentID == "" { return } - if _, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID); err != nil { + cancelled, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID) + if err != nil { s.logger.Warn("cancel displaced participant run failed", zap.String("task_id", taskID), zap.String("step_id", stepID), zap.String("agent_profile_id", displacedAgentID), zap.Error(err)) + return + } + if s.terminalShapeRecorder != nil { + s.terminalShapeRecorder.RecordCancelledRunTerminalShapes(ctx, cancelled) } } diff --git a/apps/backend/internal/office/dashboard/terminal_shape_wiring_test.go b/apps/backend/internal/office/dashboard/terminal_shape_wiring_test.go new file mode 100644 index 0000000000..d13c930d33 --- /dev/null +++ b/apps/backend/internal/office/dashboard/terminal_shape_wiring_test.go @@ -0,0 +1,139 @@ +package dashboard_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/kandev/kandev/internal/office/dashboard" + "github.com/kandev/kandev/internal/office/models" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" +) + +// recordingTerminalShapeRecorder captures RecordCancelledRunTerminalShapes +// calls so tests can verify a displaced-participant cancellation reaches +// office_loop_terminal_total's classification path. +type recordingTerminalShapeRecorder struct { + calls [][]runssqlite.CancelledRun +} + +func (r *recordingTerminalShapeRecorder) RecordCancelledRunTerminalShapes( + _ context.Context, cancelled []runssqlite.CancelledRun, +) { + r.calls = append(r.calls, cancelled) +} + +var _ dashboard.TerminalShapeRecorder = (*recordingTerminalShapeRecorder)(nil) + +// TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape is +// the Testing round 3 regression test. cancelDisplacedRun calls +// repo.CancelDisplacedParticipantRun directly, which bypassed +// office_loop_terminal_total entirely — the same bypass class Review +// round 1 fixed for the single-run CancelRun call sites. The setup mirrors +// TestAddTaskReviewer_ClaimingAutoSeatCancelsDisplacedRunAndLogsActivity; +// this test additionally asserts the terminal-shape recorder is invoked +// with the displaced run's cancelled row. +func TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape(t *testing.T) { + deps := newTestDeps(t) + recorder := &recordingTerminalShapeRecorder{} + deps.svc.SetTerminalShapeRecorder(recorder) + insertTestTask(t, deps.db, "claim-shape1", "ws-claim-shape", "C", "todo", 2) + stepID := "step-claim-shape1" + + if _, err := deps.db.Exec(` + INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at) + VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape', '', datetime('now'), datetime('now')) + `); err != nil { + t.Fatalf("seed claiming agent profile: %v", err) + } + + if _, err := deps.db.Exec(` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance) + VALUES ('auto-seat-claim-shape1', ?, 'claim-shape1', 'reviewer', 'agent-auto', 1, 0, 'auto') + `, stepID); err != nil { + t.Fatalf("seed auto seat: %v", err) + } + + run := &models.Run{ + AgentProfileID: "agent-auto", + Reason: "task_assigned", + Payload: `{"task_id":"claim-shape1","workflow_step_id":"` + stepID + `"}`, + Status: models.RunStatusQueued, + CoalescedCount: 1, + } + if err := deps.repo.CreateRun(context.Background(), run); err != nil { + t.Fatalf("seed displaced run: %v", err) + } + + body := `{"agent_profile_id":"agent-claiming"}` + req := httptest.NewRequest(http.MethodPost, + "/api/v1/office/tasks/claim-shape1/reviewers", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + deps.router.ServeHTTP(w, req) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String()) + } + + if len(recorder.calls) != 1 { + t.Fatalf("terminal shape recorder calls = %d, want 1", len(recorder.calls)) + } + if len(recorder.calls[0]) != 1 || recorder.calls[0][0].ID != run.ID { + t.Fatalf("recorded cancelled rows = %+v, want [%s]", recorder.calls[0], run.ID) + } +} + +// TestSetTerminalShapeRecorder_NilTolerated keeps the wiring optional: when +// no recorder is registered, the displaced-run cancellation itself must +// still succeed (dashboard tests run without the office service wired in). +func TestSetTerminalShapeRecorder_NilTolerated(t *testing.T) { + deps := newTestDeps(t) + insertTestTask(t, deps.db, "claim-shape-nil", "ws-claim-shape-nil", "C", "todo", 2) + stepID := "step-claim-shape-nil" + + if _, err := deps.db.Exec(` + INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at) + VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape-nil', '', datetime('now'), datetime('now')) + `); err != nil { + t.Fatalf("seed claiming agent profile: %v", err) + } + if _, err := deps.db.Exec(` + INSERT INTO workflow_step_participants + (id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance) + VALUES ('auto-seat-claim-shape-nil', ?, 'claim-shape-nil', 'reviewer', 'agent-auto', 1, 0, 'auto') + `, stepID); err != nil { + t.Fatalf("seed auto seat: %v", err) + } + run := &models.Run{ + AgentProfileID: "agent-auto", + Reason: "task_assigned", + Payload: `{"task_id":"claim-shape-nil","workflow_step_id":"` + stepID + `"}`, + Status: models.RunStatusQueued, + CoalescedCount: 1, + } + if err := deps.repo.CreateRun(context.Background(), run); err != nil { + t.Fatalf("seed displaced run: %v", err) + } + + body := `{"agent_profile_id":"agent-claiming"}` + req := httptest.NewRequest(http.MethodPost, + "/api/v1/office/tasks/claim-shape-nil/reviewers", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + deps.router.ServeHTTP(w, req) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String()) + } + gotRun, err := deps.repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get displaced run: %v", err) + } + if gotRun.Status != "cancelled" { + t.Fatalf("displaced run status = %q, want cancelled", gotRun.Status) + } +} diff --git a/apps/backend/internal/office/infra/reconcile_test.go b/apps/backend/internal/office/infra/reconcile_test.go index 8ba58181b7..bd2d207db0 100644 --- a/apps/backend/internal/office/infra/reconcile_test.go +++ b/apps/backend/internal/office/infra/reconcile_test.go @@ -134,7 +134,7 @@ func TestReconcileAgentWorkingStatus_ClearsTerminalOwner(t *testing.T) { if _, err := repo.MarkAgentWorking(ctx, agent.ID, claimed.ID); err != nil { t.Fatalf("mark agent working: %v", err) } - if err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil { + if _, err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil { t.Fatalf("finish run: %v", err) } diff --git a/apps/backend/internal/office/models/models.go b/apps/backend/internal/office/models/models.go index 8ba7175238..8aa2101791 100644 --- a/apps/backend/internal/office/models/models.go +++ b/apps/backend/internal/office/models/models.go @@ -485,6 +485,13 @@ type Run struct { // EarliestRetryAt is the earliest moment a parked run should be re- // resolved. Set only when at least one degraded route is auto-retryable. EarliestRetryAt *time.Time `json:"earliest_retry_at,omitempty" db:"earliest_retry_at"` + + // CausationID is copied from the agent_wakeup_requests row that + // created this run (REQ-OFFICE-LOOP-LIVENESS-002). "" means + // uncorrelated — either a legacy pre-migration row or a run created + // off a wake that never carried an id. Never a join/group-by key + // without excluding "" first. + CausationID string `json:"causation_id,omitempty" db:"causation_id"` } // RouteAttempt records one provider attempt inside a Run. Each fallback @@ -598,6 +605,11 @@ type RoutineRun struct { StartedAt *time.Time `json:"started_at" db:"started_at"` CompletedAt *time.Time `json:"completed_at" db:"completed_at"` CreatedAt time.Time `json:"created_at" db:"created_at"` + // CausationID is minted once per fire in dispatchRoutineRun + // (REQ-OFFICE-LOOP-LIVENESS-002) — the origin id every wakeup + // request and run this fire produces carries forward. "" only for + // rows written before this feature. + CausationID string `json:"causation_id,omitempty" db:"causation_id"` } // ApprovalType constants for approval request types. diff --git a/apps/backend/internal/office/repository/sqlite/agent_working_status_test.go b/apps/backend/internal/office/repository/sqlite/agent_working_status_test.go index 847dcc30ce..a9f1a61e20 100644 --- a/apps/backend/internal/office/repository/sqlite/agent_working_status_test.go +++ b/apps/backend/internal/office/repository/sqlite/agent_working_status_test.go @@ -226,7 +226,7 @@ func TestMarkAgentWorking_SuccessorTakeover(t *testing.T) { // Run A leaves 'claimed' (finishes) but its own clearAgentWorking has not // run yet — the un-transacted window handleAgentCompleted leaves open. - if err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil { + if _, err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil { t.Fatalf("finish run A: %v", err) } diff --git a/apps/backend/internal/office/repository/sqlite/base.go b/apps/backend/internal/office/repository/sqlite/base.go index a850fc9ded..fefc4cb2d8 100644 --- a/apps/backend/internal/office/repository/sqlite/base.go +++ b/apps/backend/internal/office/repository/sqlite/base.go @@ -127,6 +127,7 @@ func (r *Repository) initSchema() error { return fmt.Errorf("required office migration: %w", err) } r.activateRunOutcome() + r.activateLoopLiveness() return nil } @@ -413,6 +414,9 @@ func (r *Repository) createRunTables() error { -- re-deriving it against a context_snapshot a coalesced wakeup -- may have since patched. continuation_scope TEXT NOT NULL DEFAULT '', + -- causation_id (office-loop-liveness): copied from the wakeup + -- request that created this run. '' means uncorrelated. + causation_id TEXT NOT NULL DEFAULT '', requested_at TIMESTAMP NOT NULL, claimed_at TIMESTAMP, finished_at TIMESTAMP @@ -492,6 +496,7 @@ func (r *Repository) createRoutineTables() error { started_at TIMESTAMP, completed_at TIMESTAMP, created_at TIMESTAMP NOT NULL, + causation_id TEXT NOT NULL DEFAULT '', FOREIGN KEY (routine_id) REFERENCES office_routines(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_office_routine_runs_active_fingerprint @@ -733,7 +738,8 @@ func (r *Repository) createAgentWakeupRequestTable() error { run_id TEXT NOT NULL DEFAULT '', requested_at TIMESTAMP NOT NULL, claimed_at TIMESTAMP, - finished_at TIMESTAMP + finished_at TIMESTAMP, + causation_id TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_wakeup_agent_status ON agent_wakeup_requests(agent_profile_id, status); CREATE UNIQUE INDEX IF NOT EXISTS idx_wakeup_idempotency ON agent_wakeup_requests(idempotency_key) diff --git a/apps/backend/internal/office/repository/sqlite/base_migrations.go b/apps/backend/internal/office/repository/sqlite/base_migrations.go index 2e1638d730..42253d7a0f 100644 --- a/apps/backend/internal/office/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/office/repository/sqlite/base_migrations.go @@ -62,6 +62,7 @@ func (r *Repository) runMigrations() error { r.migrateParentWakeReceiptColumns() r.migrate.Apply("task_workspace_groups.ownership_generation", `ALTER TABLE task_workspace_groups ADD COLUMN ownership_generation INTEGER NOT NULL DEFAULT 1`) + r.migrateLoopLivenessCausationID() if err := r.migrate.Err(); err != nil { return err } @@ -83,6 +84,35 @@ func (r *Repository) backfillRoutineTriggerTimezones() error { return nil } +// migrateLoopLivenessCausationID adds the causation_id correlation +// column to all three rows in the wake-to-run chain +// (REQ-OFFICE-LOOP-LIVENESS-002): the routine fire, the wakeup request +// it produces, and the run the wakeup request produces. "" rather than +// NULL, because every existing row predates this feature and a reader +// must treat "legacy" and "uncorrelated" identically. Each column gets +// its own partial index (excluding "") so a backward correlation walk +// from a run to its originating fire is a direct lookup on every +// dialect, not a table scan — NFR-2 is symmetric with the forward walk. +func (r *Repository) migrateLoopLivenessCausationID() { + _ = r.migrate.Apply("office_routine_runs.causation_id", + `ALTER TABLE office_routine_runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`) + _ = r.migrate.Apply("idx_office_routine_runs_causation_id", + `CREATE INDEX IF NOT EXISTS idx_office_routine_runs_causation_id + ON office_routine_runs(causation_id) WHERE causation_id != ''`) + + _ = r.migrate.Apply("agent_wakeup_requests.causation_id", + `ALTER TABLE agent_wakeup_requests ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`) + _ = r.migrate.Apply("idx_agent_wakeup_requests_causation_id", + `CREATE INDEX IF NOT EXISTS idx_agent_wakeup_requests_causation_id + ON agent_wakeup_requests(causation_id) WHERE causation_id != ''`) + + _ = r.migrate.Apply("runs.causation_id", + `ALTER TABLE runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`) + _ = r.migrate.Apply("idx_runs_causation_id", + `CREATE INDEX IF NOT EXISTS idx_runs_causation_id + ON runs(causation_id) WHERE causation_id != ''`) +} + // migrateContinuationScope adds runs.continuation_scope for databases // created before WO-16's claim-time scope persistence. Existing rows receive // a scope from their stored context snapshot so queued or claimed taskless diff --git a/apps/backend/internal/office/repository/sqlite/failure.go b/apps/backend/internal/office/repository/sqlite/failure.go index cda1966c60..eeb4810609 100644 --- a/apps/backend/internal/office/repository/sqlite/failure.go +++ b/apps/backend/internal/office/repository/sqlite/failure.go @@ -266,20 +266,33 @@ func (r *Repository) SetWorkspaceAgentFailureThreshold( return err } -// MarkRunFailed sets a run row to status=failed with a verbatim -// error message and stamps finished_at. Idempotent: re-marking is a -// no-op when the run is already in the same state. +// MarkRunFailed sets a run row to status=failed with a verbatim error +// message and stamps finished_at. Guarded to status = 'claimed' +// (Review round 3, R3-1): every caller reaches this after reading a +// claimed run, so without the guard a cancel that commits in between +// would have its 'cancelled' status and finished_at overwritten by +// this write. Returns wrote=false — the same "nothing changed" signal +// runs/repository/sqlite.FinishRun gives its callers — when the row +// was no longer claimed, whether because it was already marked failed +// or because it reached a different terminal state first. func (r *Repository) MarkRunFailed( ctx context.Context, runID, errorMessage string, -) error { - _, err := r.db.ExecContext(ctx, r.db.Rebind(` +) (bool, error) { + res, err := r.db.ExecContext(ctx, r.db.Rebind(` UPDATE runs SET status = 'failed', error_message = ?, finished_at = COALESCE(finished_at, ?) - WHERE id = ? + WHERE id = ? AND status = 'claimed' `), errorMessage, time.Now().UTC(), runID) - return err + if err != nil { + return false, err + } + n, err := res.RowsAffected() + if err != nil { + return false, err + } + return n > 0, nil } // DismissInboxItem records a per-user dismissal. Idempotent — a second diff --git a/apps/backend/internal/office/repository/sqlite/failure_test.go b/apps/backend/internal/office/repository/sqlite/failure_test.go index 489b494108..13d156b672 100644 --- a/apps/backend/internal/office/repository/sqlite/failure_test.go +++ b/apps/backend/internal/office/repository/sqlite/failure_test.go @@ -225,9 +225,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) { "2026-01-02 09:00:00", "2026-01-02 10:00:00", "2026-01-02 11:00:00", "{}") seedSummaryRun(t, repo, "mr-other", "agent-a", "queued", "2026-01-03 09:00:00", "", "", "{}") - if err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded"); err != nil { + wrote, err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded") + if err != nil { t.Fatalf("MarkRunFailed: %v", err) } + if !wrote { + t.Error("wrote = false, want true (mr-open was claimed)") + } run, err := repo.GetRun(ctx, "mr-open") if err != nil { t.Fatalf("GetRun: %v", err) @@ -242,10 +246,21 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) { t.Error("FinishedAt is nil, want it stamped") } - // Re-marking is idempotent and must not move an existing finished_at. - if err := repo.MarkRunFailed(ctx, "mr-done", "later failure"); err != nil { + // Review round 3, R3-1: MarkRunFailed is guarded to status = 'claimed', + // so a run already in a different terminal state (here, 'finished') is + // left untouched — a guarded no-op, not a re-stamp. This is a + // deliberate change from the pre-guard "idempotent re-mark" contract: + // unconditionally re-marking an already-terminal run is exactly the + // TOCTOU bug this guard exists to close (a concurrent cancel/finish + // must never have its real terminal state overwritten by a stale + // failure write). + wrote, err = repo.MarkRunFailed(ctx, "mr-done", "later failure") + if err != nil { t.Fatalf("MarkRunFailed (already finished): %v", err) } + if wrote { + t.Error("wrote = true, want false (mr-done was already finished, not claimed)") + } run, err = repo.GetRun(ctx, "mr-done") if err != nil { t.Fatalf("GetRun: %v", err) @@ -253,8 +268,11 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) { if run.FinishedAt == nil || run.FinishedAt.Format("2006-01-02 15:04:05") != "2026-01-02 11:00:00" { t.Errorf("FinishedAt = %v, want the original 2026-01-02 11:00:00", run.FinishedAt) } - if run.ErrorMessage != "later failure" { - t.Errorf("ErrorMessage = %q, want it overwritten", run.ErrorMessage) + if run.ErrorMessage != "" { + t.Errorf("ErrorMessage = %q, want it left untouched (guarded no-op)", run.ErrorMessage) + } + if run.Status != "finished" { + t.Errorf("Status = %q, want finished (unchanged)", run.Status) } // An untouched run keeps its status. @@ -266,9 +284,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) { } // Marking a missing run is a silent no-op, not an error. - if err := repo.MarkRunFailed(ctx, "no-such-run", "x"); err != nil { + wrote, err = repo.MarkRunFailed(ctx, "no-such-run", "x") + if err != nil { t.Errorf("MarkRunFailed(missing) = %v, want nil", err) } + if wrote { + t.Error("wrote = true, want false (no such run)") + } } func TestInboxDismissals_UpsertQueryAndList(t *testing.T) { diff --git a/apps/backend/internal/office/repository/sqlite/loop_health.go b/apps/backend/internal/office/repository/sqlite/loop_health.go new file mode 100644 index 0000000000..9f298826ef --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health.go @@ -0,0 +1,301 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/kandev/kandev/internal/db/dialect" +) + +// TriggerHealthRow is one overdue or stranded cron trigger +// (REQ-OFFICE-LOOP-LIVENESS-004). Condition names which predicate +// matched so a single evidence list can carry both kinds (AC-004.7). +type TriggerHealthRow struct { + TriggerID string `db:"id"` + RoutineID string `db:"routine_id"` + RoutineName string `db:"routine_name"` + NextRunAt *time.Time `db:"next_run_at"` + LastFiredAt *time.Time `db:"last_fired_at"` + Condition string `db:"-"` +} + +// CountEligibleTriggers reports how many enabled cron triggers exist +// for the workspace, regardless of due state — the input to the +// not_armed verdict (AC-004.6): zero means no trigger is scheduled at +// all, not that none happen to be due right now. +func (r *Repository) CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error) { + var count int + err := r.ro.GetContext(ctx, &count, r.ro.Rebind(` + SELECT COUNT(*) FROM office_routine_triggers t + JOIN office_routines rt ON rt.id = t.routine_id + WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ? + `), workspaceID) + return count, err +} + +// ListOverdueOrStrandedTriggers returns eligible cron triggers that are +// overdue (armed but past grace) or stranded (claimed, or never fired +// at all, and not re-armed past grace), capped at limit, with the +// untruncated total from the same statement (AC-004.9, AC-004.12). The +// stranded clock is the claim instant COALESCE(last_fired_at, +// created_at), not last_fired_at alone: a trigger that has never fired +// (e.g. a malformed cron_expression that never got a next_run_at) +// ages from its own creation, not from the epoch. A trigger claimed +// less than strandedGrace ago is a claiming trigger — excluded here +// rather than given a third label (AC-004.15). +func (r *Repository) ListOverdueOrStrandedTriggers( + ctx context.Context, workspaceID string, now time.Time, + overdueGrace, strandedGrace time.Duration, limit int, +) ([]TriggerHealthRow, int, error) { + overdueBefore := now.Add(-overdueGrace) + strandedBefore := now.Add(-strandedGrace) + + type row struct { + TriggerHealthRow + Total int `db:"total"` + } + var rows []row + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT t.id AS id, t.routine_id AS routine_id, rt.name AS routine_name, + t.next_run_at AS next_run_at, t.last_fired_at AS last_fired_at, + COUNT(*) OVER () AS total + FROM office_routine_triggers t + JOIN office_routines rt ON rt.id = t.routine_id + WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ? + AND ( + (t.next_run_at IS NOT NULL AND t.next_run_at <= ?) + OR (t.next_run_at IS NULL AND COALESCE(t.last_fired_at, t.created_at) <= ?) + ) + ORDER BY (t.next_run_at IS NULL) DESC, t.next_run_at ASC, t.id ASC + LIMIT ? + `), workspaceID, overdueBefore, strandedBefore, limit) + if err != nil { + return nil, 0, err + } + out := make([]TriggerHealthRow, 0, len(rows)) + total := 0 + for _, rr := range rows { + hr := rr.TriggerHealthRow + if hr.NextRunAt == nil { + hr.Condition = "stranded" + } else { + hr.Condition = "overdue" + } + out = append(out, hr) + total = rr.Total + } + return out, total, nil +} + +// StuckRunRow is one queued or claimed run the scheduler should have +// progressed but has not (REQ-OFFICE-LOOP-LIVENESS-004). StuckSince is +// the clock the row aged from: claimed_at for a claimed run, +// COALESCE(scheduled_retry_at, requested_at) for a queued one. +type StuckRunRow struct { + RunID string `db:"id"` + AgentProfileID string `db:"agent_profile_id"` + Status string `db:"status"` + StuckSince *time.Time `db:"stuck_since"` + Condition string `db:"-"` +} + +// stuckRunsUnionSQL is the shared claimed+queued candidate set behind +// ListStuckRuns' stuck_since column, unchanged by dialect: both source +// expressions (`claimed_at`, `COALESCE(scheduled_retry_at, requested_at)`) +// are plain TIMESTAMP-typed columns on both engines. +const stuckRunsUnionSQL = ` + SELECT r.id, r.agent_profile_id, r.status, r.claimed_at AS stuck_since + FROM runs r + JOIN agent_profiles a ON a.id = r.agent_profile_id + WHERE a.workspace_id = ? AND r.status = 'claimed' + AND (r.claimed_at IS NULL OR r.claimed_at <= ?) + UNION ALL + SELECT r.id, r.agent_profile_id, r.status, + COALESCE(r.scheduled_retry_at, r.requested_at) AS stuck_since + FROM runs r + JOIN agent_profiles a ON a.id = r.agent_profile_id + WHERE a.workspace_id = ? AND r.status = 'queued' + AND r.current_route_attempt_seq = 0 + AND r.routing_blocked_status IS NULL + AND COALESCE(r.scheduled_retry_at, r.requested_at) <= ? + AND NOT EXISTS ( + SELECT 1 FROM runs s + WHERE s.agent_profile_id = r.agent_profile_id AND s.status = 'claimed' + ) +` + +// stuckSinceColumnExpr and stuckSinceLayout keep stuck_since portable +// across dialects. stuck_since is a computed UNION ALL column, so it +// carries no declared type for the driver to auto-parse into +// time.Time (unlike a direct table column read): on SQLite, strftime() +// normalizes it to a fixed ISO-8601 UTC string that the scan target +// (sql.NullString) parses by hand below. Postgres has no strftime; +// selecting the raw TIMESTAMP column there returns a native time.Time +// that database/sql's Scan already formats into an RFC3339Nano string +// on assignment to *sql.NullString, so one scan type and one dialect +// switch cover both engines without a second query or row type. +func stuckSinceColumnExpr(driver string) string { + if dialect.IsPostgres(driver) { + return "w.stuck_since" + } + return "strftime('%Y-%m-%dT%H:%M:%fZ', w.stuck_since)" +} + +func stuckSinceLayout(driver string) string { + if dialect.IsPostgres(driver) { + return time.RFC3339Nano + } + return "2006-01-02T15:04:05.000Z" +} + +// ListStuckRuns returns claimed-stuck and queued-stuck runs in one +// list, each row naming which condition it matched, ordered claimed +// first then by stuck instant (NULL first), capped with the same- +// statement untruncated total (AC-004.9, AC-004.12). Neither predicate +// is windowed (AC-004.5): a run stuck longer than +// officeLoopEvaluationWindow is the one most worth surfacing. +// +// The queued half mirrors ClaimNextEligibleRun's own filter +// (AC-004.17) as an accepted limitation: it restricts to +// current_route_attempt_seq = 0, so a routed run that has dispatched +// at least once and requeued reads as silent rather than stuck. That +// gap is intentional (OPERATOR DECISION F29/F30) — fixing it here +// would let this read disagree with what the scheduler would actually +// claim next. +func (r *Repository) ListStuckRuns( + ctx context.Context, workspaceID string, now time.Time, + queuedGrace, claimedGrace time.Duration, limit int, +) ([]StuckRunRow, int, error) { + claimedBefore := now.Add(-claimedGrace) + queuedBefore := now.Add(-queuedGrace) + driver := r.ro.DriverName() + + type row struct { + ID string `db:"id"` + AgentProfileID string `db:"agent_profile_id"` + Status string `db:"status"` + StuckSince sql.NullString `db:"stuck_since"` + Total int `db:"total"` + } + var rows []row + query := fmt.Sprintf(` + SELECT w.id AS id, w.agent_profile_id AS agent_profile_id, w.status AS status, + %s AS stuck_since, + COUNT(*) OVER () AS total + FROM (%s) w + ORDER BY (w.status = 'claimed') DESC, (w.stuck_since IS NULL) DESC, w.stuck_since ASC, w.id ASC + LIMIT ? + `, stuckSinceColumnExpr(driver), stuckRunsUnionSQL) + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(query), + workspaceID, claimedBefore, workspaceID, queuedBefore, limit) + if err != nil { + return nil, 0, err + } + layout := stuckSinceLayout(driver) + out := make([]StuckRunRow, 0, len(rows)) + total := 0 + for _, rr := range rows { + hr := StuckRunRow{RunID: rr.ID, AgentProfileID: rr.AgentProfileID, Status: rr.Status} + if rr.StuckSince.Valid { + parsed, perr := time.Parse(layout, rr.StuckSince.String) + if perr != nil { + return nil, 0, fmt.Errorf("parse stuck_since %q: %w", rr.StuckSince.String, perr) + } + hr.StuckSince = &parsed + } + if hr.Status == "claimed" { + hr.Condition = "claimed_stuck" + } else { + hr.Condition = "queued_stuck" + } + out = append(out, hr) + total = rr.Total + } + return out, total, nil +} + +// SilentSuccessRow is one finished run that asserted success but never +// named a session (AC-005.4 made an evidence row). +type SilentSuccessRow struct { + RunID string `db:"id"` + RequestedAt time.Time `db:"requested_at"` + FinishedAt *time.Time `db:"finished_at"` +} + +// ListSilentSuccesses returns finished/processed/sessionless runs whose +// COALESCE(finished_at, requested_at) falls in [windowStart, now), +// requested at or after the activation instant (AC-005.6: a run +// predating activation is pre_activation, never a silent success), +// newest first, capped with the same-statement untruncated total +// (AC-004.7, AC-004.9, AC-004.16). +func (r *Repository) ListSilentSuccesses( + ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int, +) ([]SilentSuccessRow, int, error) { + type row struct { + SilentSuccessRow + Total int `db:"total"` + } + var rows []row + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT r.id AS id, r.requested_at AS requested_at, r.finished_at AS finished_at, + COUNT(*) OVER () AS total + FROM runs r + JOIN agent_profiles a ON a.id = r.agent_profile_id + WHERE a.workspace_id = ? AND r.status = 'finished' AND r.outcome = 'processed' + AND (r.session_id IS NULL OR r.session_id = '') + AND r.requested_at >= ? + AND COALESCE(r.finished_at, r.requested_at) >= ? + ORDER BY COALESCE(r.finished_at, r.requested_at) DESC, r.id ASC + LIMIT ? + `), workspaceID, activationAt, windowStart, limit) + if err != nil { + return nil, 0, err + } + out := make([]SilentSuccessRow, 0, len(rows)) + total := 0 + for _, rr := range rows { + out = append(out, rr.SilentSuccessRow) + total = rr.Total + } + return out, total, nil +} + +// TerminalRunShapeInputRow carries exactly the columns +// service.ClassifyTerminalRun needs, for every terminal run (not +// queued, not claimed) in the evaluation window (AC-004.16). The +// caller classifies and buckets; this method reads no free text +// (AC-005.7) and is side-effect free. +type TerminalRunShapeInputRow struct { + Status string `db:"status"` + Outcome *string `db:"outcome"` + SessionID string `db:"session_id"` + RequestedAt time.Time `db:"requested_at"` + FinishedAt *time.Time `db:"finished_at"` +} + +// ListTerminalRunsInWindow returns every terminal run for the +// workspace whose COALESCE(finished_at, requested_at) falls in +// [windowStart, now). Unbounded by officeLoopEvidenceCap: this feeds +// the terminal-shape counts, not an evidence list. +func (r *Repository) ListTerminalRunsInWindow( + ctx context.Context, workspaceID string, windowStart time.Time, +) ([]TerminalRunShapeInputRow, error) { + var rows []TerminalRunShapeInputRow + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT r.status AS status, r.outcome AS outcome, r.session_id AS session_id, + r.requested_at AS requested_at, r.finished_at AS finished_at + FROM runs r + JOIN agent_profiles a ON a.id = r.agent_profile_id + WHERE a.workspace_id = ? AND r.status NOT IN ('queued', 'claimed') + AND COALESCE(r.finished_at, r.requested_at) >= ? + `), workspaceID, windowStart) + if err != nil { + return nil, err + } + if rows == nil { + rows = []TerminalRunShapeInputRow{} + } + return rows, nil +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_health_never_fired_trigger_test.go b/apps/backend/internal/office/repository/sqlite/loop_health_never_fired_trigger_test.go new file mode 100644 index 0000000000..72f89f7034 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health_never_fired_trigger_test.go @@ -0,0 +1,86 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +// AC-004.15 / Claim-instant terminology (Review round 3, R3-3): a +// trigger that has never fired (last_fired_at IS NULL, e.g. a +// malformed cron_expression that never got a next_run_at) is still +// claiming, not stranded, until strandedGrace has elapsed since its +// own created_at — the spec's claim instant is +// COALESCE(last_fired_at, created_at), not last_fired_at alone. The +// old predicate treated any never-fired trigger as immediately +// eligible for stranded, regardless of how recently it was created. +func TestListOverdueOrStrandedTriggers_NeverFiredTriggerNotStrandedBeforeCreatedGrace(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + routine := seedCronTrigger(t, repo, "ws-a", "NeverFired") + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "", + Timezone: "UTC", Enabled: true, NextRunAt: nil, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + // Created 1 minute before now: well inside the 5-minute stranded + // grace, never fired, never armed with a next_run_at. + createdAt := now.Add(-1 * time.Minute) + if _, err := repo.ExecRaw(ctx, + `UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`, + createdAt, trigger.ID); err != nil { + t.Fatalf("seed never-fired state: %v", err) + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50, + ) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 || len(rows) != 0 { + t.Fatalf("got %d rows, total %d, want 0/0 (a trigger created 1m ago is still claiming, not stranded)", len(rows), total) + } +} + +func TestListOverdueOrStrandedTriggers_NeverFiredTriggerStrandedAfterCreatedGrace(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + routine := seedCronTrigger(t, repo, "ws-a", "NeverFiredStale") + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "", + Timezone: "UTC", Enabled: true, NextRunAt: nil, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + // Created 6 minutes before now: past the 5-minute stranded grace, + // never fired, never armed. + createdAt := now.Add(-6 * time.Minute) + if _, err := repo.ExecRaw(ctx, + `UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`, + createdAt, trigger.ID); err != nil { + t.Fatalf("seed never-fired stale state: %v", err) + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50, + ) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1 (a trigger created 6m ago that never fired must be stranded)", len(rows), total) + } + if rows[0].Condition != "stranded" { + t.Errorf("condition = %q, want stranded", rows[0].Condition) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_health_silent_success_test.go b/apps/backend/internal/office/repository/sqlite/loop_health_silent_success_test.go new file mode 100644 index 0000000000..ef9f5fcbdf --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health_silent_success_test.go @@ -0,0 +1,169 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" +) + +func seedFinishedRun( + t *testing.T, repo *sqlite.Repository, db *sqlx.DB, + agentID, outcome, sessionID string, requestedAt time.Time, finishedAt *time.Time, +) string { + t.Helper() + ctx := context.Background() + run := &models.Run{ + AgentProfileID: agentID, Reason: "task_assigned", Payload: "{}", + Status: "queued", CoalescedCount: 1, SessionID: sessionID, + } + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + if _, err := db.Exec( + `UPDATE runs SET status = 'finished', outcome = ?, requested_at = ?, finished_at = ? WHERE id = ?`, + outcome, requestedAt, finishedAt, run.ID, + ); err != nil { + t.Fatalf("finish run: %v", err) + } + return run.ID +} + +func TestListSilentSuccesses_ReturnsFinishedProcessedSessionlessRun(t *testing.T) { + repo, db := newTestRepoWithDB(t) + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + finishedAt := now.Add(-time.Hour) + seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt) + + windowStart := now.Add(-24 * time.Hour) + activationAt := now.Add(-48 * time.Hour) + rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total) + } +} + +// AC-005.6, applied to the AC-004.5/.7 evidence list: a legacy run +// requested before the activation instant is pre_activation, never a +// silent success, even when it is otherwise indistinguishable from one +// and falls inside the evaluation window. +func TestListSilentSuccesses_PreActivationRunExcluded(t *testing.T) { + repo, db := newTestRepoWithDB(t) + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + finishedAt := now.Add(-time.Hour) + seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt) + + windowStart := now.Add(-24 * time.Hour) + activationAt := now.Add(-time.Minute) // activated after this run was requested + _, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (run predates activation, must not read as silent success)", total) + } +} + +// AC-004.16: a terminal run with a NULL finished_at still counts +// within the window, aged from requested_at via COALESCE. +func TestListSilentSuccesses_NullFinishedAtStillInsideWindow(t *testing.T) { + repo, db := newTestRepoWithDB(t) + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-time.Hour), nil) + + windowStart := now.Add(-24 * time.Hour) + activationAt := now.Add(-48 * time.Hour) + rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1 (NULL finished_at falls back to requested_at)", len(rows), total) + } + if rows[0].FinishedAt != nil { + t.Errorf("FinishedAt = %v, want nil", rows[0].FinishedAt) + } +} + +func TestListSilentSuccesses_LaunchedRunIsNotSilent(t *testing.T) { + repo, db := newTestRepoWithDB(t) + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + finishedAt := now.Add(-time.Hour) + seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt) + + windowStart := now.Add(-24 * time.Hour) + activationAt := now.Add(-48 * time.Hour) + _, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (a run with a session id is not silent)", total) + } +} + +func TestListSilentSuccesses_OutsideWindowExcluded(t *testing.T) { + repo, db := newTestRepoWithDB(t) + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + finishedAt := now.Add(-30 * time.Hour) + seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-31*time.Hour), &finishedAt) + + windowStart := now.Add(-24 * time.Hour) + activationAt := now.Add(-48 * time.Hour) + _, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (outside the 24h evaluation window)", total) + } +} + +func TestListTerminalRunsInWindow_ExcludesQueuedAndClaimed(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, queued); err != nil { + t.Fatalf("create queued: %v", err) + } + claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, claimed); err != nil { + t.Fatalf("create claimed: %v", err) + } + if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, claimed.ID); err != nil { + t.Fatalf("claim: %v", err) + } + finishedAt := now.Add(-time.Hour) + seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt) + + rows, err := repo.ListTerminalRunsInWindow(ctx, "ws-a", now.Add(-24*time.Hour)) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1 (queued/claimed excluded)", len(rows)) + } + if rows[0].Status != "finished" { + t.Errorf("status = %q, want finished", rows[0].Status) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_postgres_test.go b/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_postgres_test.go new file mode 100644 index 0000000000..402d0a96f9 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_postgres_test.go @@ -0,0 +1,105 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresListStuckRuns is the PostgreSQL twin of the ListStuckRuns cases +// in loop_health_stuck_runs_test.go. ListStuckRuns' stuck_since column is +// computed by a UNION ALL over two different source columns, which loses its +// declared type on read — the original fix normalized it with SQLite-only +// strftime(), a function Postgres does not have, so this exercised the same +// query shape (a claimed-stuck and a queued-stuck row, one list, ordered) end +// to end against a real Postgres backend. Skips unless +// KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresListStuckRuns(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("init settings store: %v", err) + } + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init office repo: %v", err) + } + + ctx := context.Background() + // Postgres timestamp columns store microsecond precision; truncate + // before the round-trip so the later Equal() comparisons aren't + // comparing against nanosecond digits the column never persisted. + now := time.Now().UTC().Truncate(time.Microsecond) + + // Two distinct agent profiles: sharing one would put the queued run + // behind a claimed sibling on the same agent_profile_id, excluding + // it under AC-OFFICE-LOOP-LIVENESS-004.17 (a busy agent's steady + // state, not a stall) — this test wants both rows reported. + // agent_profiles.agent_id carries a foreign key to agents(id), + // enforced on Postgres (not on SQLite's default connection). + for _, agentID := range []string{"agent-a", "agent-b"} { + if _, err := db.ExecContext(ctx, db.Rebind( + `INSERT INTO agents (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)`, + ), agentID, agentID, now, now); err != nil { + t.Fatalf("seed agent %s: %v", agentID, err) + } + if _, err := db.ExecContext(ctx, db.Rebind(` + INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `), agentID, agentID, agentID, agentID, "ws-pg-stuck", "developer", now, now); err != nil { + t.Fatalf("seed agent profile %s: %v", agentID, err) + } + } + + claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, claimed); err != nil { + t.Fatalf("create claimed run: %v", err) + } + longAgo := now.Add(-72 * time.Hour) + if _, err := db.ExecContext(ctx, db.Rebind( + `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, + ), longAgo, claimed.ID); err != nil { + t.Fatalf("seed claimed state: %v", err) + } + + queued := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, queued); err != nil { + t.Fatalf("create queued run: %v", err) + } + old := now.Add(-10 * time.Minute) + if _, err := db.ExecContext(ctx, db.Rebind( + `UPDATE runs SET requested_at = ? WHERE id = ?`, + ), old, queued.ID); err != nil { + t.Fatalf("backdate requested_at: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-pg-stuck", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("ListStuckRuns: %v", err) + } + if total != 2 || len(rows) != 2 { + t.Fatalf("got %d rows, total %d, want 2/2", len(rows), total) + } + // Claimed-stuck outranks queued-stuck regardless of relative age + // (loop_health.go's ORDER BY (status = 'claimed') DESC ...). + if rows[0].RunID != claimed.ID || rows[0].Condition != "claimed_stuck" { + t.Errorf("rows[0] = %+v, want claimed_stuck for %s", rows[0], claimed.ID) + } + if rows[0].StuckSince == nil || !rows[0].StuckSince.Equal(longAgo) { + t.Errorf("rows[0].StuckSince = %v, want %v", rows[0].StuckSince, longAgo) + } + if rows[1].RunID != queued.ID || rows[1].Condition != "queued_stuck" { + t.Errorf("rows[1] = %+v, want queued_stuck for %s", rows[1], queued.ID) + } + if rows[1].StuckSince == nil || !rows[1].StuckSince.Equal(old) { + t.Errorf("rows[1].StuckSince = %v, want %v", rows[1].StuckSince, old) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_test.go b/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_test.go new file mode 100644 index 0000000000..fda13a78f8 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health_stuck_runs_test.go @@ -0,0 +1,284 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +func TestListStuckRuns_ClaimedStuckReportedRegardlessOfAge(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + // Claimed a very long time ago — the predicate is not windowed + // (AC-004.5): a run stuck longer than the evaluation window is + // still reported. + longAgo := now.Add(-72 * time.Hour) + if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, longAgo, run.ID); err != nil { + t.Fatalf("seed claimed state: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total) + } + if rows[0].Condition != "claimed_stuck" { + t.Errorf("condition = %q, want claimed_stuck", rows[0].Condition) + } +} + +func TestListStuckRuns_QueuedStuckPositiveControl(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + old := now.Add(-10 * time.Minute) + if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil { + t.Fatalf("backdate requested_at: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1 (positive control must be reported)", len(rows), total) + } + if rows[0].Condition != "queued_stuck" { + t.Errorf("condition = %q, want queued_stuck", rows[0].Condition) + } +} + +// AC-004.17 exclusion 1: a future scheduled_retry_at is a deferral, +// not a stall, even though requested_at is old — the COALESCE handles +// this without a separate branch. +func TestListStuckRuns_ExcludesFutureScheduledRetry(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + old := now.Add(-10 * time.Minute) + future := now.Add(5 * time.Minute) + if _, err := db.Exec( + `UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`, + old, future, run.ID, + ); err != nil { + t.Fatalf("seed future retry: %v", err) + } + + _, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (future scheduled_retry_at is a deferral)", total) + } +} + +// AC-004.17 exclusion 2: a non-NULL routing_blocked_status is a parked +// run — parked for a human, not stuck. +func TestListStuckRuns_ExcludesRoutingBlockedRun(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + old := now.Add(-10 * time.Minute) + if _, err := db.Exec( + `UPDATE runs SET requested_at = ?, routing_blocked_status = 'waiting_for_provider_capacity' WHERE id = ?`, + old, run.ID, + ); err != nil { + t.Fatalf("seed routing blocked: %v", err) + } + + _, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (routing-blocked run is parked, not stuck)", total) + } +} + +// AC-004.17 exclusion 3: a claimed sibling on the same agent_profile_id +// means the queue is correctly holding this run back — a busy agent's +// steady state, not a stall. +func TestListStuckRuns_ExcludesRunWithClaimedSibling(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + sibling := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, sibling); err != nil { + t.Fatalf("create sibling: %v", err) + } + if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, sibling.ID); err != nil { + t.Fatalf("claim sibling: %v", err) + } + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + old := now.Add(-10 * time.Minute) + if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil { + t.Fatalf("backdate requested_at: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + // The claimed sibling itself is not "stuck" either (claimed_at is + // fresh), so the workspace should report nothing at all. + if total != 0 || len(rows) != 0 { + t.Fatalf("got %d rows, total %d, want 0/0 (busy agent is not a stall)", len(rows), total) + } +} + +// AC-004.17 aging: a run whose scheduled_retry_at has JUST passed is +// aged from that instant, not from requested_at. +func TestListStuckRuns_AgesFromScheduledRetryAtOncePassed(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + // requested_at is recent (would not itself be stuck), but + // scheduled_retry_at passed well over the grace period ago. + recent := now.Add(-30 * time.Second) + pastRetry := now.Add(-3 * time.Minute) + if _, err := db.Exec( + `UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`, + recent, pastRetry, run.ID, + ); err != nil { + t.Fatalf("seed past retry: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1 (aged from scheduled_retry_at)", len(rows), total) + } +} + +// AC-004.17 (F29/F30 accepted limitation): a queued run that has +// dispatched at least once (current_route_attempt_seq > 0) reads as +// silent rather than stuck, mirroring ClaimNextEligibleRun's own gap +// rather than fixing it here. +func TestListStuckRuns_ExcludesRunWithNonZeroRouteAttemptSeq(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + + run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + old := now.Add(-10 * time.Minute) + if _, err := db.Exec( + `UPDATE runs SET requested_at = ?, current_route_attempt_seq = 1 WHERE id = ?`, + old, run.ID, + ); err != nil { + t.Fatalf("seed routed attempt: %v", err) + } + + _, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 { + t.Fatalf("total = %d, want 0 (accepted limitation: a dispatched-once run reads as silent)", total) + } +} + +// Ordering: a claimed-stuck run outranks an older queued-stuck one, +// and a NULL claimed_at row sorts first among claimed rows. +func TestListStuckRuns_ClaimedOutranksOlderQueuedAndNullClaimedAtSortsFirst(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + seedAgentProfile(t, db, "agent-a", "ws-a") + seedAgentProfile(t, db, "agent-b", "ws-a") + seedAgentProfile(t, db, "agent-c", "ws-a") + + // Oldest queued-stuck run. + queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, queued); err != nil { + t.Fatalf("create queued: %v", err) + } + veryOld := now.Add(-60 * time.Minute) + if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, veryOld, queued.ID); err != nil { + t.Fatalf("backdate queued: %v", err) + } + + // Claimed-stuck run with a recorded claimed_at, younger than the + // queued one but still past the claimed grace. + claimedWithAge := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, claimedWithAge); err != nil { + t.Fatalf("create claimed: %v", err) + } + claimedAt := now.Add(-15 * time.Minute) + if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, claimedAt, claimedWithAge.ID); err != nil { + t.Fatalf("claim: %v", err) + } + + // Claimed-stuck run with NULL claimed_at. + claimedNull := &models.Run{AgentProfileID: "agent-c", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1} + if err := repo.CreateRun(ctx, claimedNull); err != nil { + t.Fatalf("create claimed-null: %v", err) + } + if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = NULL WHERE id = ?`, claimedNull.ID); err != nil { + t.Fatalf("claim with null claimed_at: %v", err) + } + + rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 3 || len(rows) != 3 { + t.Fatalf("got %d rows, total %d, want 3/3", len(rows), total) + } + if rows[0].RunID != claimedNull.ID { + t.Errorf("rows[0] = %s, want the NULL-claimed_at row first", rows[0].RunID) + } + if rows[1].RunID != claimedWithAge.ID { + t.Errorf("rows[1] = %s, want the aged claimed row second", rows[1].RunID) + } + if rows[2].RunID != queued.ID { + t.Errorf("rows[2] = %s, want the queued row last despite being oldest", rows[2].RunID) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_health_triggers_test.go b/apps/backend/internal/office/repository/sqlite/loop_health_triggers_test.go new file mode 100644 index 0000000000..ef8423a8b5 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_health_triggers_test.go @@ -0,0 +1,228 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +// seedCronTrigger creates a routine and one cron trigger for it, +// returning the trigger id. next_run_at/last_fired_at are set with a +// raw UPDATE afterward since CreateRoutineTrigger only accepts a +// caller-supplied NextRunAt at creation and never a NULL/backdated +// LastFiredAt. +func seedCronTrigger(t *testing.T, repo interface { + CreateRoutine(ctx context.Context, r *models.Routine) error + CreateRoutineTrigger(ctx context.Context, tr *models.RoutineTrigger) error +}, workspaceID, name string) *models.Routine { + t.Helper() + routine := &models.Routine{ + WorkspaceID: workspaceID, + Name: name, + TaskTemplate: "", + Status: "active", + ConcurrencyPolicy: "always_create", + } + if err := repo.CreateRoutine(context.Background(), routine); err != nil { + t.Fatalf("create routine: %v", err) + } + return routine +} + +func TestListOverdueOrStrandedTriggers_ReturnsOverdueArmedTrigger(t *testing.T) { + repo, db := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + routine := seedCronTrigger(t, repo, "ws-a", "Overdue") + overdueAt := now.Add(-4 * time.Minute) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50, + ) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total) + } + if rows[0].Condition != "overdue" { + t.Errorf("condition = %q, want overdue", rows[0].Condition) + } + if rows[0].RoutineName != "Overdue" { + t.Errorf("routine_name = %q, want Overdue", rows[0].RoutineName) + } + _ = db +} + +func TestListOverdueOrStrandedTriggers_ReturnsStrandedClaimedTrigger(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + routine := seedCronTrigger(t, repo, "ws-a", "Stranded") + future := now.Add(time.Minute) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &future, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + // Claim it: next_run_at -> NULL, last_fired_at -> 6 minutes ago, + // past the 5-minute stranded grace. + claimedAt := now.Add(-6 * time.Minute) + if _, err := repo.ExecRaw(ctx, + `UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`, + claimedAt, trigger.ID); err != nil { + t.Fatalf("seed stranded state: %v", err) + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50, + ) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 1 || len(rows) != 1 { + t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total) + } + if rows[0].Condition != "stranded" { + t.Errorf("condition = %q, want stranded", rows[0].Condition) + } +} + +// AC-004.15: a trigger claimed less than strandedGrace ago is a +// claiming trigger — excluded from the evidence list entirely, not +// given a third label. +func TestListOverdueOrStrandedTriggers_ExcludesClaimingTrigger(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + routine := seedCronTrigger(t, repo, "ws-a", "Claiming") + future := now.Add(time.Minute) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &future, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger: %v", err) + } + claimedAt := now.Add(-1 * time.Minute) + if _, err := repo.ExecRaw(ctx, + `UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`, + claimedAt, trigger.ID); err != nil { + t.Fatalf("seed claiming state: %v", err) + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers( + ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50, + ) + if err != nil { + t.Fatalf("list: %v", err) + } + if total != 0 || len(rows) != 0 { + t.Fatalf("got %d rows, total %d, want 0/0 (claiming trigger must be excluded)", len(rows), total) + } + + eligible, err := repo.CountEligibleTriggers(ctx, "ws-a") + if err != nil { + t.Fatalf("count eligible: %v", err) + } + if eligible != 1 { + t.Errorf("eligible count = %d, want 1 (still an eligible trigger, just not overdue/stranded)", eligible) + } +} + +// Ordering: stranded (NULL next_run_at) sorts before overdue. +func TestListOverdueOrStrandedTriggers_StrandedSortsBeforeOverdue(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + overdueRoutine := seedCronTrigger(t, repo, "ws-a", "Overdue") + overdueAt := now.Add(-10 * time.Minute) + overdueTrigger := &models.RoutineTrigger{ + RoutineID: overdueRoutine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt, + } + if err := repo.CreateRoutineTrigger(ctx, overdueTrigger); err != nil { + t.Fatalf("create overdue trigger: %v", err) + } + + strandedRoutine := seedCronTrigger(t, repo, "ws-a", "Stranded") + future := now.Add(time.Minute) + strandedTrigger := &models.RoutineTrigger{ + RoutineID: strandedRoutine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &future, + } + if err := repo.CreateRoutineTrigger(ctx, strandedTrigger); err != nil { + t.Fatalf("create stranded trigger: %v", err) + } + if _, err := repo.ExecRaw(ctx, + `UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`, + now.Add(-10*time.Minute), strandedTrigger.ID); err != nil { + t.Fatalf("seed stranded state: %v", err) + } + + rows, _, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(rows) != 2 { + t.Fatalf("got %d rows, want 2", len(rows)) + } + if rows[0].Condition != "stranded" || rows[1].Condition != "overdue" { + t.Errorf("order = [%s, %s], want [stranded, overdue]", rows[0].Condition, rows[1].Condition) + } +} + +func TestListOverdueOrStrandedTriggers_CapsAndReportsTotal(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + ctx := context.Background() + now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + + for i := 0; i < 3; i++ { + routine := seedCronTrigger(t, repo, "ws-a", "R") + overdueAt := now.Add(-10 * time.Minute) + trigger := &models.RoutineTrigger{ + RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *", + Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt, + } + if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil { + t.Fatalf("create trigger %d: %v", i, err) + } + } + + rows, total, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 2) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(rows) != 2 { + t.Fatalf("got %d rows, want capped at 2", len(rows)) + } + if total != 3 { + t.Fatalf("total = %d, want 3 (untruncated)", total) + } +} + +func TestCountEligibleTriggers_ZeroForEmptyWorkspace(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + count, err := repo.CountEligibleTriggers(context.Background(), "ws-empty") + if err != nil { + t.Fatalf("count: %v", err) + } + if count != 0 { + t.Errorf("count = %d, want 0", count) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_liveness_activation.go b/apps/backend/internal/office/repository/sqlite/loop_liveness_activation.go new file mode 100644 index 0000000000..85d8822f3f --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_liveness_activation.go @@ -0,0 +1,89 @@ +package sqlite + +import ( + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/persistence" +) + +// loopLivenessActivationKey is the kandev_meta key published once a +// boot has verified all three causation_id columns exist +// (REQ-OFFICE-LOOP-LIVENESS-002/005). See docs/specs/office/system-design/ +// loop-liveness.md, "Activation". +const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at" + +// causationIDColumn is the column all three activation probes check. +const causationIDColumn = "causation_id" + +// activateLoopLiveness writes loopLivenessActivationKey only after a +// positive probe confirms all three causation_id columns exist. Mirrors +// activateRunOutcome: the migration runner swallows failures at WARN, +// so this probe is what stands between a failed migration and a reader +// wrongly believing correlation and terminal-shape classification are +// live. WriteMetaKeyIfAbsent makes the write replay-safe — the instant +// is never overwritten once set, because every run requested before it +// must classify as pre_activation forever, not just until the next +// boot. Any failure here is logged and swallowed — activation is +// best-effort published fact, never a boot-blocking requirement. +func (r *Repository) activateLoopLiveness() { + if err := persistence.EnsureMetaTable(r.db); err != nil { + r.logActivationWarn("ensure kandev_meta table failed", err) + return + } + for _, probe := range []struct{ table, column string }{ + {"office_routine_runs", causationIDColumn}, + {"agent_wakeup_requests", causationIDColumn}, + {"runs", causationIDColumn}, + } { + exists, err := columnExists(r.db, probe.table, probe.column) + if err != nil { + r.logActivationWarn("loop liveness schema probe failed", err) + return + } + if !exists { + return + } + } + if _, err := persistence.WriteMetaKeyIfAbsent( + r.db, loopLivenessActivationKey, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + r.logActivationWarn("write loop liveness activation key failed", err) + } +} + +// LoopLivenessActivation returns the published activation instant and +// whether it has been published at all. A run requested before the +// instant, or read when activation was never published, must classify +// as pre_activation (AC-005.6) rather than risk a false silent_success. +// Read failures are swallowed here (logged only) because every existing +// caller uses this for a safe-default classification, not a decision +// that must fail loud; the /loop-health read that must distinguish +// "unpublished" from "unreadable" uses LoopLivenessActivationChecked. +func (r *Repository) LoopLivenessActivation() (at time.Time, published bool) { + at, published, err := r.LoopLivenessActivationChecked() + if err != nil && r.log != nil { + r.log.Warn("read loop liveness activation key failed", zap.Error(err)) + } + return at, published +} + +// LoopLivenessActivationChecked is LoopLivenessActivation with the read +// or parse error surfaced instead of swallowed, for a caller that must +// fail loud (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.10) rather than let a +// broken kandev_meta read silently render as "not yet activated". +func (r *Repository) LoopLivenessActivationChecked() (at time.Time, published bool, err error) { + raw, err := persistence.ReadMetaKey(r.db, loopLivenessActivationKey) + if err != nil { + return time.Time{}, false, err + } + if raw == "" { + return time.Time{}, false, nil + } + parsed, err := time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, false, err + } + return parsed, true, nil +} diff --git a/apps/backend/internal/office/repository/sqlite/loop_liveness_migration_test.go b/apps/backend/internal/office/repository/sqlite/loop_liveness_migration_test.go new file mode 100644 index 0000000000..1deec3ed68 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/loop_liveness_migration_test.go @@ -0,0 +1,185 @@ +package sqlite_test + +import ( + "database/sql" + "strings" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/persistence" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at" + +// TestLoopLivenessMigration_AddsCausationIDColumns covers +// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: all three +// causation_id columns exist and default to ” (not NULL) on a +// freshly-migrated database. +func TestLoopLivenessMigration_AddsCausationIDColumns(t *testing.T) { + _, db := newTestRepoWithDB(t) + + for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} { + var notnull int + var dflt sql.NullString + rows, err := db.Queryx("PRAGMA table_info(" + table + ")") + if err != nil { + t.Fatalf("table_info(%s): %v", table, err) + } + found := false + for rows.Next() { + var cid, nn, pk int + var name, ctype string + var d sql.NullString + if err := rows.Scan(&cid, &name, &ctype, &nn, &d, &pk); err != nil { + t.Fatalf("scan table_info(%s): %v", table, err) + } + if name == "causation_id" { + found = true + notnull = nn + dflt = d + } + } + _ = rows.Close() + if !found { + t.Fatalf("%s.causation_id column missing", table) + } + if notnull != 1 { + t.Fatalf("%s.causation_id NOT NULL = %d, want 1", table, notnull) + } + if !dflt.Valid || dflt.String != "''" { + t.Fatalf("%s.causation_id default = %v, want ''", table, dflt) + } + } +} + +// TestLoopLivenessMigration_CausationIDIndexesArePartial covers +// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: each +// causation_id index exists and is partial (WHERE causation_id != +// ”), not a plain index over every row — a full index would defeat +// the point of excluding the common empty-string case from the +// lookup path. sqlite_master.sql is the only way to see the stored +// WHERE predicate; PRAGMA index_list/index_info do not expose it. +func TestLoopLivenessMigration_CausationIDIndexesArePartial(t *testing.T) { + _, db := newTestRepoWithDB(t) + + cases := []struct { + table string + indexName string + }{ + {"office_routine_runs", "idx_office_routine_runs_causation_id"}, + {"agent_wakeup_requests", "idx_agent_wakeup_requests_causation_id"}, + {"runs", "idx_runs_causation_id"}, + } + for _, tc := range cases { + var stored sql.NullString + err := db.QueryRow( + `SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?`, tc.indexName, + ).Scan(&stored) + if err != nil { + t.Fatalf("%s: look up index %s: %v", tc.table, tc.indexName, err) + } + if !stored.Valid || stored.String == "" { + t.Fatalf("%s: index %s missing or has no stored definition", tc.table, tc.indexName) + } + if !strings.Contains(stored.String, "WHERE") || !strings.Contains(stored.String, "causation_id != ''") { + t.Fatalf("%s: index %s definition = %q, want a WHERE causation_id != '' partial predicate", + tc.table, tc.indexName, stored.String) + } + } +} + +// TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe mirrors +// TestRunOutcomeActivation_WrittenOnceAfterSchemaProbe: the key is +// written only after a positive probe of all three columns, and a +// replayed boot never re-writes it. +func TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe(t *testing.T) { + _, db := newTestRepoWithDB(t) + + val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey) + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected telemetry.office_loop_liveness.activated_at to be written after first boot") + } + if _, err := time.Parse(time.RFC3339, val); err != nil { + t.Fatalf("activation value %q is not RFC3339: %v", val, err) + } + + if _, err := db.Exec(db.Rebind( + `UPDATE kandev_meta SET value = ? WHERE key = ?`, + ), "sentinel-value", loopLivenessActivationKey); err != nil { + t.Fatalf("seed sentinel: %v", err) + } + + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay init: %v", err) + } + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay init twice: %v", err) + } + + got, err := persistence.ReadMetaKey(db, loopLivenessActivationKey) + if err != nil { + t.Fatalf("read activation key after replay: %v", err) + } + if got != "sentinel-value" { + t.Fatalf("activation key = %q after replay, want unchanged sentinel-value", got) + } +} + +// TestLoopLivenessActivation_ReadHelperRoundTrips covers +// Repository.LoopLivenessActivation: published=true with the parsed +// instant after boot, and published=false when the key is absent. +func TestLoopLivenessActivation_ReadHelperRoundTrips(t *testing.T) { + repo, _ := newTestRepoWithDB(t) + + at, published := repo.LoopLivenessActivation() + if !published { + t.Fatal("expected activation to be published after boot") + } + if at.IsZero() { + t.Fatal("expected a non-zero activation instant") + } +} + +// TestPostgresLoopLivenessMigration_AddsColumnsAndActivates is the +// PostgreSQL twin required by ADR 0027. Skips unless +// KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresLoopLivenessMigration_AddsColumnsAndActivates(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init postgres schema: %v", err) + } + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay postgres schema: %v", err) + } + + for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} { + var dataType string + err := db.QueryRow(` + SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND column_name = 'causation_id' + `, table).Scan(&dataType) + if err != nil { + t.Fatalf("inspect %s.causation_id: %v", table, err) + } + if dataType != "text" { + t.Fatalf("%s.causation_id data type = %q, want text", table, dataType) + } + } + + val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey) + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected activation key to be written on postgres") + } +} diff --git a/apps/backend/internal/office/repository/sqlite/mark_run_failed_postgres_test.go b/apps/backend/internal/office/repository/sqlite/mark_run_failed_postgres_test.go new file mode 100644 index 0000000000..7721ae9ffd --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/mark_run_failed_postgres_test.go @@ -0,0 +1,129 @@ +package sqlite_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/kandev/kandev/internal/office/models" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun is +// the two-connection regression for Review round 3, R3-1: MarkRunFailed's +// write used to carry no status guard at all, so a run cancelled on +// another connection between HandleAgentFailure's read and this write +// would have its 'cancelled' status and real finished_at silently +// overwritten to 'failed'. Real Postgres row locking forces the +// interleaving this needs: one connection holds an uncommitted cancel on +// the row, MarkRunFailed's write blocks on that row lock, and only +// proceeds once the cancel commits — at which point the guard +// (status = 'claimed') must see the fresh 'cancelled' status and write +// nothing. Skips unless KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) { + dsn := testutil.PostgresDSNFromEnv(t) + db := testutil.OpenIsolatedPostgres(t, dsn) + + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init office repo: %v", err) + } + ctx := context.Background() + + var schema string + if err := db.Get(&schema, "SELECT current_schema()"); err != nil { + t.Fatalf("read isolated schema name: %v", err) + } + cancellerDB, err := sqlx.Open("pgx", dsn) + if err != nil { + t.Fatalf("open canceller connection: %v", err) + } + cancellerDB.SetMaxOpenConns(1) + cancellerDB.SetMaxIdleConns(1) + t.Cleanup(func() { _ = cancellerDB.Close() }) + if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil { + t.Fatalf("set canceller search_path: %v", err) + } + + run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + now := time.Now().UTC() + if _, err := db.ExecContext(ctx, db.Rebind( + `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, + ), now, run.ID); err != nil { + t.Fatalf("seed claimed: %v", err) + } + + tx, err := cancellerDB.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin canceller tx: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, cancellerDB.Rebind( + `UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`, + ), "too late", now, run.ID); err != nil { + t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err) + } + + var ( + wg sync.WaitGroup + wrote bool + markErr error + markDone = make(chan struct{}) + commitErr error + ) + wg.Add(1) + go func() { + defer wg.Done() + defer close(markDone) + wrote, markErr = repo.MarkRunFailed(ctx, run.ID, "provider exploded") + }() + + // Give MarkRunFailed's write time to reach Postgres and block on the + // row lock the canceller's uncommitted transaction holds, before that + // transaction commits. Without this window the two goroutines could + // run in either order and the test would not exercise the lock-wait + // path this bug depends on. + select { + case <-markDone: + t.Fatal("MarkRunFailed returned before the canceller committed; it never blocked on the row lock") + case <-time.After(200 * time.Millisecond): + } + + commitErr = tx.Commit() + wg.Wait() + if commitErr != nil { + t.Fatalf("canceller commit: %v", commitErr) + } + if markErr != nil { + t.Fatalf("mark run failed: %v", markErr) + } + + if wrote { + t.Fatal("wrote = true, want false: a run that committed cancelled must not be reported failed") + } + if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" { + t.Fatalf("run status = %q, want cancelled (the concurrent fail must not have overwritten it)", got) + } +} + +func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string { + t.Helper() + var status string + if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil { + t.Fatalf("get run status: %v", err) + } + return status +} diff --git a/apps/backend/internal/office/repository/sqlite/participant_provenance_test.go b/apps/backend/internal/office/repository/sqlite/participant_provenance_test.go index f9d9a583da..8aa268b9c3 100644 --- a/apps/backend/internal/office/repository/sqlite/participant_provenance_test.go +++ b/apps/backend/internal/office/repository/sqlite/participant_provenance_test.go @@ -437,8 +437,11 @@ func TestCancelDisplacedParticipantRun_CancelsOnlyTheFanOutRunForThatAgentStep(t if err != nil { t.Fatalf("CancelDisplacedParticipantRun: %v", err) } - if cancelled != 1 { - t.Fatalf("cancelled = %d, want 1", cancelled) + if len(cancelled) != 1 { + t.Fatalf("cancelled = %d, want 1", len(cancelled)) + } + if cancelled[0].ID != "run-target" { + t.Fatalf("cancelled[0].ID = %q, want run-target", cancelled[0].ID) } assertRunStatus(t, repo, "run-target", "cancelled") @@ -460,8 +463,8 @@ func TestCancelDisplacedParticipantRun_NoMatchIsNotAFailure(t *testing.T) { if err != nil { t.Fatalf("CancelDisplacedParticipantRun: %v", err) } - if cancelled != 0 { - t.Errorf("cancelled = %d, want 0", cancelled) + if len(cancelled) != 0 { + t.Errorf("cancelled = %d, want 0", len(cancelled)) } } diff --git a/apps/backend/internal/office/repository/sqlite/participants.go b/apps/backend/internal/office/repository/sqlite/participants.go index 74d36cd5ee..de14e8e1b1 100644 --- a/apps/backend/internal/office/repository/sqlite/participants.go +++ b/apps/backend/internal/office/repository/sqlite/participants.go @@ -11,6 +11,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/kandev/kandev/internal/db/dialect" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" "github.com/kandev/kandev/internal/workflow/models" workflowrepo "github.com/kandev/kandev/internal/workflow/repository" ) @@ -672,9 +673,11 @@ const displacedRunCancelReason = "participant_seat_claimed" // SQLite-flavoured and the server dialect needs dialect.JSONExtract, which // is why this selector lives here rather than in the shared runs writer // (mirrors CancelRunsForTasks in tree_holds.go). -func (r *Repository) CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error) { +func (r *Repository) CancelDisplacedParticipantRun( + ctx context.Context, taskID, stepID, agentProfileID string, +) ([]runssqlite.CancelledRun, error) { if taskID == "" || stepID == "" || agentProfileID == "" { - return 0, nil + return nil, nil } driver := r.db.DriverName() selector := fmt.Sprintf( diff --git a/apps/backend/internal/office/repository/sqlite/routine_last_run_test.go b/apps/backend/internal/office/repository/sqlite/routine_last_run_test.go new file mode 100644 index 0000000000..72759ce5e6 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/routine_last_run_test.go @@ -0,0 +1,195 @@ +package sqlite_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +func createTestRoutineForLastRun(t *testing.T, repo interface { + CreateRoutine(ctx context.Context, routine *models.Routine) error +}) *models.Routine { + t.Helper() + routine := &models.Routine{ + WorkspaceID: "ws-1", + Name: "Last Run Test", + TaskTemplate: "{}", + Status: "active", + ConcurrencyPolicy: "skip_if_active", + Variables: "{}", + } + if err := repo.CreateRoutine(context.Background(), routine); err != nil { + t.Fatalf("create routine: %v", err) + } + return routine +} + +// AC-OFFICE-LOOP-LIVENESS-001.1/.3/.4/.6: TouchRoutineLastRun advances +// last_run_at from NULL, is monotonic against an equal or older instant +// (zero-row no-op, not an error), and moves forward for a newer instant. +func TestTouchRoutineLastRun_Monotonic(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + routine := createTestRoutineForLastRun(t, repo) + + first := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil { + t.Fatalf("touch (initial): %v", err) + } + got, err := repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.LastRunAt == nil || !got.LastRunAt.Equal(first) { + t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, first) + } + + // Older instant: no-op. + older := first.Add(-time.Hour) + if err := repo.TouchRoutineLastRun(ctx, routine.ID, older); err != nil { + t.Fatalf("touch (older): %v", err) + } + got, err = repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !got.LastRunAt.Equal(first) { + t.Fatalf("last_run_at moved backwards: got %v, want %v", got.LastRunAt, first) + } + + // Equal instant: no-op. + if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil { + t.Fatalf("touch (equal): %v", err) + } + got, err = repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !got.LastRunAt.Equal(first) { + t.Fatalf("last_run_at changed on equal instant: got %v, want %v", got.LastRunAt, first) + } + + // Newer instant: advances. + newer := first.Add(time.Hour) + if err := repo.TouchRoutineLastRun(ctx, routine.ID, newer); err != nil { + t.Fatalf("touch (newer): %v", err) + } + got, err = repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !got.LastRunAt.Equal(newer) { + t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, newer) + } +} + +// AC-OFFICE-LOOP-LIVENESS-001.6: a touch matching zero rows (unknown +// routine) is treated as success, not an error. +func TestTouchRoutineLastRun_UnknownRoutineIsNoop(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + + if err := repo.TouchRoutineLastRun(ctx, "does-not-exist", time.Now().UTC()); err != nil { + t.Fatalf("touch unknown routine: %v", err) + } +} + +// AC-OFFICE-LOOP-LIVENESS-001.5: the write touches only last_run_at (and +// updated_at) — every other column is left exactly as it was. +func TestTouchRoutineLastRun_TouchesOnlyLastRunAtAndUpdatedAt(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + routine := createTestRoutineForLastRun(t, repo) + beforeUpdatedAt := routine.UpdatedAt + + at := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + if err := repo.TouchRoutineLastRun(ctx, routine.ID, at); err != nil { + t.Fatalf("touch: %v", err) + } + + got, err := repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Name != routine.Name || got.Status != routine.Status || + got.ConcurrencyPolicy != routine.ConcurrencyPolicy || got.TaskTemplate != routine.TaskTemplate { + t.Fatalf("touch mutated a non-last_run_at column: %+v", got) + } + if !got.UpdatedAt.After(beforeUpdatedAt) { + t.Fatalf("updated_at did not advance: got %v, before %v", got.UpdatedAt, beforeUpdatedAt) + } +} + +// AC-OFFICE-LOOP-LIVENESS-001.8: UpdateRoutine no longer carries +// last_run_at — a caller holding a stale (or nil) snapshot cannot move +// the fresh instant this dispatch just wrote backwards or clear it. +func TestUpdateRoutine_DoesNotWriteLastRunAt(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + routine := createTestRoutineForLastRun(t, repo) + + fresh := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + if err := repo.TouchRoutineLastRun(ctx, routine.ID, fresh); err != nil { + t.Fatalf("touch: %v", err) + } + + // A caller holding a stale snapshot (older LastRunAt) must not move + // the stored instant backwards via an unrelated field update. + stale := fresh.Add(-24 * time.Hour) + routine.LastRunAt = &stale + routine.Name = "Renamed While Stale" + if err := repo.UpdateRoutine(ctx, routine); err != nil { + t.Fatalf("update (stale snapshot): %v", err) + } + got, err := repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) { + t.Fatalf("stale UpdateRoutine moved last_run_at: got %v, want %v", got.LastRunAt, fresh) + } + if got.Name != "Renamed While Stale" { + t.Fatalf("update did not apply other fields: name = %q", got.Name) + } + + // A caller that never populated LastRunAt (nil) must not clear it either. + routine.LastRunAt = nil + routine.Description = "Updated from nil snapshot" + if err := repo.UpdateRoutine(ctx, routine); err != nil { + t.Fatalf("update (nil snapshot): %v", err) + } + got, err = repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) { + t.Fatalf("nil-snapshot UpdateRoutine cleared last_run_at: got %v, want %v", got.LastRunAt, fresh) + } + if got.Description != "Updated from nil snapshot" { + t.Fatalf("update did not apply other fields: description = %q", got.Description) + } + + // A caller holding a snapshot newer than the stored instant must not + // advance it either: a stale-or-nil case alone cannot distinguish "the + // column is not written" from "the column is written monotonically", + // since both pass under a stale or nil snapshot. Only a newer snapshot + // tells them apart. + newer := fresh.Add(24 * time.Hour) + routine.LastRunAt = &newer + routine.Name = "Renamed From Newer Snapshot" + if err := repo.UpdateRoutine(ctx, routine); err != nil { + t.Fatalf("update (newer snapshot): %v", err) + } + got, err = repo.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) { + t.Fatalf("newer-snapshot UpdateRoutine advanced last_run_at: got %v, want %v", got.LastRunAt, fresh) + } + if got.Name != "Renamed From Newer Snapshot" { + t.Fatalf("update did not apply other fields: name = %q", got.Name) + } +} diff --git a/apps/backend/internal/office/repository/sqlite/routines.go b/apps/backend/internal/office/repository/sqlite/routines.go index 6cc98e9c83..b60d841d2f 100644 --- a/apps/backend/internal/office/repository/sqlite/routines.go +++ b/apps/backend/internal/office/repository/sqlite/routines.go @@ -196,7 +196,13 @@ func (r *Repository) ListRoutines(ctx context.Context, workspaceID string) ([]*m return routines, nil } -// UpdateRoutine updates an existing routine. +// UpdateRoutine updates an existing routine. It deliberately does not +// carry last_run_at (AC-OFFICE-LOOP-LIVENESS-001.8): that column only +// ever moves forward, written by TouchRoutineLastRun from a dispatch, and +// a caller here may be holding a routine loaded before a fire (or one +// that never populated the field) — writing it unconditionally would let +// that stale or nil snapshot silently move the value backwards or clear +// it. routine.LastRunAt stays on the struct for reads. func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine) error { routine.UpdatedAt = time.Now().UTC() _, err := r.db.ExecContext(ctx, r.db.Rebind(` @@ -204,12 +210,27 @@ func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine) name = ?, description = ?, task_template = ?, assignee_agent_profile_id = ?, status = ?, concurrency_policy = ?, catch_up_policy = ?, catch_up_max = ?, - variables = ?, last_run_at = ?, updated_at = ? + variables = ?, updated_at = ? WHERE id = ? `), routine.Name, routine.Description, routine.TaskTemplate, routine.AssigneeAgentProfileID, routine.Status, routine.ConcurrencyPolicy, routine.CatchUpPolicy, routine.CatchUpMax, - routine.Variables, routine.LastRunAt, routine.UpdatedAt, routine.ID) + routine.Variables, routine.UpdatedAt, routine.ID) + return err +} + +// TouchRoutineLastRun advances office_routines.last_run_at to at, +// monotonically: the write is a zero-row no-op, not an error, when the +// stored value is already at or after at (AC-OFFICE-LOOP-LIVENESS-001.3, +// .4, .6). Deliberately narrower than UpdateRoutine's eleven-column +// read-modify-write, which would turn a routine dispatch into a race +// against a concurrent UI or config-sync edit (AC-OFFICE-LOOP-LIVENESS-001.5). +func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error { + _, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE office_routines + SET last_run_at = ?, updated_at = ? + WHERE id = ? AND (last_run_at IS NULL OR last_run_at < ?) + `), at, time.Now().UTC(), routineID, at) return err } @@ -279,11 +300,11 @@ func (r *Repository) CreateRoutineRun(ctx context.Context, run *models.RoutineRu INSERT INTO office_routine_runs ( id, routine_id, trigger_id, source, status, trigger_payload, linked_task_id, coalesced_into_run_id, dispatch_fingerprint, - started_at, completed_at, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + started_at, completed_at, created_at, causation_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), run.ID, run.RoutineID, run.TriggerID, run.Source, run.Status, run.TriggerPayload, run.LinkedTaskID, run.CoalescedIntoRunID, - run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt) + run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt, run.CausationID) return err } @@ -404,18 +425,6 @@ func (r *Repository) GetTaskTerminalStatus(ctx context.Context, taskID string) ( } } -// TouchRoutineLastRun updates only last_run_at (+ updated_at) on a -// routine. Deliberately narrower than UpdateRoutine, whose whole-row -// snapshot write can revert concurrent edits to other columns (see -// UpdateRoutineConfigFields above) — a dispatch in flight should never -// clobber a config change made while it ran, or vice versa. -func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error { - _, err := r.db.ExecContext(ctx, r.db.Rebind(` - UPDATE office_routines SET last_run_at = ?, updated_at = ? WHERE id = ? - `), at, time.Now().UTC(), routineID) - return err -} - // UpdateRunStatus updates a run's status and optionally its linked task. func (r *Repository) UpdateRunStatus( ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string, diff --git a/apps/backend/internal/office/repository/sqlite/runs_cancel_test.go b/apps/backend/internal/office/repository/sqlite/runs_cancel_test.go index 57cd2ea93f..9bbe2b3acf 100644 --- a/apps/backend/internal/office/repository/sqlite/runs_cancel_test.go +++ b/apps/backend/internal/office/repository/sqlite/runs_cancel_test.go @@ -59,7 +59,7 @@ func TestCancelRun_QueuedRunIsCancelled(t *testing.T) { repo := newTestRepo(t) id := seedCancelRun(t, repo, "t1", "queued", nil, nil) - if err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil { + if _, err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil { t.Fatalf("cancel: %v", err) } @@ -79,7 +79,7 @@ func TestCancelRun_ClaimedRunIsCancelled(t *testing.T) { repo := newTestRepo(t) id := seedCancelRun(t, repo, "t1", "claimed", nil, nil) - if err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil { + if _, err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil { t.Fatalf("cancel: %v", err) } @@ -115,7 +115,7 @@ func TestCancelRun_TerminalRunIsUntouched(t *testing.T) { repo := newTestRepo(t) id := seedCancelRun(t, repo, "t1", tc.status, tc.cancelReason, &finished) - if err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil { + if _, err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil { t.Fatalf("cancel: %v", err) } @@ -155,16 +155,16 @@ func TestCancelRunsWhere_RowsAffected(t *testing.T) { if err != nil { t.Fatalf("cancel terminal: %v", err) } - if rows != 0 { - t.Errorf("rows affected = %d, want 0 for an already-terminal run", rows) + if len(rows) != 0 { + t.Errorf("rows affected = %d, want 0 for an already-terminal run", len(rows)) } rows, err = repo.CancelRunsWhere(ctx, "tree_cancelled", `id = ?`, queued) if err != nil { t.Fatalf("cancel queued: %v", err) } - if rows != 1 { - t.Errorf("rows affected = %d, want 1", rows) + if len(rows) != 1 { + t.Errorf("rows affected = %d, want 1", len(rows)) } } @@ -177,7 +177,7 @@ func TestBulkCancelRuns_OnlyCancelsEligibleRuns(t *testing.T) { claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil) done := seedCancelRun(t, repo, "t1", "finished", nil, &finished) - if err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil { + if _, err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil { t.Fatalf("bulk cancel: %v", err) } @@ -204,12 +204,12 @@ func TestCancelRunsForTasks_MixedSetCountMatches(t *testing.T) { claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil) done := seedCancelRun(t, repo, "t1", "finished", nil, &finished) - count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused") + cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused") if err != nil { t.Fatalf("cancel for tasks: %v", err) } - if count != 2 { - t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", count) + if len(cancelled) != 2 { + t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", len(cancelled)) } for _, id := range []string{queued, claimed} { if run := fetchRun(t, repo, id); run.Status != "cancelled" { @@ -229,12 +229,12 @@ func TestCancelRunsForTasks_MatchesPayloadTaskID(t *testing.T) { mine := seedCancelRun(t, repo, "t1", "queued", nil, nil) other := seedCancelRun(t, repo, "t2", "queued", nil, nil) - count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled") + cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled") if err != nil { t.Fatalf("cancel for tasks: %v", err) } - if count != 1 { - t.Fatalf("count = %d, want 1", count) + if len(cancelled) != 1 { + t.Fatalf("count = %d, want 1", len(cancelled)) } if run := fetchRun(t, repo, mine); run.Status != "cancelled" { t.Errorf("t1 run status = %q, want cancelled", run.Status) @@ -253,15 +253,15 @@ func TestCancelPaths_EmptySliceIsANoOp(t *testing.T) { ctx := context.Background() id := seedCancelRun(t, repo, "t1", "queued", nil, nil) - if err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil { + if _, err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil { t.Errorf("bulk cancel with no ids: %v", err) } - count, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled") + cancelled, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled") if err != nil { t.Errorf("cancel for tasks with no task ids: %v", err) } - if count != 0 { - t.Errorf("count = %d, want 0", count) + if len(cancelled) != 0 { + t.Errorf("count = %d, want 0", len(cancelled)) } if run := fetchRun(t, repo, id); run.Status != "queued" { t.Errorf("unrelated run status = %q, want queued", run.Status) diff --git a/apps/backend/internal/office/repository/sqlite/runs_test.go b/apps/backend/internal/office/repository/sqlite/runs_test.go index f6a0b1918e..1057bedbf6 100644 --- a/apps/backend/internal/office/repository/sqlite/runs_test.go +++ b/apps/backend/internal/office/repository/sqlite/runs_test.go @@ -235,14 +235,15 @@ func TestCleanExpired(t *testing.T) { AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", - Status: "queued", + Status: "claimed", CoalescedCount: 1, } if err := repo.CreateRun(ctx, req); err != nil { t.Fatalf("create: %v", err) } - // Finish it. - if err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil { + // Finish it. FinishRun is guarded to status = 'claimed' (Review round + // 3, R3-1), matching the seeded status above. + if _, err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil { t.Fatalf("finish: %v", err) } diff --git a/apps/backend/internal/office/repository/sqlite/tree_holds.go b/apps/backend/internal/office/repository/sqlite/tree_holds.go index 76fc20ea84..081a262a71 100644 --- a/apps/backend/internal/office/repository/sqlite/tree_holds.go +++ b/apps/backend/internal/office/repository/sqlite/tree_holds.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/kandev/kandev/internal/office/models" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" ) func (r *Repository) createTreeHoldTables() error { @@ -208,23 +209,22 @@ func (r *Repository) CountActiveRunsForTasks(ctx context.Context, taskIDs []stri } // CancelRunsForTasks cancels the queued or claimed runs belonging to the -// given tasks and returns how many rows were actually cancelled. +// given tasks and returns the rows actually cancelled, so a caller can +// classify and count each one's loop-liveness terminal shape. // // The terminal write itself lives in the runs repository's CancelRunsWhere, // which applies the queued/claimed guard; only the by-task-id selector stays // here. json_extract is SQLite-flavoured — Postgres is a supported driver // (internal/persistence/provider.go) and would need dialect.JSONExtract — so // keeping it local avoids baking dialect-specific SQL into the shared writer. -func (r *Repository) CancelRunsForTasks(ctx context.Context, taskIDs []string, reason string) (int, error) { +func (r *Repository) CancelRunsForTasks( + ctx context.Context, taskIDs []string, reason string, +) ([]runssqlite.CancelledRun, error) { if len(taskIDs) == 0 { - return 0, nil + return nil, nil } selector, args := taskIDInQuery(`json_extract(payload, '$.task_id') IN (%s)`, taskIDs) - rows, err := r.CancelRunsWhere(ctx, reason, selector, args...) - if err != nil { - return 0, err - } - return int(rows), nil + return r.CancelRunsWhere(ctx, reason, selector, args...) } func (r *Repository) BulkUpdateTaskState(ctx context.Context, taskIDs []string, state string) error { diff --git a/apps/backend/internal/office/repository/sqlite/wakeup_requests.go b/apps/backend/internal/office/repository/sqlite/wakeup_requests.go index 5eac5f8086..55749bcd22 100644 --- a/apps/backend/internal/office/repository/sqlite/wakeup_requests.go +++ b/apps/backend/internal/office/repository/sqlite/wakeup_requests.go @@ -49,6 +49,11 @@ type WakeupRequest struct { RequestedAt time.Time `db:"requested_at"` ClaimedAt sql.NullTime `db:"claimed_at"` FinishedAt sql.NullTime `db:"finished_at"` + // CausationID is copied from the routine fire that produced this + // wake, or minted here when the wake has no routine origin + // (REQ-OFFICE-LOOP-LIVENESS-002). "" for rows written before this + // feature or for a wake with no identifiable origin. + CausationID string `db:"causation_id"` } // CreateWakeupRequest inserts a new wakeup-request row. When @@ -85,12 +90,12 @@ func (r *Repository) CreateWakeupRequest(ctx context.Context, req *WakeupRequest INSERT INTO agent_wakeup_requests ( id, agent_profile_id, source, reason, payload, status, coalesced_count, idempotency_key, run_id, - requested_at, claimed_at, finished_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + requested_at, claimed_at, finished_at, causation_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), req.ID, req.AgentProfileID, req.Source, req.Reason, req.Payload, req.Status, req.CoalescedCount, req.IdempotencyKey, req.RunID, - req.RequestedAt, req.ClaimedAt, req.FinishedAt, + req.RequestedAt, req.ClaimedAt, req.FinishedAt, req.CausationID, ) if err != nil && isUniqueConstraintErr(err) { return fmt.Errorf("%w: %v", ErrWakeupIdempotencyConflict, err) @@ -105,7 +110,7 @@ func (r *Repository) GetWakeupRequest(ctx context.Context, id string) (*WakeupRe err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(` SELECT id, agent_profile_id, source, reason, payload, status, coalesced_count, idempotency_key, run_id, - requested_at, claimed_at, finished_at + requested_at, claimed_at, finished_at, causation_id FROM agent_wakeup_requests WHERE id = ? `), id).StructScan(&row) @@ -125,7 +130,7 @@ func (r *Repository) ListQueuedWakeupRequestsForAgent( err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` SELECT id, agent_profile_id, source, reason, payload, status, coalesced_count, idempotency_key, run_id, - requested_at, claimed_at, finished_at + requested_at, claimed_at, finished_at, causation_id FROM agent_wakeup_requests WHERE agent_profile_id = ? AND status = ? ORDER BY requested_at ASC diff --git a/apps/backend/internal/office/routines/causation_id_heavy_test.go b/apps/backend/internal/office/routines/causation_id_heavy_test.go new file mode 100644 index 0000000000..805f293b92 --- /dev/null +++ b/apps/backend/internal/office/routines/causation_id_heavy_test.go @@ -0,0 +1,134 @@ +package routines_test + +import ( + "context" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/routines" +) + +// newTestRoutineServiceWithTaskTables mirrors newTestRoutineService but +// also creates minimal tasks/task_sessions tables on the same database, +// so a heavy-path test can walk the real +// office_routine_runs.linked_task_id -> tasks.id -> task_sessions.task_id +// chain instead of only asserting the id fields in isolation. +func newTestRoutineServiceWithTaskTables(t *testing.T) (*routines.RoutineService, *sqlx.DB) { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '' + )`); err != nil { + t.Fatalf("create tasks table: %v", err) + } + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS task_sessions ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL + )`); err != nil { + t.Fatalf("create task_sessions table: %v", err) + } + + log := logger.Default() + return routines.NewRoutineService(repo, log, &noopActivity{}), db +} + +// dbBackedTaskCreator is CreateOfficeTaskInWorkflow's fake for this +// file only: unlike fakeTaskCreator (a synthetic id with nothing +// behind it), it inserts a real tasks row plus a task_sessions row +// naming it, so the produced id is actually reachable by a join — +// standing in for auto_start_agent launching the routine's task. +type dbBackedTaskCreator struct { + db *sqlx.DB + taskID string + sessID string + nextSeq int + captured struct{ workflowID string } +} + +func (f *dbBackedTaskCreator) CreateOfficeTaskInWorkflow( + _ context.Context, workspaceID, _, _, workflowID, title, _ string, +) (string, error) { + f.nextSeq++ + f.captured.workflowID = workflowID + f.taskID = "task-heavy-causation" + f.sessID = "session-heavy-causation" + if _, err := f.db.Exec(`INSERT INTO tasks (id, workspace_id, title) VALUES (?, ?, ?)`, + f.taskID, workspaceID, title); err != nil { + return "", err + } + if _, err := f.db.Exec(`INSERT INTO task_sessions (id, task_id) VALUES (?, ?)`, + f.sessID, f.taskID); err != nil { + return "", err + } + return f.taskID, nil +} + +// AC-OFFICE-LOOP-LIVENESS-002.9: a heavy routine fire's run row carries +// both the minted causation id and the linked task id, and that task id +// is a real, reachable row — walkable forward to the session the task +// launched, the correlation chain a loop-liveness diagnosis depends on. +func TestDispatchRoutineRun_HeavyPathCausationIDAndLinkedTaskReachSession(t *testing.T) { + svc, db := newTestRoutineServiceWithTaskTables(t) + ctx := context.Background() + + routine := &models.Routine{ + WorkspaceID: "ws-1", + Name: "Heavy Causation", + TaskTemplate: `{"title":"Heavy {{name}}","description":"D"}`, + AssigneeAgentProfileID: "agent-heavy", + Status: "active", + ConcurrencyPolicy: "always_create", + Variables: `{"name":{"default":"routine"}}`, + } + if err := svc.CreateRoutine(ctx, routine); err != nil { + t.Fatalf("create routine: %v", err) + } + + wf := &fakeWorkflowEnsurer{} + tc := &dbBackedTaskCreator{db: db} + svc.SetWorkflowEnsurer(wf) + svc.SetTaskCreator(tc) + + run, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("fire manual: %v", err) + } + if run.CausationID == "" { + t.Fatal("expected non-empty causation id on the heavy-path run") + } + if run.LinkedTaskID != tc.taskID { + t.Fatalf("LinkedTaskID = %q, want %q", run.LinkedTaskID, tc.taskID) + } + + var sessionID string + err = db.QueryRow(` + SELECT task_sessions.id + FROM office_routine_runs + JOIN tasks ON tasks.id = office_routine_runs.linked_task_id + JOIN task_sessions ON task_sessions.task_id = tasks.id + WHERE office_routine_runs.id = ? + `, run.ID).Scan(&sessionID) + if err != nil { + t.Fatalf("two-hop join from routine run to session: %v", err) + } + if sessionID != tc.sessID { + t.Fatalf("joined session id = %q, want %q", sessionID, tc.sessID) + } +} diff --git a/apps/backend/internal/office/routines/causation_id_test.go b/apps/backend/internal/office/routines/causation_id_test.go new file mode 100644 index 0000000000..a658fafd83 --- /dev/null +++ b/apps/backend/internal/office/routines/causation_id_test.go @@ -0,0 +1,72 @@ +package routines_test + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/office/models" +) + +// AC-OFFICE-LOOP-LIVENESS-002.1: dispatchRoutineRun mints one causation +// id per fire — non-empty and distinct across separate fires. +func TestDispatchRoutineRun_MintsOneCausationIDPerFire(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + routine := createTestRoutine(t, svc, "Causation Mint", "always_create") + + first, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("first fire: %v", err) + } + if first.CausationID == "" { + t.Fatal("expected non-empty causation id on first fire") + } + + second, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("second fire: %v", err) + } + if second.CausationID == "" { + t.Fatal("expected non-empty causation id on second fire") + } + if second.CausationID == first.CausationID { + t.Fatalf("expected distinct causation ids across fires, both were %q", first.CausationID) + } +} + +// AC-OFFICE-LOOP-LIVENESS-002.2: the lightweight (taskless) path copies +// the fire's minted causation id onto the WakeupRequest it builds. +func TestDispatchRoutineRun_LightweightPathCopiesCausationIDOntoWakeupRequest(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := &models.Routine{ + WorkspaceID: "ws-1", + Name: "Lightweight Causation", + TaskTemplate: "", // lightweight + AssigneeAgentProfileID: "agent-1", + Status: "active", + ConcurrencyPolicy: "always_create", + } + if err := svc.CreateRoutine(ctx, routine); err != nil { + t.Fatalf("create routine: %v", err) + } + + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + run, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "alpha"}) + if err != nil { + t.Fatalf("fire manual: %v", err) + } + if run.CausationID == "" { + t.Fatal("expected non-empty causation id on the routine run") + } + if len(enq.created) != 1 { + t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created)) + } + if enq.created[0].CausationID != run.CausationID { + t.Fatalf("wakeup request causation id = %q, want %q (copied from the fire)", + enq.created[0].CausationID, run.CausationID) + } +} diff --git a/apps/backend/internal/office/routines/loop_counters_test.go b/apps/backend/internal/office/routines/loop_counters_test.go new file mode 100644 index 0000000000..0e3ccfe4a1 --- /dev/null +++ b/apps/backend/internal/office/routines/loop_counters_test.go @@ -0,0 +1,233 @@ +package routines_test + +import ( + "context" + "expvar" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" +) + +// loopCounterInt reads one labelled entry off a registered +// office_loop_* expvar map. Missing entries read as zero so a test can +// assert a fresh delta without pre-seeding the map. +func loopCounterInt(t *testing.T, mapName, key string) int64 { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %s not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("%s is not a Map", mapName) + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("%s[%q] is not an Int", mapName, key) + } + return iv.Value() +} + +// REQ-OFFICE-LOOP-LIVENESS-003 / AC-003.1: TickScheduledTriggers counts +// one cron loop tick evaluation regardless of whether any trigger is due. +func TestTickScheduledTriggers_IncrementsCronTickCounter(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + before := loopCounterInt(t, "office_loop_cron_tick_total", "") + if err := svc.TickScheduledTriggers(ctx, time.Now().UTC()); err != nil { + t.Fatalf("tick: %v", err) + } + after := loopCounterInt(t, "office_loop_cron_tick_total", "") + if after != before+1 { + t.Fatalf("cron_tick delta = %d, want 1", after-before) + } +} + +// AC-003.1/AC-003.9: a due, successfully-claimed trigger increments +// office_loop_trigger_claimed_total labelled by the owning routine's +// workspace, at the site the claim persists. +func TestTickScheduledTriggers_IncrementsTriggerClaimedCounter(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{ + RoutineID: routine.ID, + Kind: "cron", + CronExpression: "* * * * *", + Timezone: "UTC", + Enabled: true, + }); err != nil { + t.Fatalf("create trigger: %v", err) + } + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + key := "workspace=" + routine.WorkspaceID + before := loopCounterInt(t, "office_loop_trigger_claimed_total", key) + + if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil { + t.Fatalf("tick: %v", err) + } + + after := loopCounterInt(t, "office_loop_trigger_claimed_total", key) + if after != before+1 { + t.Fatalf("trigger_claimed delta = %d, want 1", after-before) + } +} + +// AC-003.9: a claim that persists must count even when the routine +// lookup afterward fails, so the claim can't be attributed to a +// workspace. Without the fallback, a claim that already wrote to the +// database silently vanished from office_loop_trigger_claimed_total +// (Review round 1, R1-4) — the counter must reflect the persisted +// write, not the lookup that happens to follow it. +func TestTickScheduledTriggers_ClaimedCounterSurvivesRoutineLookupFailure(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{ + RoutineID: routine.ID, + Kind: "cron", + CronExpression: "* * * * *", + Timezone: "UTC", + Enabled: true, + }); err != nil { + t.Fatalf("create trigger: %v", err) + } + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + // Orphan the trigger: delete the routine it points at without + // touching the trigger row, so ClaimTrigger still succeeds but the + // subsequent GetRoutineFromConfig fails. + if err := svc.DeleteRoutine(ctx, routine.ID); err != nil { + t.Fatalf("delete routine: %v", err) + } + + key := "workspace=_unattributed" + before := loopCounterInt(t, "office_loop_trigger_claimed_total", key) + + if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil { + t.Fatalf("tick: %v", err) + } + + after := loopCounterInt(t, "office_loop_trigger_claimed_total", key) + if after != before+1 { + t.Fatalf("trigger_claimed[_unattributed] delta = %d, want 1", after-before) + } +} + +// AC-003.2: a fire that runs to completion (lightweight, always_create) +// counts office_loop_routine_run_total with disposition=task_created. +func TestDispatchRoutineRun_CountsTaskCreatedDisposition(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created" + before := loopCounterInt(t, "office_loop_routine_run_total", key) + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("fire manual: %v", err) + } + + after := loopCounterInt(t, "office_loop_routine_run_total", key) + if after != before+1 { + t.Fatalf("routine_run[task_created] delta = %d, want 1", after-before) + } +} + +// AC-003.2: a fire that the concurrency policy skips counts +// office_loop_routine_run_total with disposition=skipped, distinct from +// the launched fire that decided the skip. +func TestDispatchRoutineRun_CountsSkippedDisposition(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{}) + svc.SetTaskCreator(&fakeTaskCreator{}) + + routine := createTestRoutine(t, svc, "Skip Counter", "skip_if_active") + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("first fire: %v", err) + } + + key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=skipped" + before := loopCounterInt(t, "office_loop_routine_run_total", key) + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("second fire: %v", err) + } + + after := loopCounterInt(t, "office_loop_routine_run_total", key) + if after != before+1 { + t.Fatalf("routine_run[skipped] delta = %d, want 1", after-before) + } +} + +// AC-003.2: a fire the concurrency policy coalesces into an existing +// active run counts office_loop_routine_run_total with +// disposition=coalesced. +func TestDispatchRoutineRun_CountsCoalescedDisposition(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{}) + svc.SetTaskCreator(&fakeTaskCreator{}) + + routine := createTestRoutine(t, svc, "Coalesce Counter", "coalesce_if_active") + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("first fire: %v", err) + } + + key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=coalesced" + before := loopCounterInt(t, "office_loop_routine_run_total", key) + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("second fire: %v", err) + } + + after := loopCounterInt(t, "office_loop_routine_run_total", key) + if after != before+1 { + t.Fatalf("routine_run[coalesced] delta = %d, want 1", after-before) + } +} + +// AC-003.1: the lightweight path's successfully-created wakeup request +// counts office_loop_wakeup_created_total labelled by the row's own +// source column, reusing routines.WakeupRequest.Source rather than +// inventing a parallel vocabulary. +func TestMaterialiseLightweightRoutineRun_IncrementsWakeupCreatedCounter(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + enq := &fakeWakeupEnqueuer{} + svc.SetWakeupEnqueuer(enq) + + key := "workspace=" + routine.WorkspaceID + ";source=routine" + before := loopCounterInt(t, "office_loop_wakeup_created_total", key) + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("fire manual: %v", err) + } + if len(enq.created) != 1 { + t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created)) + } + + after := loopCounterInt(t, "office_loop_wakeup_created_total", key) + if after != before+1 { + t.Fatalf("wakeup_created delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/routines/routine_last_run_test.go b/apps/backend/internal/office/routines/routine_last_run_test.go new file mode 100644 index 0000000000..0a6e8ef320 --- /dev/null +++ b/apps/backend/internal/office/routines/routine_last_run_test.go @@ -0,0 +1,128 @@ +package routines_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/routines" +) + +// AC-OFFICE-LOOP-LIVENESS-001.1: last_run_at advances on the coalesced +// and skipped concurrency-policy paths too, not only on a fresh +// dispatch — the call site runs before applyConcurrencyPolicy can +// short-circuit the return. +func TestDispatch_AdvancesLastRunAtOnSkippedAndCoalescedPaths(t *testing.T) { + t.Run("skipped", func(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{}) + svc.SetTaskCreator(&fakeTaskCreator{}) + routine := createTestRoutine(t, svc, "Skip Last Run", "skip_if_active") + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("first fire: %v", err) + } + afterFirst, err := svc.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if afterFirst.LastRunAt == nil { + t.Fatal("expected last_run_at set after first fire") + } + + time.Sleep(2 * time.Millisecond) + run2, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("second fire: %v", err) + } + if run2.Status != "skipped" { + t.Fatalf("second fire status = %q, want skipped", run2.Status) + } + afterSecond, err := svc.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) { + t.Fatalf("last_run_at did not advance on skipped fire: before %v, after %v", + afterFirst.LastRunAt, afterSecond.LastRunAt) + } + }) + + t.Run("coalesced", func(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{}) + svc.SetTaskCreator(&fakeTaskCreator{}) + routine := createTestRoutine(t, svc, "Coalesce Last Run", "coalesce_if_active") + + if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil { + t.Fatalf("first fire: %v", err) + } + afterFirst, err := svc.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + + time.Sleep(2 * time.Millisecond) + run2, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("second fire: %v", err) + } + if run2.Status != "coalesced" { + t.Fatalf("second fire status = %q, want coalesced", run2.Status) + } + afterSecond, err := svc.GetRoutine(ctx, routine.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) { + t.Fatalf("last_run_at did not advance on coalesced fire: before %v, after %v", + afterFirst.LastRunAt, afterSecond.LastRunAt) + } + }) +} + +// failingTouchRepo wraps the real sqlite repository but forces +// TouchRoutineLastRun to fail, so the dispatch's error-tolerance +// (AC-OFFICE-LOOP-LIVENESS-001.7) can be exercised without a real +// storage failure. +type failingTouchRepo struct { + *sqlite.Repository +} + +func (f *failingTouchRepo) TouchRoutineLastRun(context.Context, string, time.Time) error { + return errors.New("simulated last_run_at write failure") +} + +// AC-OFFICE-LOOP-LIVENESS-001.7: a TouchRoutineLastRun error does not +// fail the dispatch — the routine still fires and its run is created. +func TestDispatch_SurvivesLastRunAtWriteFailure(t *testing.T) { + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + svc := routines.NewRoutineService(&failingTouchRepo{Repository: repo}, logger.Default(), &noopActivity{}) + svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{}) + svc.SetTaskCreator(&fakeTaskCreator{}) + + routine := createTestRoutine(t, svc, "Failing Touch", "always_create") + run, err := svc.FireManual(context.Background(), routine.ID, nil) + if err != nil { + t.Fatalf("fire manual with failing touch: %v", err) + } + if run.Status != "task_created" { + t.Fatalf("status = %q, want task_created", run.Status) + } +} diff --git a/apps/backend/internal/office/routines/routine_run_counter_survives_materialize_failure_test.go b/apps/backend/internal/office/routines/routine_run_counter_survives_materialize_failure_test.go new file mode 100644 index 0000000000..165adffce9 --- /dev/null +++ b/apps/backend/internal/office/routines/routine_run_counter_survives_materialize_failure_test.go @@ -0,0 +1,75 @@ +package routines_test + +import ( + "context" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/routines" +) + +// newTestRoutineServiceRejectingRunStatusWrite is newTestRoutineService +// plus a trigger that aborts every UPDATE of office_routine_runs.status +// to 'task_created' or 'done', so a test can force materialiseRoutineRun's +// write to fail (on either the heavy or the lightweight path) without +// disturbing CreateRoutineRun (the write whose success is what +// office_loop_routine_run_total must count). +func newTestRoutineServiceRejectingRunStatusWrite(t *testing.T) *routines.RoutineService { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + repo, err := sqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + if _, err := db.Exec(` + CREATE TRIGGER reject_run_status_write + BEFORE UPDATE OF status ON office_routine_runs + WHEN NEW.status IN ('task_created', 'done') + BEGIN + SELECT RAISE(ABORT, 'run status write rejected for test'); + END; + `); err != nil { + t.Fatalf("create trigger: %v", err) + } + + log := logger.Default() + return routines.NewRoutineService(repo, log, &noopActivity{}) +} + +// AC-003.1 / AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total +// must count a routine run CreateRoutineRun genuinely persisted even when +// materialiseRoutineRun errors out afterward. Before the fix, +// IncLoopRoutineRun was only called inside the applyConcurrencyPolicy and +// materialiseRoutineRun success branches — an error from either meant +// dispatchRoutineRun returned without incrementing the counter at all, +// silently undercounting a fire that genuinely created a persisted row +// (the exact defect class this capability exists to catch: the reference +// incident was 322 consecutive fires that read as healthy because +// nothing counted them correctly). +func TestDispatchRoutineRun_CountsRunEvenWhenMaterializeFails(t *testing.T) { + svc := newTestRoutineServiceRejectingRunStatusWrite(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + + key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed" + before := loopCounterInt(t, "office_loop_routine_run_total", key) + + if _, err := svc.FireManual(ctx, routine.ID, nil); err == nil { + t.Fatal("expected FireManual to return an error from the rejected status write") + } + + after := loopCounterInt(t, "office_loop_routine_run_total", key) + if after != before+1 { + t.Fatalf("routine_run[failed] delta = %d, want 1 (the row was genuinely created, must not go uncounted)", after-before) + } +} diff --git a/apps/backend/internal/office/routines/service.go b/apps/backend/internal/office/routines/service.go index 5145268635..3575cf6aa1 100644 --- a/apps/backend/internal/office/routines/service.go +++ b/apps/backend/internal/office/routines/service.go @@ -13,6 +13,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" "github.com/kandev/kandev/internal/office/shared" taskservice "github.com/kandev/kandev/internal/task/service" ) @@ -38,6 +39,7 @@ type Repository interface { ListRoutines(ctx context.Context, workspaceID string) ([]*Routine, error) UpdateRoutine(ctx context.Context, routine *Routine) error DeleteRoutine(ctx context.Context, id string) error + TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error CreateRoutineTrigger(ctx context.Context, t *RoutineTrigger) error ListTriggersByRoutineID(ctx context.Context, routineID string) ([]*RoutineTrigger, error) @@ -62,7 +64,6 @@ type Repository interface { // read-then-write race between them. UpdateRunStatusIfTaskCreated(ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string) (bool, error) UpdateRunCoalesced(ctx context.Context, runID, coalescedIntoRunID string) error - TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error // GetTaskTerminalStatus reads a task's real lifecycle state directly // (not via the TaskMoved event — see applyConcurrencyPolicy) and // reports "" when it is active, "done", "failed", "cancelled", or @@ -95,6 +96,10 @@ type WakeupRequest struct { Payload string IdempotencyKey string RequestedAt time.Time + // CausationID is copied from the firing RoutineRun + // (AC-OFFICE-LOOP-LIVENESS-002.2) so downstream runs and wakeup + // requests correlate back to the fire that produced them. + CausationID string } // RoutineWorkflowEnsurer materialises (lazily) the routine system @@ -384,6 +389,7 @@ func (s *RoutineService) ListAllRoutineRuns(ctx context.Context, wsID string, li // TickScheduledTriggers queries due cron triggers, claims each, and dispatches. func (s *RoutineService) TickScheduledTriggers(ctx context.Context, now time.Time) error { + service.IncLoopCronTick(now.Format(time.RFC3339)) triggers, err := s.repo.GetDueTriggers(ctx, now) if err != nil { return fmt.Errorf("get due triggers: %w", err) @@ -407,8 +413,12 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin } routine, err := s.GetRoutineFromConfig(ctx, trigger.RoutineID) if err != nil { + // The claim already persisted, so the counter must still see it + // even though the workspace it belongs to can't be resolved. + service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace) return fmt.Errorf("get routine: %w", err) } + service.IncLoopTriggerClaimed(routine.WorkspaceID) // Catch-up cap: count how many cron ticks elapsed between the missed // trigger.NextRunAt (inclusive) and now, capped at routine.CatchUpMax. // Mirror of the agent_heartbeat catch-up math, adapted for cron @@ -587,28 +597,65 @@ func (s *RoutineService) dispatchRoutineRun( TriggerPayload: string(payloadJSON), DispatchFingerprint: fingerprint, StartedAt: &now, + // CausationID is minted once per fire (AC-OFFICE-LOOP-LIVENESS-002.1) + // and copied onto every wakeup request and run this fire produces. + CausationID: uuid.New().String(), } if err := s.repo.CreateRoutineRun(ctx, run); err != nil { return nil, fmt.Errorf("create run: %w", err) } + // AC-003.1/AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total + // counts this fire's CreateRoutineRun above — a write that just + // succeeded — not whatever applyConcurrencyPolicy/materialiseRoutineRun + // do afterward. A single deferred increment fires exactly once no + // matter which return path below is taken, labelled by whatever + // disposition was reached; it defaults to "failed" (the existing + // RoutineRunStatus value, not an invented one) so a fire that created + // a real row but then errored is still counted rather than silently + // dropped. + disposition := string(models.RoutineRunStatusFailed) + defer func() { + service.IncLoopRoutineRun(routine.WorkspaceID, source, disposition) + }() + + // AC-OFFICE-LOOP-LIVENESS-001.1: advances last_run_at for every + // dispatch, whatever disposition the run later reaches — so it must + // run before applyConcurrencyPolicy, which can short-circuit into a + // coalesced or skipped return. Errors are logged and counted, never + // returned (AC-001.7): a routine that fired must not read as having + // failed to fire because a bookkeeping write lost a lock. + if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, *run.StartedAt); err != nil { + service.IncLoopLastRunAtWriteFailed(routine.WorkspaceID) + s.logger.Warn("touch routine last_run_at", + zap.String("routine_id", routine.ID), zap.Error(err)) + } + status, err := s.applyConcurrencyPolicy(ctx, routine, run, fingerprint) if err != nil { return run, err } if status != "" { + disposition = string(status) return run, nil } if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil { return run, err } - - if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, now); err != nil { - s.logger.Warn("touch routine last_run_at", - zap.String("routine", routine.Name), zap.Error(err)) + // A nil error only means materialiseRoutineRun's terminal write + // succeeded, not that the write it made was a success: the + // lightweight path finalizes with a nil error even when the wakeup + // enqueue itself failed, persisting run.Status=failed. Disposition + // must reflect that instead of assuming "task_created" (the + // dispatched-successfully label, shared by the heavy path and a + // lightweight done) whenever this call merely didn't error. + if run.Status == models.RoutineRunStatusFailed { + disposition = string(models.RoutineRunStatusFailed) + } else { + disposition = string(models.RoutineRunStatusTaskCreated) } - routine.LastRunAt = &now + s.logger.Info("routine run dispatched", zap.String("routine", routine.Name), zap.String("run_id", run.ID), @@ -713,6 +760,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun( Payload: payloadStr, IdempotencyKey: idemKey, RequestedAt: time.Now().UTC(), + CausationID: run.CausationID, } if err := s.wakeup.CreateWakeupRequest(ctx, req); err != nil { if errors.Is(err, ErrWakeupAlreadyRequested) { @@ -724,6 +772,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun( zap.String("routine", routine.Name), zap.Error(err)) return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusFailed) } + service.IncLoopWakeupCreated(routine.WorkspaceID, req.Source) if err := s.wakeup.Dispatch(ctx, req.ID); err != nil { s.logger.Warn("dispatch routine wakeup request", zap.String("routine", routine.Name), diff --git a/apps/backend/internal/office/routines/service_test.go b/apps/backend/internal/office/routines/service_test.go index ccd2bac3b7..336f14737c 100644 --- a/apps/backend/internal/office/routines/service_test.go +++ b/apps/backend/internal/office/routines/service_test.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" @@ -716,9 +717,13 @@ func TestDispatch_LastRunAt(t *testing.T) { t.Fatal("expected last_run_at to be set after a materialised fire") } firstStamp := *afterFirst.LastRunAt + time.Sleep(2 * time.Millisecond) - // Second fire is skipped (run1's task is still open) — last_run_at - // must not advance for a fire that didn't materialise. + // Second fire is skipped (run1's task is still open), but + // AC-OFFICE-LOOP-LIVENESS-001.1 advances last_run_at for every + // dispatch regardless of disposition — the write runs before + // applyConcurrencyPolicy can short-circuit into skipped/coalesced, + // so a skip still counts as evidence the loop is alive. run2, err := svc.FireManual(ctx, routine.ID, nil) if err != nil { t.Fatalf("second fire: %v", err) @@ -730,7 +735,7 @@ func TestDispatch_LastRunAt(t *testing.T) { if err != nil { t.Fatalf("get routine: %v", err) } - if !afterSecond.LastRunAt.Equal(firstStamp) { - t.Errorf("last_run_at changed from %v to %v after a skipped fire", firstStamp, afterSecond.LastRunAt) + if !afterSecond.LastRunAt.After(firstStamp) { + t.Errorf("last_run_at did not advance on a skipped fire: before %v, after %v", firstStamp, afterSecond.LastRunAt) } } diff --git a/apps/backend/internal/office/routines/wakeup_enqueue_fail_disposition_test.go b/apps/backend/internal/office/routines/wakeup_enqueue_fail_disposition_test.go new file mode 100644 index 0000000000..e43eefa250 --- /dev/null +++ b/apps/backend/internal/office/routines/wakeup_enqueue_fail_disposition_test.go @@ -0,0 +1,64 @@ +package routines_test + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/routines" +) + +// failingCreateWakeupEnqueuer fails every CreateWakeupRequest call with a +// generic (non-ErrWakeupAlreadyRequested) error, but still lets Dispatch/ +// FailWakeupRequest succeed trivially — they are never reached once +// CreateWakeupRequest itself fails. +type failingCreateWakeupEnqueuer struct{ fakeWakeupEnqueuer } + +func (f *failingCreateWakeupEnqueuer) CreateWakeupRequest(context.Context, *routines.WakeupRequest) error { + return errors.New("wakeup store unavailable") +} + +// TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated is the +// PR fixup round 2 regression test (CodeRabbit). materialiseLightweightRoutineRun +// finalizes a run as RoutineRunStatusFailed when CreateWakeupRequest errors, +// but returns a nil error once that finalizing UPDATE itself succeeds. The +// caller (dispatchRoutineRun) used to read that nil error as "materialise +// succeeded" and hard-code disposition = task_created regardless of what +// status was actually persisted — so a routine run that genuinely failed to +// enqueue its wakeup still recorded office_loop_routine_run_total as a +// success, corrupting the loop-health evidence this capability exists to +// produce. +func TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated(t *testing.T) { + svc := newTestRoutineService(t) + ctx := context.Background() + + routine := newLightweightTestRoutine(t, svc) + svc.SetWakeupEnqueuer(&failingCreateWakeupEnqueuer{}) + + failedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed" + taskCreatedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created" + failedBefore := loopCounterInt(t, "office_loop_routine_run_total", failedKey) + taskCreatedBefore := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey) + + run, err := svc.FireManual(ctx, routine.ID, nil) + if err != nil { + t.Fatalf("FireManual: %v", err) + } + if run.Status != models.RoutineRunStatusFailed { + t.Fatalf("run.Status = %q, want %q — the wakeup enqueue failed and finalizeLightweightRun persisted 'failed'", + run.Status, models.RoutineRunStatusFailed) + } + + failedAfter := loopCounterInt(t, "office_loop_routine_run_total", failedKey) + if failedAfter != failedBefore+1 { + t.Fatalf("routine_run[failed] delta = %d, want 1 — a failed wakeup enqueue must not be "+ + "reported as disposition=task_created", failedAfter-failedBefore) + } + + taskCreatedAfter := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey) + if taskCreatedAfter != taskCreatedBefore { + t.Fatalf("routine_run[task_created] delta = %d, want 0 for a run whose wakeup enqueue failed", + taskCreatedAfter-taskCreatedBefore) + } +} diff --git a/apps/backend/internal/office/scheduler/dispatch_routing.go b/apps/backend/internal/office/scheduler/dispatch_routing.go index 902d3af09e..13fbcd459a 100644 --- a/apps/backend/internal/office/scheduler/dispatch_routing.go +++ b/apps/backend/internal/office/scheduler/dispatch_routing.go @@ -14,8 +14,23 @@ import ( "github.com/kandev/kandev/internal/office/models" sqliterepo "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/office/routing" + "github.com/kandev/kandev/internal/office/service" ) +// TaskStarterWithSession optionally returns the id of the agent session +// a routed launch created, mirroring service.TaskStarterWithSession for +// the legacy launch path (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that +// does not implement this falls back to StartTaskWithRoute, and the +// run's session id stays empty. +type TaskStarterWithSession interface { + StartTaskWithRouteReturningSession( + ctx context.Context, + taskID, agentProfileID string, + launch LaunchContext, + route RouteOverride, + ) (sessionID string, err error) +} + // RouteAttemptOutcomeLaunched is the in-flight outcome string the // dispatcher writes when an attempt is appended. It is overwritten with // failed_provider_unavailable / failed_other on classifier verdict. @@ -257,7 +272,7 @@ func (ss *SchedulerService) tryCandidates( if err != nil { return false, nil, err } - launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch) + sessionID, launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch) if launchErr == nil { if prev != nil { // We walked past at least one prior candidate — that's @@ -270,7 +285,7 @@ func (ss *SchedulerService) tryCandidates( } ss.recordRouteAttempt(agent.WorkspaceID, string(candidate.ProviderID), metricOutcomeSuccess, "") - return ss.handleLaunchSuccess(ctx, run, agent, candidate) + return ss.handleLaunchSuccess(ctx, run, agent, candidate, sessionID) } // Emit the failure-side route_attempt before classifying. The // outcome label is refined by handleLaunchFailure. @@ -402,20 +417,20 @@ func (ss *SchedulerService) recordAttemptStart( func (ss *SchedulerService) launchCandidate( ctx context.Context, taskID, agentID string, candidate routing.Candidate, launch LaunchContext, -) error { +) (string, error) { if taskID == "" { - return fmt.Errorf("dispatch: empty task id in run payload") + return "", fmt.Errorf("dispatch: empty task id in run payload") } if _, ok := routingerr.InjectedCode(string(candidate.ProviderID)); ok { // Synthesize a launch failure via Classify so injection is // honoured for deterministic E2E specs. Classify short-circuits // to the injected code at the head of its decision chain. - return routingerr.Classify(routingerr.Input{ + return "", routingerr.Classify(routingerr.Input{ Phase: routingerr.PhaseProcessStart, ProviderID: string(candidate.ProviderID), }) } - return ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, RouteOverride{ + route := RouteOverride{ ExecutionProfileID: candidate.ExecutionProfileID, ProviderID: string(candidate.ProviderID), Model: candidate.Model, @@ -423,24 +438,72 @@ func (ss *SchedulerService) launchCandidate( Mode: candidate.Mode, Flags: candidate.Flags, Env: candidate.Env, - }) + } + if starter, ok := ss.taskStarter.(TaskStarterWithSession); ok { + return starter.StartTaskWithRouteReturningSession(ctx, taskID, agentID, launch, route) + } + return "", ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, route) } -// handleLaunchSuccess records the resolved provider/model on the run -// row and flips every health scope this candidate touched back to -// healthy. Returns (launched=true, nil, nil) on success. +// handleLaunchSuccess persists the launched session id, counts the +// launch, records the resolved provider/model on the run row, and +// flips every health scope this candidate touched back to healthy. +// Always returns (launched=true, nil, nil): by the time this is called +// launchCandidate has already started a real agent session, so nothing +// downstream may report the launch as failed or leave it uncounted +// (AC-002.7, AC-003.9). The session id and the launch counter are +// persisted first and unconditionally — they are the correlation this +// capability exists to preserve — before the best-effort +// resolved-route snapshot, whose own failure (Review round 3, R3-2) +// must not undo them or cause a caller to retry-launch an agent that +// is already running. func (ss *SchedulerService) handleLaunchSuccess( ctx context.Context, run *models.Run, - agent *models.AgentInstance, candidate routing.Candidate, + agent *models.AgentInstance, candidate routing.Candidate, sessionID string, ) (bool, *routing.BlockReason, error) { + service.IncLoopLaunch(agent.WorkspaceID) + ss.persistLaunchedSession(ctx, run.ID, agent.WorkspaceID, sessionID) if err := ss.repo.SetRunResolvedRoute(ctx, run.ID, candidate.ExecutionProfileID, string(candidate.ProviderID), candidate.Model); err != nil { - return false, nil, err + ss.logger.Warn("persist resolved route failed", + zap.String("run_id", run.ID), zap.String("provider_id", string(candidate.ProviderID)), zap.Error(err)) } ss.markHealthScopes(ctx, agent.WorkspaceID, candidate) return true, nil, nil } +// persistLaunchedSession stores the session id a successful launch +// produced (AC-OFFICE-LOOP-LIVENESS-002.7). An empty id is counted as a +// without-session launch (AC-002.8) rather than attempted as a write. +// A write failure does not fail the launch — the agent is already +// running — and is counted separately from the without-session case +// (AC-002.11) so the two causes of an identical "processed, no +// session" row stay distinguishable in the counters. A non-empty id +// whose write affects zero rows (e.g. the run row vanished between +// claim and this write) is the same AC-002.11 case as a hard write +// error — a real session id that failed to persist — not the AC-002.8 +// case, so it counts under the same persist-failed counter. +func (ss *SchedulerService) persistLaunchedSession( + ctx context.Context, runID, workspaceID, sessionID string, +) { + if sessionID == "" { + service.IncLoopLaunchWithoutSession(workspaceID) + return + } + wrote, err := ss.repo.SetRunSessionID(ctx, runID, sessionID) + if err != nil { + ss.logger.Warn("persist launched session id failed", + zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err)) + service.IncLoopSessionPersistFailed(workspaceID) + return + } + if !wrote { + ss.logger.Warn("persist launched session id matched no row", + zap.String("run_id", runID), zap.String("session_id", sessionID)) + service.IncLoopSessionPersistFailed(workspaceID) + } +} + // markHealthScopes marks the provider, tier, and model scopes healthy // for the launched candidate. Best-effort: errors are logged but do // not fail the launch. diff --git a/apps/backend/internal/office/scheduler/dispatch_routing_resolved_route_failure_test.go b/apps/backend/internal/office/scheduler/dispatch_routing_resolved_route_failure_test.go new file mode 100644 index 0000000000..5767380269 --- /dev/null +++ b/apps/backend/internal/office/scheduler/dispatch_routing_resolved_route_failure_test.go @@ -0,0 +1,85 @@ +package scheduler_test + +import ( + "context" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/routing" + "github.com/kandev/kandev/internal/office/scheduler" +) + +// newTestRepoSchedRejectingResolvedRouteWrite is newTestRepoSched plus a +// trigger that aborts every write to resolved_execution_profile_id, so a +// test can force SetRunResolvedRoute to fail without disturbing the +// session_id write persistLaunchedSession performs — the two writes +// this test needs to tell apart. +func newTestRepoSchedRejectingResolvedRouteWrite(t *testing.T) *officesqlite.Repository { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + if _, err := db.Exec(` + CREATE TRIGGER reject_resolved_route_write + BEFORE UPDATE OF resolved_execution_profile_id ON runs + BEGIN + SELECT RAISE(ABORT, 'resolved route write rejected for test'); + END; + `); err != nil { + t.Fatalf("create trigger: %v", err) + } + return repo +} + +// AC-002.7 / AC-002.11 / AC-003.9 (Review round 3, R3-2): a launch that +// already started a real agent session must not lose that session's +// correlation, or go uncounted as a launch, just because an unrelated +// bookkeeping write (the resolved provider/model snapshot) fails +// afterward. Before the fix, handleLaunchSuccess returned early on a +// SetRunResolvedRoute error, before ever reaching persistLaunchedSession +// or IncLoopLaunch — silently abandoning a live, now-uncorrelated +// session and under-counting a launch that genuinely happened. +func TestDispatch_RoutedLaunchSurvivesResolvedRouteWriteFailure(t *testing.T) { + repo := newTestRepoSchedRejectingResolvedRouteWrite(t) + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-resolved-route-fail"} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-resolved-route-fail"}`) + + beforeLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID) + + launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}) + if err != nil || !launched || parked { + t.Fatalf("launched=%v parked=%v err=%v, want launched despite the resolved-route write failure", launched, parked, err) + } + if !starter.returnedCall { + t.Fatal("expected StartTaskWithRouteReturningSession to be called") + } + + updated, err := repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if updated.SessionID != "sess-resolved-route-fail" { + t.Fatalf("session_id = %q, want sess-resolved-route-fail (a live launch must not lose its session correlation)", updated.SessionID) + } + + afterLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID) + if afterLaunch != beforeLaunch+1 { + t.Fatalf("office_loop_launch_total delta = %d, want 1 (the launch genuinely happened)", afterLaunch-beforeLaunch) + } +} diff --git a/apps/backend/internal/office/scheduler/dispatch_routing_session_persist_failure_test.go b/apps/backend/internal/office/scheduler/dispatch_routing_session_persist_failure_test.go new file mode 100644 index 0000000000..1392c2e5ef --- /dev/null +++ b/apps/backend/internal/office/scheduler/dispatch_routing_session_persist_failure_test.go @@ -0,0 +1,152 @@ +package scheduler_test + +import ( + "context" + "fmt" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + + settingsstore "github.com/kandev/kandev/internal/agent/settings/store" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/routing" + "github.com/kandev/kandev/internal/office/scheduler" +) + +// newTestRepoSchedRejectingSessionWrite is newTestRepoSched plus a +// trigger that aborts only an UPDATE setting runs.session_id to +// wantSessionID — the exact value the launch under test will produce — +// so a test can force SetRunSessionID to fail (AC-002.11) without +// disturbing any other write persistLaunchedSession's callers depend on +// (route-attempt bookkeeping, resolved-route persistence). +func newTestRepoSchedRejectingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + if _, err := db.Exec(fmt.Sprintf(` + CREATE TRIGGER reject_session_id_write + BEFORE UPDATE OF session_id ON runs + WHEN NEW.session_id = '%s' + BEGIN + SELECT RAISE(ABORT, 'session_id write rejected for test'); + END; + `, wantSessionID)); err != nil { + t.Fatalf("create trigger: %v", err) + } + return repo +} + +// AC-002.11: on a routed launch, a session persist failure is counted +// separately from a without-session launch, and does not fail the +// launch itself or overwrite the run's session_id — the agent is +// already running by the time this write happens. +func TestDispatch_RoutedLaunchSessionPersistFailureCountsSeparately(t *testing.T) { + repo := newTestRepoSchedRejectingSessionWrite(t, "sess-routed-persist-fail") + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-persist-fail"} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-session-persist-fail"}`) + + beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID) + beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + + launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}) + if err != nil || !launched || parked { + t.Fatalf("launched=%v parked=%v err=%v, want launched despite the persist failure", launched, parked, err) + } + + updated, err := repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if updated.SessionID != "" { + t.Fatalf("session_id = %q, want unchanged/empty after a rejected write", updated.SessionID) + } + + afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID) + if afterFailed != beforeFailed+1 { + t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed) + } + afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + if afterWithoutSession != beforeWithoutSession { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession) + } +} + +// newTestRepoSchedSilentlyDroppingSessionWrite is newTestRepoSched plus a +// trigger that uses RAISE(IGNORE) rather than RAISE(ABORT): the UPDATE +// setting runs.session_id to wantSessionID matches zero rows and returns +// no error, exactly the shape a run row that vanished between claim and +// launch (an unrelated delete/archive racing the session-persist write) +// would produce. This is a different failure mode than +// newTestRepoSchedRejectingSessionWrite's hard DB error: SetRunSessionID +// reports (wrote=false, err=nil), not (wrote=false, err=non-nil). +func newTestRepoSchedSilentlyDroppingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository { + t.Helper() + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if _, _, err := settingsstore.Provide(db, db, nil); err != nil { + t.Fatalf("settings store: %v", err) + } + repo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + if _, err := db.Exec(fmt.Sprintf(` + CREATE TRIGGER ignore_session_id_write + BEFORE UPDATE OF session_id ON runs + WHEN NEW.session_id = '%s' + BEGIN + SELECT RAISE(IGNORE); + END; + `, wantSessionID)); err != nil { + t.Fatalf("create trigger: %v", err) + } + return repo +} + +// AC-002.11: a launch that yields a real, non-empty session id whose +// write affects zero rows (no error) must be counted as a session +// persist failure, not miscounted as "launch yielded no session id" +// (AC-002.8's counter) — the two causes of an identical "processed, no +// session" row must stay distinguishable. +func TestDispatch_RoutedLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) { + repo := newTestRepoSchedSilentlyDroppingSessionWrite(t, "sess-routed-vanished") + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-vanished"} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-session-vanished"}`) + + beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID) + beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + + launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}) + if err != nil || !launched || parked { + t.Fatalf("launched=%v parked=%v err=%v, want launched despite the dropped write", launched, parked, err) + } + + afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID) + if afterFailed != beforeFailed+1 { + t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed) + } + afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + if afterWithoutSession != beforeWithoutSession { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession) + } +} diff --git a/apps/backend/internal/office/scheduler/dispatch_routing_session_test.go b/apps/backend/internal/office/scheduler/dispatch_routing_session_test.go new file mode 100644 index 0000000000..bece72052c --- /dev/null +++ b/apps/backend/internal/office/scheduler/dispatch_routing_session_test.go @@ -0,0 +1,153 @@ +package scheduler_test + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/routing" + "github.com/kandev/kandev/internal/office/scheduler" +) + +// fakeTaskStarterWithSession wraps fakeTaskStarter and additionally +// implements scheduler.TaskStarterWithSession, so dispatch tests can +// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7) +// without touching the production orchestrator adapter. +type fakeTaskStarterWithSession struct { + *fakeTaskStarter + sessionID string + sessionErr error + calledRoute scheduler.RouteOverride + returnedCall bool +} + +func (f *fakeTaskStarterWithSession) StartTaskWithRouteReturningSession( + ctx context.Context, taskID, agentID string, + launch scheduler.LaunchContext, route scheduler.RouteOverride, +) (string, error) { + f.returnedCall = true + f.calledRoute = route + if err := f.StartTaskWithRoute(ctx, taskID, agentID, launch, route); err != nil { + return "", err + } + if f.sessionErr != nil { + return "", f.sessionErr + } + return f.sessionID, nil +} + +// expvarMapInt reads one label key out of a package-global expvar.Map, +// returning 0 when the key was never incremented. Used to assert +// counter deltas without a reset seam — expvar's registry is global and +// process-wide, so this reads exactly what production code would. +func expvarMapInt(t *testing.T, mapName, key string) int64 { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a Map", mapName) + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("expvar %q[%q] is not an Int", mapName, key) + } + return iv.Value() +} + +// AC-OFFICE-LOOP-LIVENESS-002.7: on a routed launch, the session id the +// starter returns is persisted onto the run row. +func TestDispatch_RoutedLaunchPersistsSessionID(t *testing.T) { + repo := newTestRepoSched(t) + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-1"} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-session-1"}`) + + before := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID) + + launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}) + if err != nil || !launched || parked { + t.Fatalf("launched=%v parked=%v err=%v, want launched", launched, parked, err) + } + if !starter.returnedCall { + t.Fatal("expected StartTaskWithRouteReturningSession to be called") + } + + updated, err := repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if updated.SessionID != "sess-routed-1" { + t.Fatalf("session_id = %q, want sess-routed-1", updated.SessionID) + } + + after := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID) + if after != before+1 { + t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before) + } +} + +// AC-002.8: an empty returned session id is left empty and counted +// under launch_without_session, never replaced with a placeholder. +func TestDispatch_RoutedLaunchEmptySessionCountsWithoutSession(t *testing.T) { + repo := newTestRepoSched(t) + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: ""} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-session-2"}`) + + before := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + + launched, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}) + if err != nil || !launched { + t.Fatalf("launched=%v err=%v, want launched", launched, err) + } + + updated, err := repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if updated.SessionID != "" { + t.Fatalf("session_id = %q, want empty (no placeholder)", updated.SessionID) + } + after := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID) + if after != before+1 { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before) + } +} + +// AC-002.10: relaunching the same run with a new session id overwrites +// the previous one — last non-empty wins within one claim. +func TestDispatch_RoutedRelaunchKeepsLatestSessionID(t *testing.T) { + repo := newTestRepoSched(t) + seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"}) + starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-first"} + ss := buildScheduler(t, repo, starter) + run := seedRun(t, repo, `{"task_id":"t-session-3"}`) + + if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil { + t.Fatalf("first dispatch: %v", err) + } + starter.sessionID = "sess-second" + // Re-fetch: DispatchWithRouting mutates run.LogicalProviderOrder in + // place, so reuse the same struct to simulate a relaunch of the + // same claimed row. + if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil { + t.Fatalf("second dispatch: %v", err) + } + + updated, err := repo.GetRunByID(context.Background(), run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if updated.SessionID != "sess-second" { + t.Fatalf("session_id = %q, want sess-second (last non-empty wins)", updated.SessionID) + } +} diff --git a/apps/backend/internal/office/scheduler/run_processing.go b/apps/backend/internal/office/scheduler/run_processing.go index 755d84fa59..0c8a1a7620 100644 --- a/apps/backend/internal/office/scheduler/run_processing.go +++ b/apps/backend/internal/office/scheduler/run_processing.go @@ -33,13 +33,15 @@ func (ss *SchedulerService) ClaimNextRun(ctx context.Context) (*models.Run, erro // happened, so it is not bucketed as processed (docs/specs/ // task-delivery-ledger/spec.md, "Office run outcome"). func (ss *SchedulerService) FinishRun(ctx context.Context, id string) error { - return ss.repo.FinishRun(ctx, id, RunStatusFinished, nil) + _, err := ss.repo.FinishRun(ctx, id, RunStatusFinished, nil) + return err } // FailRun marks a claimed run as failed. outcome is written as NULL, same // as the service-layer FailRun: a failed run is bucketed on status alone. func (ss *SchedulerService) FailRun(ctx context.Context, id string) error { - return ss.repo.FinishRun(ctx, id, RunStatusFailed, nil) + _, err := ss.repo.FinishRun(ctx, id, RunStatusFailed, nil) + return err } // ProcessRunGuard checks if the agent is still eligible to be woken. diff --git a/apps/backend/internal/office/service/budget_admission.go b/apps/backend/internal/office/service/budget_admission.go index 7a6fdbcd29..618e7277b8 100644 --- a/apps/backend/internal/office/service/budget_admission.go +++ b/apps/backend/internal/office/service/budget_admission.go @@ -172,7 +172,18 @@ func (si *SchedulerIntegration) finishPolicyBlock( } else { incBudgetBlockedByLimit(provenance) } - _ = si.svc.FinishRun(ctx, run.ID, outcome) + wrote, err := si.svc.FinishRun(ctx, run.ID, outcome) + if err != nil { + si.logger.Error("failed to finish policy-blocked run", + zap.String("run_id", run.ID), zap.Error(err)) + return false + } + if !wrote { + // Another writer already moved the run out of claimed between the + // admission decision and this write; it did not actually end via a + // budget block, so there is nothing to log (Review round 3, R3-1). + return false + } fields := map[string]string{"degraded": strconv.FormatBool(p.Degraded)} if p.IsDefault { @@ -197,11 +208,21 @@ func (si *SchedulerIntegration) cancelBudgetRun( si.releaseCheckoutIfNeeded(ctx, run) si.svc.clearAgentWorking(ctx, agent.ID, run.ID) - if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil { + cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason) + if err != nil { si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err)) - } else { - si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) + return false } + if !cancelled { + // Another writer already moved the run out of claimed between the + // admission decision and this write; it did not actually end via + // this cancellation, so there is nothing to classify, publish, or + // log. + return false + } + + si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil) + si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) fields := map[string]string{activityFieldCeiling: ceilingNotDetermined} for k, v := range extraFields { @@ -383,11 +404,17 @@ func (si *SchedulerIntegration) cancelUnresolvableAgentRun(ctx context.Context, incBudgetCancelledNoWorkspace(provenance) si.cleanupWorkspaceLookupRun(ctx, run) - if err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace"); err != nil { + cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace") + if err != nil { si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err)) - } else { - si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) + return } + if !cancelled { + return + } + + si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil) + si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) si.svc.LogActivityWithRun(ctx, "", "scheduler", "office-scheduler", "run_budget_workspace_unresolvable", "run", run.ID, mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "") @@ -409,11 +436,19 @@ func (si *SchedulerIntegration) deferWorkspaceLookupFailure(ctx context.Context, if run.RetryCount >= MaxRetryCount { incBudgetFailedWorkspaceLookup(provenance) - if err := si.svc.FailRun(ctx, run.ID); err != nil { + wrote, err := si.svc.FailRun(ctx, run.ID) + if err != nil { si.logger.Error("failed to fail run without escalation", zap.String("run_id", run.ID), zap.Error(err)) return } + if !wrote { + // Already terminal via another writer (e.g. a concurrent + // cancel) between the lookup failure and this write; it did + // not actually end via this failure, so there is nothing to + // log (Review round 3, R3-1). + return + } si.svc.LogActivityWithRun(ctx, "", "system", "scheduler", "run_budget_workspace_lookup_failed", "run", run.ID, mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "") diff --git a/apps/backend/internal/office/service/budget_admission_lost_race_test.go b/apps/backend/internal/office/service/budget_admission_lost_race_test.go new file mode 100644 index 0000000000..8d8039e104 --- /dev/null +++ b/apps/backend/internal/office/service/budget_admission_lost_race_test.go @@ -0,0 +1,201 @@ +package service_test + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// cancelBudgetRun and cancelUnresolvableAgentRun mirror cancelStaleRun's +// release/cancel/classify/publish/log sequence, but their CancelRun call +// must still be gated the same way: a run that another writer already +// moved out of claimed/queued before this cancel runs must not be +// classified, broadcast, or logged as cancelled. +func TestCancelBudgetRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + agent := makeAgent("worker-budget-cancel-race", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + + // Another writer wins the terminal-transition race before gate 1's + // cancel below runs: the row is already finished, so CancelRun's + // guarded UPDATE (status IN ('queued','claimed')) will match nothing. + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish (simulating the winning writer): %v", err) + } + + got := make(chan *bus.Event, 1) + sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error { + got <- e + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer func() { _ = sub.Unsubscribe() }() + + unresolvedAgent := *agent + unresolvedAgent.WorkspaceID = "" + service.AdmitRunForTest(svc, ctx, run, &unresolvedAgent) + + select { + case e := <-got: + t.Fatalf("expected no OfficeRunProcessed publish for a lost budget-cancel race; got %+v", e) + default: + } + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries)) + } + + survivor, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if survivor.Status != service.RunStatusFinished { + t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status) + } +} + +// deferWorkspaceLookupFailure's MaxRetryCount-exhausted branch (CodeRabbit, +// PR fixup round 2) has the same lost-race obligation as cancelBudgetRun +// above: it discarded FailRun's wrote result and logged +// run_budget_workspace_lookup_failed unconditionally whenever err was nil, +// even when another writer had already made the run terminal. +func TestDeferWorkspaceLookupFailure_LostRaceDoesNotLog(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + agent := makeAgent("worker-workspace-lookup-race", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + run.RetryCount = service.MaxRetryCount + + // Another writer wins the terminal-transition race before this call's + // FailRun below runs. + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish (simulating the winning writer): %v", err) + } + + got := make(chan *bus.Event, 1) + sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error { + got <- e + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer func() { _ = sub.Unsubscribe() }() + + service.DeferWorkspaceLookupFailureForTest(svc, ctx, run) + + select { + case e := <-got: + t.Fatalf("expected no OfficeRunProcessed publish for a lost workspace-lookup-failure race; got %+v", e) + default: + } + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_lookup_failed", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_budget_workspace_lookup_failed activity entries = %d, want 0 for a lost race", len(entries)) + } + + survivor, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if survivor.Status != service.RunStatusFinished { + t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status) + } +} + +// cancelUnresolvableAgentRun (reached when the run's agent was deleted) +// has the same lost-race obligation as cancelBudgetRun above. +func TestCancelUnresolvableAgentRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + agent := makeAgent("worker-unresolvable-cancel-race", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + if err := svc.RepoForTest().DeleteAgentInstance(ctx, agent.ID); err != nil { + t.Fatalf("delete agent: %v", err) + } + + // Another writer wins the terminal-transition race before the + // missing-agent cancel below runs. + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish (simulating the winning writer): %v", err) + } + + got := make(chan *bus.Event, 1) + sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error { + got <- e + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer func() { _ = sub.Unsubscribe() }() + + service.ProcessRunForTest(svc, ctx, run) + + select { + case e := <-got: + t.Fatalf("expected no OfficeRunProcessed publish for a lost missing-agent cancel race; got %+v", e) + default: + } + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries)) + } + + survivor, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if survivor.Status != service.RunStatusFinished { + t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status) + } +} diff --git a/apps/backend/internal/office/service/budget_admission_test.go b/apps/backend/internal/office/service/budget_admission_test.go index a1b7d07f86..1a2fd5a33e 100644 --- a/apps/backend/internal/office/service/budget_admission_test.go +++ b/apps/backend/internal/office/service/budget_admission_test.go @@ -180,7 +180,14 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T) // Drive the run to MaxRetryCount and re-process it directly (bypassing // the real backoff delay, exactly as the existing retry tests do). + // FailRun's guarded write only takes effect while the row is still + // status='claimed' (Review round 3, R3-1), so the deferred row — + // left 'queued' by the first tick above — must be reclaimed before + // ProcessRunForTest, mirroring scheduler_run_outcome_test.go. + svc.ExecSQL(t, `UPDATE runs SET retry_count = ?, status = 'claimed' WHERE id = ?`, + service.MaxRetryCount, run.ID) run.RetryCount = service.MaxRetryCount + run.Status = "claimed" service.ProcessRunForTest(svc, ctx, run) failed, err := svc.GetRun(ctx, run.ID) diff --git a/apps/backend/internal/office/service/event_subscribers.go b/apps/backend/internal/office/service/event_subscribers.go index b4ebd616f1..970ce12b36 100644 --- a/apps/backend/internal/office/service/event_subscribers.go +++ b/apps/backend/internal/office/service/event_subscribers.go @@ -421,14 +421,6 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er } return err } - // Lifecycle: terminal "complete" event for the run detail Events log. - s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{ - "task_id": data.TaskID, - "session_id": data.SessionID, - }) - s.markRoutingSuccess(ctx, run) - s.recordRunOutputSummary(ctx, run, *data) - s.warnIfReviewDecisionMissing(ctx, run) // resolveLifecycleRun prefers the immutable run ID from the event and // falls back to task plus agent only for legacy events. Releasing here // (rather than unconditionally inside transitionRunTerminal) is safe — @@ -438,10 +430,35 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er // must stay held so ReapStaleCheckouts (not a same-agent race) is what // eventually reclaims it, instead of releasing a lock for a run that // never actually reached a terminal state. - if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil { + wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed) + if err != nil { s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) return err } + if !wrote { + // The run reached a terminal state through another writer (e.g. a + // concurrent cancel) between resolveLifecycleRun's read and this + // write — it never actually held the checkout from this call's + // point of view, so releasing it here would steal a live lock the + // same way an unconditional release would (Review round 3, R3-1). + // The agent may still be stuck "working" from the launch though. + // The completion side effects below (timeline event, routing + // health, output summary, review-decision warning) must not run + // either: this run did not actually complete from this call's + // point of view, so persisting them would attribute another + // writer's outcome (e.g. a cancel) with this one's evidence + // (CodeRabbit, PR fixup round 2). + s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) + return nil + } + // Lifecycle: terminal "complete" event for the run detail Events log. + s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{ + "task_id": data.TaskID, + "session_id": data.SessionID, + }) + s.markRoutingSuccess(ctx, run) + s.recordRunOutputSummary(ctx, run, *data) + s.warnIfReviewDecisionMissing(ctx, run) s.releaseTaskCheckoutForRun(ctx, run) s.stampRunFinished(ctx, run) return nil @@ -507,6 +524,12 @@ func (s *Service) markRoutingSuccess(ctx context.Context, run *models.Run) { rd.MarkRunSuccessHealth(ctx, run, agent) } +// runEventFieldAgentID is the run-event payload key for an agent id. +// Named to avoid a duplicate-literal lint failure — "agent_id" also +// appears as a JSON struct tag elsewhere in this file, and those two +// uses are otherwise unrelated to each other. +const runEventFieldAgentID = "agent_id" + // handleTasklessAgentCompleted attributes a taskless run completion, // finishes the run, and refreshes the per-agent, per-scope continuation // summary so the next fire has bridge context. The @@ -540,12 +563,6 @@ func (s *Service) handleTasklessAgentCompleted( } return err } - s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{ - "agent_id": data.AgentID, - "session_id": data.SessionID, - }) - s.refreshContinuationSummary(ctx, run, run.AgentProfileID) - s.recordRunOutputSummary(ctx, run, *data) // run came from GetClaimedTasklessRunForAgent: it is the run that // actually launched. Taskless runs typically carry no task_id, so this // is a no-op in the common case, but call it for the same reason as @@ -553,9 +570,26 @@ func (s *Service) handleTasklessAgentCompleted( // // Finish before releasing, same as handleAgentCompleted: a failed // FinishRun must not still give up the checkout. - if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil { + wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed) + if err != nil { return err } + if !wrote { + // Already terminal via another writer — see handleAgentCompleted's + // matching branch (Review round 3, R3-1). The completion side + // effects below must not run either: this run did not actually + // complete from this call's point of view, so recording them + // (timeline event, continuation summary, output summary) would + // attribute another writer's outcome with this one's evidence + // (CodeRabbit, PR fixup round 2). + return nil + } + s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{ + runEventFieldAgentID: data.AgentID, + "session_id": data.SessionID, + }) + s.refreshContinuationSummary(ctx, run, run.AgentProfileID) + s.recordRunOutputSummary(ctx, run, *data) s.releaseTaskCheckoutForRun(ctx, run) s.stampRunFinished(ctx, run) return nil @@ -708,9 +742,16 @@ func (s *Service) handleAgentFailed(ctx context.Context, event *bus.Event) error // rate-limit-retry callers; we deliberately do NOT call into it // here. See docs/specs/office/requirements/runtime.md. errMsg := enrichModelFailureMessage(run, data.ErrorMessage) - if err := s.HandleAgentFailure(ctx, run, errMsg); err != nil { + wrote, err := s.HandleAgentFailure(ctx, run, errMsg) + if err != nil { return err } + if !wrote { + // Already terminal via another writer — waking the CEO agent + // over a run that never actually failed would be a false alarm + // (Review round 3, R3-1). + return nil + } s.dispatchAgentErrorTrigger(ctx, run, data.TaskID, data.SessionID, errMsg) return nil } diff --git a/apps/backend/internal/office/service/event_subscribers_lost_race_side_effects_test.go b/apps/backend/internal/office/service/event_subscribers_lost_race_side_effects_test.go new file mode 100644 index 0000000000..96ef4c03a7 --- /dev/null +++ b/apps/backend/internal/office/service/event_subscribers_lost_race_side_effects_test.go @@ -0,0 +1,147 @@ +package service_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects is the +// PR fixup round 2 regression test (CodeRabbit). handleAgentCompleted used +// to record the "complete" run event and the output summary BEFORE checking +// FinishRun's result at all — so a run whose terminal write never actually +// landed (a genuine persistence failure here, or a lost race to a +// concurrent writer in production) still got its evidence recorded as if +// this call had won. Both outcomes reach the same guard now: the +// completion side effects only run once FinishRun confirms wrote=true. +func TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + ensureTaskSessionMessagesTable(t, svc) + + createTestAgent(t, svc, "ws-1", "worker-race") + taskID := createOfficeTask(t, svc, "ws-1", "worker-race") + + if err := svc.QueueRun( + ctx, "worker-race", service.RunReasonTaskAssigned, + `{"task_id":"`+taskID+`"}`, "run-output-race", + ); err != nil { + t.Fatalf("queue run: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim run: %v (run=%v)", err, run) + } + claimedAt := requireClaimedAt(t, run) + + svc.ExecSQL(t, ` + INSERT INTO task_session_messages (id, task_session_id, type, author_type, content, created_at) + VALUES ('m-race-1', 'sess-race', 'message', 'agent', 'should never be recorded', ?) + `, claimedAt.Add(5*time.Second)) + + // Force FinishRun's UPDATE to fail with a genuine error, matching + // TestSchedulerTick_AgentCompletedKeepsCheckoutWhenFinishRunFails' + // fault injection: a targeted fault (drop the column FinishRun sets) + // rather than a global read-only pragma. + svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at") + + completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{ + "task_id": taskID, + "session_id": "sess-race", + "agent_profile_id": "worker-race", + }) + if pErr := eb.Publish(ctx, events.AgentCompleted, completed); pErr != nil { + t.Fatalf("publish completed: %v", pErr) + } + + got := findRunByID(t, svc, "ws-1", run.ID) + if got.OutputSummary != "" { + t.Errorf("output_summary = %q, want empty — FinishRun never landed, so recordRunOutputSummary "+ + "must not have run", got.OutputSummary) + } + if got.Status != service.RunStatusClaimed { + t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status) + } + + runEvents, err := svc.ListRunEventsForTest(ctx, run.ID) + if err != nil { + t.Fatalf("list run events: %v", err) + } + for _, e := range runEvents { + if e.EventType == "complete" { + t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e) + } + } +} + +// TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects +// is the taskless-path counterpart: refreshContinuationSummary and +// recordRunOutputSummary must not persist evidence for a completion whose +// terminal write never landed either. +func TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) { + svc := newTestService(t) + svc.SetSyncHandlers(true) + ctx := context.Background() + eb := bus.NewMemoryEventBus(logger.Default()) + if err := svc.RegisterEventSubscribers(eb); err != nil { + t.Fatalf("register subscribers: %v", err) + } + + agent := &models.AgentInstance{ + ID: "coordinator-taskless-race-effects", + WorkspaceID: "ws-1", + Name: "coordinator-taskless-race-effects", + Role: models.AgentRoleCEO, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, ` + INSERT INTO runs ( + id, agent_profile_id, reason, payload, status, coalesced_count, + context_snapshot, requested_at, claimed_at + ) VALUES ( + 'run-taskless-race-1', ?, 'routine_trigger', '{}', + 'claimed', 1, '{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + `, agent.ID) + + svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at") + + completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{ + "agent_id": agent.ID, + "agent_profile_id": agent.ID, + "session_id": "sess-taskless-race", + }) + if err := eb.Publish(ctx, events.AgentCompleted, completed); err != nil { + t.Fatalf("publish completed: %v", err) + } + + got := findRunByID(t, svc, "ws-1", "run-taskless-race-1") + if got.Status != service.RunStatusClaimed { + t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status) + } + + runEvents, err := svc.ListRunEventsForTest(ctx, "run-taskless-race-1") + if err != nil { + t.Fatalf("list run events: %v", err) + } + for _, e := range runEvents { + if e.EventType == "complete" { + t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e) + } + } + + summary, err := svc.GetContinuationSummaryForTest(ctx, agent.ID, "") + if err == nil && summary != nil { + t.Fatalf("expected no continuation summary written when FinishRun's write failed; got %+v", summary) + } +} diff --git a/apps/backend/internal/office/service/event_subscribers_session_attribution_test.go b/apps/backend/internal/office/service/event_subscribers_session_attribution_test.go index 5412179bad..3b63781f28 100644 --- a/apps/backend/internal/office/service/event_subscribers_session_attribution_test.go +++ b/apps/backend/internal/office/service/event_subscribers_session_attribution_test.go @@ -200,7 +200,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) { taskID := uuidish("task-fail-runner", i) insertSyntheticTask(t, svc, taskID, "ws-1", "runner-pm4") w := queueAndReadRun(t, svc, "runner-pm4", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure (runner) %d: %v", i, err) } } @@ -208,7 +208,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) { taskID := uuidish("task-fail-critic", i) insertSyntheticTask(t, svc, taskID, "ws-1", "critic4") w := queueAndReadRun(t, svc, "critic4", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure (critic) %d: %v", i, err) } } diff --git a/apps/backend/internal/office/service/failure.go b/apps/backend/internal/office/service/failure.go index 9da61abf2e..cd3e7cec11 100644 --- a/apps/backend/internal/office/service/failure.go +++ b/apps/backend/internal/office/service/failure.go @@ -33,18 +33,36 @@ const ( // // No retry is scheduled — the user resolves via Resume session in the // chat or Mark fixed in the inbox. +// +// Returns wrote=false when MarkRunFailed's guarded write was a no-op — +// the run reached a terminal state through another writer (e.g. a +// concurrent cancel) between the caller's read and this call — so +// callers know not to treat a cancelled/already-terminal run as a +// genuine agent failure (Review round 3, R3-1). func (s *Service) HandleAgentFailure( ctx context.Context, run *models.Run, errorMessage string, -) error { - if err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage); err != nil { - return fmt.Errorf("mark run failed: %w", err) +) (bool, error) { + wrote, err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage) + if err != nil { + return false, fmt.Errorf("mark run failed: %w", err) + } + if !wrote { + // Nothing to classify, release, or escalate: the row's real + // terminal state was written by someone else. Still make sure + // the agent isn't left stuck "working" from the launch. + s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) + return false, nil } // MarkRunFailed bypasses transitionRunTerminal (this is the office v1 - // failure path, not FailRun), so the checkout release that lives there - // has to be duplicated here — otherwise every agent-error terminal - // transition leaks the task checkout the same way FinishRun used to. + // failure path, not FailRun), so the checkout release and terminal-shape + // count that live there have to be duplicated here — otherwise every + // agent-error terminal transition leaks the task checkout the same way + // FinishRun used to, and every genuine post-launch crash goes uncounted + // in office_loop_terminal_total. MarkRunFailed writes status='failed' + // with outcome left untouched (NULL for a launched run), matching FailRun. + s.recordTerminalShape(ctx, run, RunStatusFailed, nil) s.releaseTaskCheckoutForRun(ctx, run) // Leave "working" before the auto-pause decision below, not after: the // reset is a working → idle CAS, so running it first lets a subsequent @@ -78,7 +96,7 @@ func (s *Service) HandleAgentFailure( } } - return nil + return true, nil } // RecordAgentSuccess resets the consecutive-failure counter for the diff --git a/apps/backend/internal/office/service/failure_test.go b/apps/backend/internal/office/service/failure_test.go index b4f52e9034..8d4077ccd4 100644 --- a/apps/backend/internal/office/service/failure_test.go +++ b/apps/backend/internal/office/service/failure_test.go @@ -5,15 +5,20 @@ import ( "encoding/json" "strings" "testing" + "time" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/service" ) // queueAndReadRun enqueues a run with a unique idempotency key -// (so multiple calls in one test don't coalesce) and reads it back. -// Mirrors the production path (QueueRun) without going through the -// claim step (which lives on the repository). +// (so multiple calls in one test don't coalesce), reads it back, and +// marks it claimed — the precondition every production caller of +// HandleAgentFailure actually has (resolveLifecycleRun resolves via +// GetClaimedRunByID before ever reaching it). Review round 3, R3-1 +// guards MarkRunFailed's write to status = 'claimed', so a test that +// skipped this step would see every HandleAgentFailure call in this +// file silently no-op. func queueAndReadRun( t *testing.T, svc *service.Service, agentID, taskID string, ) *models.Run { @@ -30,6 +35,9 @@ func queueAndReadRun( } for _, w := range rows { if w.AgentProfileID == agentID && taskIDFromPayload(t, w.Payload) == taskID && w.Status != service.RunStatusFailed { + svc.ExecSQL(t, `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, + time.Now().UTC(), w.ID) + w.Status = service.RunStatusClaimed return w } } @@ -94,7 +102,7 @@ func TestHandleAgentFailure_AutoPausesAtThreshold(t *testing.T) { taskID := uuidish("task-pause", i) insertSyntheticTask(t, svc, taskID, "ws-1", "agent-pause") w := queueAndReadRun(t, svc, "agent-pause", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure %d: %v", i, err) } } @@ -128,7 +136,7 @@ func TestRecordAgentSuccess_ResetsCounter(t *testing.T) { taskID := uuidish("task-succ", i) insertSyntheticTask(t, svc, taskID, "ws-1", "agent-success") w := queueAndReadRun(t, svc, "agent-success", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure %d: %v", i, err) } } @@ -172,7 +180,7 @@ func TestHandleAgentFailure_RespectsPerAgentThreshold(t *testing.T) { taskID := "task-tight-1" insertSyntheticTask(t, svc, taskID, "ws-1", "agent-tight") w := queueAndReadRun(t, svc, "agent-tight", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure: %v", err) } @@ -198,7 +206,7 @@ func autoPauseAgent( taskID := agentID + "-task-" + uuidish("p", i) insertSyntheticTask(t, svc, taskID, wsID, agentID) w := queueAndReadRun(t, svc, agentID, taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure %d: %v", i, err) } } @@ -328,7 +336,7 @@ func TestMarkAgentRunFailedFixed_LeavesInboxWhenRequeueFails(t *testing.T) { taskID := "task-retry-after-queue-error" insertSyntheticTask(t, svc, taskID, "ws-1", "agent-task-retry") run := queueAndReadRun(t, svc, "agent-task-retry", taskID) - if err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil { t.Fatalf("handle failure: %v", err) } if err := svc.UpdateAgentStatusFields(ctx, "agent-task-retry", @@ -358,7 +366,7 @@ func TestMarkAgentPausedFixed_UsesPauseSnapshot(t *testing.T) { oldTaskID := "old-failure-task" insertSyntheticTask(t, svc, oldTaskID, "ws-1", "agent-snapshot") oldRun := queueAndReadRun(t, svc, "agent-snapshot", oldTaskID) - if err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil { t.Fatalf("old failure: %v", err) } svc.RecordAgentSuccess(ctx, "agent-snapshot") @@ -392,7 +400,7 @@ func TestMarkAgentPausedFixed_DiscardsRecoveryForReassignedTask(t *testing.T) { reassignedTaskID := "reassigned-task" insertSyntheticTask(t, svc, reassignedTaskID, "ws-1", "agent-reassigned-from") failedRun := queueAndReadRun(t, svc, "agent-reassigned-from", reassignedTaskID) - if err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil { t.Fatalf("handle failure: %v", err) } // Two more failures (on other tasks) to cross the default threshold @@ -520,6 +528,35 @@ func TestMarkAgentPausedFixed_NonPausedAgentGuardAndRequeueError(t *testing.T) { } } +// HandleAgentFailure is Office's v1 post-launch failure path +// (event_subscribers.go's handleAgentFailed) and bypasses +// transitionRunTerminal's own counting, so it must record its own +// terminal shape or office_loop_terminal_total silently misses every +// genuine agent crash (Review round 1, R1-1). +func TestHandleAgentFailure_RecordsTerminalShape(t *testing.T) { + svc, _ := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "agent-crash") + taskID := "task-crash" + insertSyntheticTask(t, svc, taskID, "ws-1", "agent-crash") + run := queueAndReadRun(t, svc, "agent-crash", taskID) + svc.ExecSQL(t, `UPDATE runs SET session_id = ? WHERE id = ?`, "session-crash", run.ID) + run.SessionID = "session-crash" + + key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeLaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil { + t.Fatalf("handle failure: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("launched_failed delta = %d, want 1", after-before) + } +} + // Pins that reassigning a task auto-dismisses the per-task inbox // entry for the OLD agent without resetting that agent's counter. func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing.T) { @@ -532,7 +569,7 @@ func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing taskID := "task-reassign-1" insertSyntheticTask(t, svc, taskID, "ws-1", "old-agent") w := queueAndReadRun(t, svc, "old-agent", taskID) - if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { + if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil { t.Fatalf("handle failure: %v", err) } diff --git a/apps/backend/internal/office/service/finish_run_error_logging_test.go b/apps/backend/internal/office/service/finish_run_error_logging_test.go new file mode 100644 index 0000000000..0f3f081737 --- /dev/null +++ b/apps/backend/internal/office/service/finish_run_error_logging_test.go @@ -0,0 +1,71 @@ +package service_test + +import ( + "context" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged is the PR fixup +// round 2 regression test (CodeRabbit). The idle-skip FinishRun call site +// used to discard FinishRun's error (`wrote, _ := ...`), treating a genuine +// persistence failure exactly like a benign lost terminal-write race +// (wrote=false, err=nil) — silently, with no log line either way. A run +// stuck 'claimed' because the terminal write itself failed needs to be +// distinguishable in logs from one that lost a race to another writer, so +// operators (and ReapStaleCheckouts/RecoverStale, the actual recovery path) +// aren't flying blind on which case occurred. +func TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged(t *testing.T) { + core, logs := observer.New(zapcore.ErrorLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("create observer logger: %v", err) + } + svc := newTestService(t, service.ServiceOptions{Logger: log}) + ctx := context.Background() + + // Reject the UPDATE FinishRun issues (status -> 'finished') so the + // idle-skip path's FinishRun call returns a genuine persistence error + // instead of a lost-race no-op. + svc.ExecSQL(t, ` + CREATE TRIGGER reject_idle_skip_finish + BEFORE UPDATE OF status ON runs + WHEN NEW.status = 'finished' + BEGIN + SELECT RAISE(ABORT, 'finish run write rejected for test'); + END; + `) + + agent := &models.AgentInstance{ + WorkspaceID: "ws-1", + Name: "idle-outcome-worker-err", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + if logs.FilterMessage("failed to finish idle-skipped run").Len() == 0 { + t.Fatal("expected the FinishRun persistence error to be logged, not silently swallowed") + } + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonHeartbeat) + if run.Status != "claimed" { + t.Fatalf("status = %q, want claimed — the write was rejected, so the run must stay "+ + "claimed for the stale-claim reaper to recover, not silently read as finished", run.Status) + } +} diff --git a/apps/backend/internal/office/service/loop_metrics.go b/apps/backend/internal/office/service/loop_metrics.go new file mode 100644 index 0000000000..bdb76f9bf0 --- /dev/null +++ b/apps/backend/internal/office/service/loop_metrics.go @@ -0,0 +1,137 @@ +package service + +import ( + "expvar" + "strings" +) + +// Loop-liveness counters (REQ-OFFICE-LOOP-LIVENESS-003). Same +// "k1=v1;k2=v2" expvar.NewMap label convention as +// internal/office/scheduler/metrics_vars.go and +// internal/orchestrator/office_stall_metrics.go, so a Prometheus +// translation layer can split on the same delimiters everywhere. +// +// AC-003.9's two increment rules: the six counters naming a persisted +// change (trigger_claimed, routine_run, wakeup_created, run_claimed, +// launch, terminal) increment at the persisting site after the write +// succeeds. The five naming an event that persists nothing (cron_tick, +// launch_without_session, session_persist_failed, +// last_run_at_write_failed, liveness_degraded) increment at the +// observing site regardless of any write outcome. +var ( + loopCronTickTotal = expvar.NewMap("office_loop_cron_tick_total") + loopTriggerClaimedTotal = expvar.NewMap("office_loop_trigger_claimed_total") + loopRoutineRunTotal = expvar.NewMap("office_loop_routine_run_total") + loopWakeupCreatedTotal = expvar.NewMap("office_loop_wakeup_created_total") + loopRunClaimedTotal = expvar.NewMap("office_loop_run_claimed_total") + loopLaunchTotal = expvar.NewMap("office_loop_launch_total") + loopLaunchWithoutSessionTotal = expvar.NewMap("office_loop_launch_without_session_total") + loopSessionPersistFailedTotal = expvar.NewMap("office_loop_session_persist_failed_total") + loopTerminalTotal = expvar.NewMap("office_loop_terminal_total") + loopLastRunAtWriteFailedTotal = expvar.NewMap("office_loop_last_run_at_write_failed_total") + loopLivenessDegradedTotal = expvar.NewMap("office_loop_liveness_degraded_total") + + // Instants, not counters — see LoopMetricLabel's callers below. + // Both are monotonic (never decrease), unlike the drifting gauges + // metrics_vars.go warns against: without them a flat counter is + // ambiguous between a restarted process and a stopped loop + // (AC-003.6). + loopCronTickAt = expvar.NewString("office_loop_cron_tick_at") + loopProcessStartedAt = expvar.NewString("office_loop_process_started_at") +) + +// LoopUnattributedWorkspace is the explicit label used when an event's +// workspace cannot be resolved (AC-003.8) — never dropped, never +// guessed, and never merged into a real workspace's totals. +const LoopUnattributedWorkspace = "_unattributed" + +// LoopMetricLabel builds a "k1=v1;k2=v2;..." expvar map key. pairs must +// be an even-length (key, value) sequence. +func LoopMetricLabel(pairs ...string) string { + if len(pairs)%2 != 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2) + for i := 0; i < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + return strings.Join(parts, ";") +} + +// IncLoopCronTick counts one cron loop tick evaluation and publishes +// the tick instant (AC-003.1, AC-003.6). +func IncLoopCronTick(atRFC3339 string) { + loopCronTickTotal.Add("", 1) + loopCronTickAt.Set(atRFC3339) +} + +// RecordLoopProcessStarted publishes the process-start instant once, +// at boot (AC-003.6). +func RecordLoopProcessStarted(atRFC3339 string) { + loopProcessStartedAt.Set(atRFC3339) +} + +// IncLoopTriggerClaimed counts one eligible trigger claimed for firing. +func IncLoopTriggerClaimed(workspaceID string) { + loopTriggerClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopRoutineRun counts one routine run created, by (source, +// disposition) so a coalesced, a skipped, and a launched fire are +// separately countable (AC-003.2). +func IncLoopRoutineRun(workspaceID, source, disposition string) { + loopRoutineRunTotal.Add( + LoopMetricLabel("workspace", workspaceID, "source", source, "disposition", disposition), 1) +} + +// IncLoopWakeupCreated counts one wakeup request created. +func IncLoopWakeupCreated(workspaceID, source string) { + loopWakeupCreatedTotal.Add(LoopMetricLabel("workspace", workspaceID, "source", source), 1) +} + +// IncLoopRunClaimed counts one run claimed off the queue. +func IncLoopRunClaimed(workspaceID string) { + loopRunClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopLaunch counts one agent launch attempt that reached the +// adapter successfully. +func IncLoopLaunch(workspaceID string) { + loopLaunchTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopLaunchWithoutSession counts a launch that reported success but +// yielded no session id (AC-002.8). +func IncLoopLaunchWithoutSession(workspaceID string) { + loopLaunchWithoutSessionTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopSessionPersistFailed counts a launch that yielded a session id +// which then failed to persist (AC-002.11) — distinct from +// IncLoopLaunchWithoutSession because the underlying rows are otherwise +// byte-identical. +func IncLoopSessionPersistFailed(workspaceID string) { + loopSessionPersistFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopTerminal counts one terminal run transition by its classified +// shape. Counts transitions, not entities: a run that reaches terminal +// twice (a requeue after a post-start fallback) contributes two events. +func IncLoopTerminal(workspaceID, shape string) { + loopTerminalTotal.Add(LoopMetricLabel("workspace", workspaceID, "shape", shape), 1) +} + +// IncLoopLastRunAtWriteFailed counts a TouchRoutineLastRun write that +// returned an error (AC-001.7). The dispatch still completes — a +// routine that fired must not read as having failed to fire because a +// bookkeeping write lost a lock. +func IncLoopLastRunAtWriteFailed(workspaceID string) { + loopLastRunAtWriteFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1) +} + +// IncLoopLivenessDegraded counts one /loop-health read that failed +// loudly, labelled by which input could not be read (AC-004.10, +// "Failure" in the design). +func IncLoopLivenessDegraded(workspaceID, reason string) { + loopLivenessDegradedTotal.Add(LoopMetricLabel("workspace", workspaceID, "reason", reason), 1) +} diff --git a/apps/backend/internal/office/service/retry.go b/apps/backend/internal/office/service/retry.go index 01662981cf..7838c8c41c 100644 --- a/apps/backend/internal/office/service/retry.go +++ b/apps/backend/internal/office/service/retry.go @@ -95,15 +95,23 @@ func isRetryStale(run *models.Run) (bool, string) { return false, "" } -// cancelRetry cancels a run that is too stale to retry. +// cancelRetry cancels a run that is too stale to retry. Terminal-shape +// recording and the OfficeRunProcessed broadcast only happen when the +// cancel actually applied: a run another writer already finished or +// failed must not be reported to subscribers as cancelled. func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason string) error { s.logger.Info("cancelling stale run retry", zap.String("run_id", run.ID), zap.String("reason", reason)) - if err := s.repo.CancelRun(ctx, run.ID, reason); err != nil { + cancelled, err := s.repo.CancelRun(ctx, run.ID, reason) + if err != nil { return err } s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) + if !cancelled { + return nil + } + s.recordTerminalShape(ctx, run, RunStatusCancelled, nil) s.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) return nil } @@ -113,10 +121,17 @@ func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason strin func (s *Service) escalateFailure( ctx context.Context, run *models.Run, runErr error, ) error { - if err := s.FailRun(ctx, run.ID); err != nil { + wrote, err := s.FailRun(ctx, run.ID) + if err != nil { return fmt.Errorf("fail run: %w", err) } s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) + if !wrote { + // Already terminal via another writer (e.g. a concurrent cancel) + // between the retry decision and this write — nothing left to + // escalate (Review round 3, R3-1). + return nil + } agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID) if err != nil { @@ -158,10 +173,17 @@ func (s *Service) failRunNoEscalation( ctx context.Context, run *models.Run, agent *models.AgentInstance, cause budgetDeferralCause, policyID string, ) error { - if err := s.FailRun(ctx, run.ID); err != nil { + wrote, err := s.FailRun(ctx, run.ID) + if err != nil { return fmt.Errorf("fail run: %w", err) } s.clearAgentWorking(ctx, run.AgentProfileID, run.ID) + if !wrote { + // Already terminal via another writer (e.g. a concurrent cancel) + // between the budget decision and this write — nothing left to + // report (Review round 3, R3-1). + return nil + } fields := map[string]string{activityFieldCeiling: ceilingNotDetermined} if policyID != "" { diff --git a/apps/backend/internal/office/service/retry_cancel.go b/apps/backend/internal/office/service/retry_cancel.go index d3dff8d449..dce987eac6 100644 --- a/apps/backend/internal/office/service/retry_cancel.go +++ b/apps/backend/internal/office/service/retry_cancel.go @@ -25,5 +25,10 @@ func (s *Service) CancelPendingRetriesForTask(ctx context.Context, taskID string zap.String("task_id", taskID), zap.Int("count", len(ids))) - return s.repo.BulkCancelRuns(ctx, ids, "task_reassigned") + cancelled, err := s.repo.BulkCancelRuns(ctx, ids, "task_reassigned") + if err != nil { + return err + } + s.recordTerminalShapesForCancelledRuns(ctx, cancelled) + return nil } diff --git a/apps/backend/internal/office/service/retry_cancel_lost_race_test.go b/apps/backend/internal/office/service/retry_cancel_lost_race_test.go new file mode 100644 index 0000000000..4b1132bca4 --- /dev/null +++ b/apps/backend/internal/office/service/retry_cancel_lost_race_test.go @@ -0,0 +1,149 @@ +package service_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// A stale-retry cancel that loses the terminal-transition race (another +// writer already finished the run) must not tell subscribers the run was +// cancelled: the run's real terminal state is finished, and a status +// mismatch between the persisted row and the broadcast event misleads +// every WS-driven UI reading it. +func TestHandleRunFailure_LostCancelRaceDoesNotPublishFalseCancelled(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + agent := makeAgent("worker-stale-retry-race", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry-race"); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + + staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour) + svc.ExecSQL(t, + `UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`, + staleRequestedAt.Add(-time.Hour).Format(time.RFC3339)) + svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID) + run.RequestedAt = staleRequestedAt + + // Another writer wins the terminal-transition race before the stale + // cancel below runs: the row is already finished, so CancelRun's + // guarded UPDATE (status IN ('queued','claimed')) will match it. + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish (simulating the winning writer): %v", err) + } + + got := make(chan *bus.Event, 1) + sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error { + got <- e + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer func() { _ = sub.Unsubscribe() }() + + netErr := errors.New("connection timed out") + if err := svc.HandleRunFailure(ctx, run, netErr); err != nil { + t.Fatalf("handle failure: %v", err) + } + + select { + case e := <-got: + t.Fatalf("expected no OfficeRunProcessed publish for a lost cancel race; got %+v", e) + default: + } + + reqs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(reqs) != 1 || reqs[0].Status != service.RunStatusFinished { + t.Fatalf("run status = %+v, want the winning writer's finished state to survive", reqs) + } +} + +// The stale-claim cancel path (evaluateRunStaleness's workflow_step_changed +// branch, reached from the scheduler tick) has the same obligation as the +// stale-retry path above: a lost cancel race must not publish a false +// "cancelled" event or log a false cancel activity entry. +func TestSchedulerIntegration_CancelStaleRunLostRaceDoesNotPublishOrLog(t *testing.T) { + mock := &mockTaskStarter{} + eb := bus.NewMemoryEventBus(logger.Default()) + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock, EventBus: eb}) + ctx := context.Background() + + agent := makeAgent("worker-moved-step-race", models.AgentRoleWorker) + agent.ExecutorPreference = `{"type":"local_pc"}` + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks + (id, workspace_id, workflow_step_id, title, created_at, updated_at) + VALUES ('task-moved-step-race', 'ws-1', 'step-current', 'Moved task', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-moved-step-race","workflow_step_id":"step-old"}`, ""); err != nil { + t.Fatalf("queue run: %v", err) + } + + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil || len(runs) != 1 { + t.Fatalf("list runs: %v (%d)", err, len(runs)) + } + runID := runs[0].ID + + // Simulate another writer winning the race for this exact row: the + // tick's own CancelRun write below will match zero rows and return + // no error, the same shape a genuinely lost race produces. + svc.ExecSQL(t, fmt.Sprintf(` + CREATE TRIGGER ignore_stale_cancel_write + BEFORE UPDATE OF status ON runs + WHEN NEW.status = 'cancelled' AND OLD.id = '%s' + BEGIN + SELECT RAISE(IGNORE); + END; + `, runID)) + + got := make(chan *bus.Event, 1) + sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error { + got <- e + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer func() { _ = sub.Unsubscribe() }() + + service.RunSchedulerTick(svc, ctx) + + select { + case e := <-got: + t.Fatalf("expected no OfficeRunProcessed publish for a lost stale-cancel race; got %+v", e) + default: + } + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_cancelled_stale", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_cancelled_stale activity entries = %d, want 0 for a lost cancel race", len(entries)) + } +} diff --git a/apps/backend/internal/office/service/retry_cancel_test.go b/apps/backend/internal/office/service/retry_cancel_test.go new file mode 100644 index 0000000000..9aa860d58a --- /dev/null +++ b/apps/backend/internal/office/service/retry_cancel_test.go @@ -0,0 +1,56 @@ +package service_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// TestCancelPendingRetriesForTask_RecordsTerminalShape is the Testing +// round 3 regression test. CancelPendingRetriesForTask calls +// repo.BulkCancelRuns directly, which bypassed office_loop_terminal_total +// entirely — the same bypass class Review round 1 (R1-1) fixed for +// HandleAgentFailure and the single-run CancelRun call sites, just missed +// here. A pending retry never launched, so cancelling it must classify +// as unlaunched_failed. +func TestCancelPendingRetriesForTask_RecordsTerminalShape(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("retry-cancel-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + taskID := "task-retry-cancel" + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) + VALUES (?, ?, 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + taskID, agent.WorkspaceID) + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"`+taskID+`"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + runs, err := svc.ListRuns(ctx, agent.WorkspaceID) + if err != nil || len(runs) != 1 { + t.Fatalf("list runs: runs=%v err=%v", runs, err) + } + run := runs[0] + if err := svc.RepoForTest().ScheduleRetry(ctx, run.ID, run.RequestedAt.Add(time.Hour), 1); err != nil { + t.Fatalf("schedule retry: %v", err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if err := svc.CancelPendingRetriesForTask(ctx, taskID); err != nil { + t.Fatalf("cancel pending retries: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/service/retry_stale_terminal_shape_test.go b/apps/backend/internal/office/service/retry_stale_terminal_shape_test.go new file mode 100644 index 0000000000..eea40aeaf2 --- /dev/null +++ b/apps/backend/internal/office/service/retry_stale_terminal_shape_test.go @@ -0,0 +1,74 @@ +package service_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// cancelRetry (the 24h-stale-retry path reached through HandleRunFailure) +// bypasses transitionRunTerminal's own counting the same way +// HandleAgentFailure and cancelStaleRun do, so it must record its own +// terminal shape or office_loop_terminal_total silently misses every +// abandoned-retry cancellation (Review round 1, R1-1). This is a distinct +// trigger condition from cancelStaleRun's workflow_step_changed path, +// covered separately in TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape. +func TestHandleRunFailure_CancelsStaleRetryAndRecordsTerminalShape(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-stale-retry", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry"); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + + // Backdate past retryMaxAge (24h) so isRetryStale routes this failure + // to cancelRetry instead of a normal backoff reschedule. Also push the + // loop-liveness activation instant back further still, so the + // backdated run doesn't fall outside the activation window and + // classify as pre_activation instead of the shape under test. + staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour) + svc.ExecSQL(t, + `UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`, + staleRequestedAt.Add(-time.Hour).Format(time.RFC3339)) + svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID) + run.RequestedAt = staleRequestedAt + + key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + netErr := errors.New("connection timed out") + if err := svc.HandleRunFailure(ctx, run, netErr); err != nil { + t.Fatalf("handle failure: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } + + reqs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(reqs) != 1 { + t.Fatalf("expected 1 run, got %d", len(reqs)) + } + if reqs[0].Status != service.RunStatusCancelled { + t.Errorf("status = %q, want cancelled", reqs[0].Status) + } + if reqs[0].CancelReason == nil || *reqs[0].CancelReason != "run_too_old" { + t.Errorf("cancel_reason = %v, want run_too_old", reqs[0].CancelReason) + } +} diff --git a/apps/backend/internal/office/service/run_claimed_counter_test.go b/apps/backend/internal/office/service/run_claimed_counter_test.go new file mode 100644 index 0000000000..0dd09fad25 --- /dev/null +++ b/apps/backend/internal/office/service/run_claimed_counter_test.go @@ -0,0 +1,81 @@ +package service_test + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +func runClaimedExpvarInt(t *testing.T, key string) int64 { + t.Helper() + v := expvar.Get("office_loop_run_claimed_total") + if v == nil { + t.Fatal("expvar map office_loop_run_claimed_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatal("office_loop_run_claimed_total is not a Map") + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("office_loop_run_claimed_total[%q] is not an Int", key) + } + return iv.Value() +} + +// REQ-OFFICE-LOOP-LIVENESS-003: ClaimNextRun counts a successfully +// claimed run labelled by the claiming agent's workspace. +func TestClaimNextRun_IncrementsRunClaimedCounter(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("run-claimed-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { + t.Fatalf("queue: %v", err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID) + before := runClaimedExpvarInt(t, key) + + req, err := svc.ClaimNextRun(ctx) + if err != nil || req == nil { + t.Fatalf("claim: req=%v err=%v", req, err) + } + + after := runClaimedExpvarInt(t, key) + if after != before+1 { + t.Fatalf("run_claimed delta = %d, want 1", after-before) + } +} + +// ClaimNextRun with no queued work never increments the counter. +func TestClaimNextRun_EmptyQueueDoesNotIncrementRunClaimedCounter(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + key := service.LoopMetricLabel("workspace", "ws-1") + before := runClaimedExpvarInt(t, key) + + req, err := svc.ClaimNextRun(ctx) + if err != nil { + t.Fatalf("claim: %v", err) + } + if req != nil { + t.Fatalf("expected nil run, got %+v", req) + } + + after := runClaimedExpvarInt(t, key) + if after != before { + t.Fatalf("run_claimed delta = %d, want 0", after-before) + } +} diff --git a/apps/backend/internal/office/service/run_lifecycle_events_test.go b/apps/backend/internal/office/service/run_lifecycle_events_test.go index dd818d50d9..61ec97f307 100644 --- a/apps/backend/internal/office/service/run_lifecycle_events_test.go +++ b/apps/backend/internal/office/service/run_lifecycle_events_test.go @@ -314,7 +314,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) { } defer func() { _ = sub.Unsubscribe() }() - if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -375,7 +375,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T) } defer func() { _ = sub.Unsubscribe() }() - if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -427,7 +427,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) { } defer func() { _ = sub.Unsubscribe() }() - if err := svc.FailRun(ctx, run.ID); err != nil { + if _, err := svc.FailRun(ctx, run.ID); err != nil { t.Fatalf("fail: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration.go b/apps/backend/internal/office/service/scheduler_integration.go index efd72d638b..251eb297e4 100644 --- a/apps/backend/internal/office/service/scheduler_integration.go +++ b/apps/backend/internal/office/service/scheduler_integration.go @@ -229,7 +229,10 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run) si.logger.Info("run skipped (agent not active)", zap.String("run_id", runID), zap.String("agent_status", string(agent.Status))) - _ = si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive) + if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive); err != nil { + si.logger.Error("failed to finish agent-inactive run", + zap.String("run_id", runID), zap.Error(err)) + } return } @@ -251,6 +254,19 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run) si.logger.Info("run skipped (no actionable tasks)", zap.String("run_id", runID), zap.String("agent", agent.Name)) + si.svc.clearAgentWorking(ctx, agent.ID, runID) + wrote, err := si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped) + if err != nil { + si.logger.Error("failed to finish idle-skipped run", + zap.String("run_id", runID), zap.Error(err)) + return + } + if !wrote { + // Another writer already moved the run out of claimed; it did + // not actually end via an idle skip, so there is nothing to + // log. + return + } si.svc.LogActivityWithRun(ctx, agent.WorkspaceID, "scheduler", "office-scheduler", "run_idle_skipped", "run", runID, @@ -258,8 +274,6 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run) "agent": agent.Name, "agent_id": agent.ID, }), runID, "") - si.svc.clearAgentWorking(ctx, agent.ID, runID) - _ = si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped) return } @@ -547,7 +561,10 @@ func (si *SchedulerIntegration) isTaskTreeGated(ctx context.Context, runID, task zap.String("task_id", taskID), zap.String("hold_id", hold.ID), zap.String("mode", hold.Mode)) - _ = si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld) + if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld); err != nil { + si.logger.Error("failed to finish tree-held run", + zap.String("run_id", runID), zap.String("task_id", taskID), zap.Error(err)) + } return true } @@ -606,12 +623,19 @@ func (si *SchedulerIntegration) launchAgent( return launched } var err error - if starter, ok := si.svc.taskStarter.(TaskStarterWithLaunchContext); ok { + var sessionID string + switch starter := si.svc.taskStarter.(type) { + case TaskStarterWithLaunchContextSession: + sessionID, err = starter.StartTaskWithLaunchContextReturningSession(ctx, taskID, launch.ProfileID, launch) + case TaskStarterWithLaunchContext: err = starter.StartTaskWithLaunchContext(ctx, taskID, launch.ProfileID, launch) - } else if starter, ok := si.svc.taskStarter.(TaskStarterWithEnv); ok { + case TaskStarterWithSession: + sessionID, err = starter.StartTaskWithEnvReturningSession(ctx, taskID, launch.ProfileID, "", "", "", + launch.Prompt, "", false, nil, launch.Env) + case TaskStarterWithEnv: err = starter.StartTaskWithEnv(ctx, taskID, launch.ProfileID, "", "", "", launch.Prompt, "", false, nil, launch.Env) - } else { + default: err = si.svc.taskStarter.StartTask(ctx, taskID, launch.ProfileID, "", "", "", launch.Prompt, "", false, nil) } @@ -626,9 +650,37 @@ func (si *SchedulerIntegration) launchAgent( _ = si.svc.HandleRunFailure(ctx, run, err) return false } + IncLoopLaunch(agent.WorkspaceID) + si.persistLaunchedSession(ctx, runID, agent.WorkspaceID, sessionID) return true } +// persistLaunchedSession stores the session id a successful direct +// launch produced (AC-OFFICE-LOOP-LIVENESS-002.7) — the scheduler-side +// counterpart to scheduler.SchedulerService.persistLaunchedSession for +// the routed launch path. See that method's doc comment for the +// without-session / persist-failed counter semantics (AC-002.8, .11). +func (si *SchedulerIntegration) persistLaunchedSession( + ctx context.Context, runID, workspaceID, sessionID string, +) { + if sessionID == "" { + IncLoopLaunchWithoutSession(workspaceID) + return + } + wrote, err := si.svc.repo.SetRunSessionID(ctx, runID, sessionID) + if err != nil { + si.logger.Warn("persist launched session id failed", + zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err)) + IncLoopSessionPersistFailed(workspaceID) + return + } + if !wrote { + si.logger.Warn("persist launched session id matched no row", + zap.String("run_id", runID), zap.String("session_id", sessionID)) + IncLoopSessionPersistFailed(workspaceID) + } +} + // failTasklessRun terminally fails a run that launchAgent determined has // no task_id. This is a scheduler capability gap, not an agent failure, so // it must NOT go through HandleAgentFailure's consecutive-failure/ @@ -665,16 +717,25 @@ func (si *SchedulerIntegration) launchAgent( func (si *SchedulerIntegration) failTasklessRun( ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string, ) { - si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{ - "phase": "scheduler.launch", - "error_message": msg, - }) si.releaseCheckoutIfNeeded(ctx, run) - if err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg); err != nil { + wrote, err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg) + if err != nil { si.logger.Error("failed to mark taskless run as failed", zap.String("run_id", run.ID), zap.Error(err)) return // don't publish a terminal event when persistence failed } + if !wrote { + // Already terminal via another writer (e.g. a concurrent cancel) + // between the caller's read and this write — nothing to classify, + // publish, or log as a scheduler-launch error on this run's + // timeline (Review round 3, R3-1). + return + } + si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{ + "phase": "scheduler.launch", + "error_message": msg, + }) + si.svc.recordTerminalShape(ctx, run, RunStatusFailed, nil) run.ErrorMessage = msg // Settle the auto-dismiss decision before publishing: the frontend @@ -728,11 +789,17 @@ func (si *SchedulerIntegration) failUnlaunchableRun( "error_message": msg, }) si.releaseCheckoutIfNeeded(ctx, run) - if err := si.svc.HandleAgentFailure(ctx, run, msg); err != nil { + wrote, err := si.svc.HandleAgentFailure(ctx, run, msg) + if err != nil { si.logger.Error("failed to handle agent failure for unlaunchable run", zap.String("run_id", run.ID), zap.Error(err)) return // don't publish a terminal event when persistence failed } + if !wrote { + // Already terminal via another writer — nothing to publish + // (Review round 3, R3-1). + return + } run.ErrorMessage = msg si.svc.publishRunProcessedForWorkspace(ctx, run.ID, RunStatusFailed, run, agent.WorkspaceID) } diff --git a/apps/backend/internal/office/service/scheduler_integration_test.go b/apps/backend/internal/office/service/scheduler_integration_test.go index 5fa4309b1b..7e012f39d5 100644 --- a/apps/backend/internal/office/service/scheduler_integration_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_test.go @@ -44,7 +44,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) { } // Finish the run. - if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -94,6 +94,40 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) { } } +// cancelStaleRun bypasses transitionRunTerminal's own counting the same +// way HandleAgentFailure does, so it must record its own terminal shape +// or office_loop_terminal_total silently misses every stale-claim +// cancellation (Review round 1, R1-1). +func TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape(t *testing.T) { + mock := &mockTaskStarter{} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := makeAgent("worker-moved-step-shape", models.AgentRoleWorker) + agent.ExecutorPreference = `{"type":"local_pc"}` + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks + (id, workspace_id, workflow_step_id, title, created_at, updated_at) + VALUES ('task-moved-step-shape', 'ws-1', 'step-current', 'Moved task', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-moved-step-shape","workflow_step_id":"step-old"}`, ""); err != nil { + t.Fatalf("queue run: %v", err) + } + + key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + service.RunSchedulerTick(svc, ctx) + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} + func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) { mock := &mockTaskStarter{} svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) @@ -236,7 +270,7 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) { } // Finish the first run. - if err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil { + if _, err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -314,7 +348,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) { t.Errorf("prompt should contain %q, got: %s", tt.contains, prompt) } - _ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed) + _, _ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed) }) } } diff --git a/apps/backend/internal/office/service/scheduler_lost_race_activity_test.go b/apps/backend/internal/office/service/scheduler_lost_race_activity_test.go new file mode 100644 index 0000000000..b58e1413b7 --- /dev/null +++ b/apps/backend/internal/office/service/scheduler_lost_race_activity_test.go @@ -0,0 +1,126 @@ +package service_test + +import ( + "context" + "fmt" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// ignoreFinishedWrite installs a trigger that makes the given run's +// terminal-transition write to status='finished' a no-op, simulating +// another writer having already claimed the terminal-transition race. +func ignoreFinishedWrite(t *testing.T, svc *service.Service, runID string) { + t.Helper() + svc.ExecSQL(t, fmt.Sprintf(` + CREATE TRIGGER ignore_finished_write + BEFORE UPDATE OF status ON runs + WHEN NEW.status = 'finished' AND OLD.id = '%s' + BEGIN + SELECT RAISE(IGNORE); + END; + `, runID)) +} + +// A heartbeat run that loses the idle-skip terminal-transition race (another +// writer already moved it out of 'claimed') must not log a false +// run_idle_skipped activity entry: the run's real terminal state came from +// whichever writer actually won, and a stray activity entry misreports why +// it ended. +func TestSchedulerIntegration_IdleSkipLostRaceDoesNotLogFalseActivity(t *testing.T) { + mock := &mockTaskStarter{} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + WorkspaceID: "ws-1", + Name: "idle-worker-race", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + // Worker defaults to skip_idle_runs=true, no tasks assigned. + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil || len(runs) != 1 { + t.Fatalf("list runs: %v (%d)", err, len(runs)) + } + ignoreFinishedWrite(t, svc, runs[0].ID) + + service.RunSchedulerTick(svc, ctx) + + if mock.callCount() != 0 { + t.Errorf("expected 0 StartTask calls, got %d", mock.callCount()) + } + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_idle_skipped", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_idle_skipped activity entries = %d, want 0 for a lost terminal-transition race", len(entries)) + } +} + +// A run that loses the budget-blocked terminal-transition race has the same +// obligation: no false run_budget_blocked activity entry. +func TestSchedulerIntegration_BudgetBlockedLostRaceDoesNotLogFalseActivity(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := &models.AgentInstance{ + WorkspaceID: "ws-1", + Name: "budget-worker-race", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + policy := &models.BudgetPolicy{ + WorkspaceID: "ws-1", + ScopeType: "agent", + ScopeID: agent.ID, + LimitSubcents: 100, + Period: "monthly", + AlertThresholdPct: 80, + ActionOnExceed: "pause_agent", + } + if err := svc.CreateBudgetPolicy(ctx, policy); err != nil { + t.Fatalf("create policy: %v", err) + } + insertTestTask(t, svc, "task-budget-race", "ws-1") + insertTestCostEvent(t, svc, agent.ID, "task-budget-race", int64(600)) + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-budget-race"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil || len(runs) != 1 { + t.Fatalf("list runs: %v (%d)", err, len(runs)) + } + ignoreFinishedWrite(t, svc, runs[0].ID) + + service.RunSchedulerTick(svc, ctx) + + entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_budget_blocked", 10) + if err != nil { + t.Fatalf("list activity: %v", err) + } + if len(entries) != 0 { + t.Fatalf("run_budget_blocked activity entries = %d, want 0 for a lost terminal-transition race", len(entries)) + } +} diff --git a/apps/backend/internal/office/service/scheduler_runs.go b/apps/backend/internal/office/service/scheduler_runs.go index 60b428d918..5575febc4e 100644 --- a/apps/backend/internal/office/service/scheduler_runs.go +++ b/apps/backend/internal/office/service/scheduler_runs.go @@ -12,6 +12,7 @@ import ( "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/runs/commentkeys" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" ) // ClaimNextRun atomically claims the next eligible run from the queue. @@ -28,6 +29,11 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) { zap.String("id", req.ID), zap.String("agent", req.AgentProfileID), zap.String("reason", req.Reason)) + workspaceID := LoopUnattributedWorkspace + if agent, err := s.GetAgentFromConfig(ctx, req.AgentProfileID); err == nil && agent != nil { + workspaceID = agent.WorkspaceID + } + IncLoopRunClaimed(workspaceID) return req, nil } @@ -37,7 +43,12 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) { // an OfficeRunProcessed bus event. The run row is fetched first so the // published payload carries enough context (agent, task, comment, reason) // for downstream WS consumers to scope updates. -func (s *Service) FinishRun(ctx context.Context, id, outcome string) error { +// +// Returns wrote=false when the guarded write was a no-op — the run was +// no longer claimed (already terminal via a concurrent writer, e.g. a +// cancel) by the time this statement ran — so there is nothing to +// classify or publish (Review round 3, R3-1). +func (s *Service) FinishRun(ctx context.Context, id, outcome string) (bool, error) { return s.transitionRunTerminal(ctx, id, RunStatusFinished, &outcome) } @@ -45,17 +56,26 @@ func (s *Service) FinishRun(ctx context.Context, id, outcome string) error { // OfficeRunProcessed event. outcome is written as NULL: a failed run is // bucketed by RunCountsByDayForAgent on status alone and never reaches the // outcome-derived buckets, so no value from the five-value vocabulary is -// invented for it. See FinishRun for the lifecycle contract. -func (s *Service) FailRun(ctx context.Context, id string) error { +// invented for it. See FinishRun for the lifecycle contract and the +// wrote=false no-op case. +func (s *Service) FailRun(ctx context.Context, id string) (bool, error) { return s.transitionRunTerminal(ctx, id, RunStatusFailed, nil) } // transitionRunTerminal updates the run row to the given terminal status // and outcome (nil writes SQL NULL) and emits OfficeRunProcessed. -// Pre-fetching the run keeps the published payload self-contained even -// when the caller doesn't hold a reference to the model. Publish errors -// are logged at debug and swallowed; persistence errors are returned to -// the caller. +// FinishRun returns the row as it stands right after that write (via +// RETURNING), so the published payload and the terminal-shape +// classification both read the same statement that persisted the change — +// neither depends on a separate read succeeding independently of it. +// Publish errors are logged at debug and swallowed; persistence errors are +// returned to the caller. +// +// FinishRun's guarded write (status = 'claimed') returns a nil run when +// another writer already moved the row to a different terminal state +// between the caller's read and this statement — nothing to classify or +// publish in that case, so this returns wrote=false without touching +// recordTerminalShape or publishRunProcessed (Review round 3, R3-1). // // Deliberately does NOT release the task checkout: transitionRunTerminal // is reached by every terminal run, including ones that never held the @@ -67,24 +87,68 @@ func (s *Service) FailRun(ctx context.Context, id string) error { // for the SAME agent + task races a first run that is genuinely still // executing and holds the checkout — that second run's release matches // the first run's own checkout_agent_id and steals its own live lock out -// from under it (Review round 3, BLOCKING FINDING 1). Callers that KNOW -// their run actually held the checkout call releaseTaskCheckoutForRun -// explicitly instead: handleAgentCompleted / handleTasklessAgentCompleted -// (event_subscribers.go) for the launched-run completion path, and -// HandleAgentFailure (failure.go) for the launched-run failure path. -func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) error { - run, getErr := s.repo.GetRunByID(ctx, id) - if getErr != nil && !errors.Is(getErr, sql.ErrNoRows) { - s.logger.Debug("get run for terminal transition failed", - zap.String("run_id", id), - zap.String("status", status), - zap.Error(getErr)) +// from under it. Callers that KNOW their run actually held the checkout +// call releaseTaskCheckoutForRun explicitly instead, and only when this +// method reports wrote=true: handleAgentCompleted / +// handleTasklessAgentCompleted (event_subscribers.go) for the +// launched-run completion path, and HandleAgentFailure (failure.go) for +// the launched-run failure path. +func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) (bool, error) { + run, err := s.repo.FinishRun(ctx, id, status, outcome) + if err != nil { + return false, err } - if err := s.repo.FinishRun(ctx, id, status, outcome); err != nil { - return err + if run == nil { + return false, nil } + s.recordTerminalShape(ctx, run, status, outcome) s.publishRunProcessed(ctx, id, status, run) - return nil + return true, nil +} + +// recordTerminalShape classifies the just-persisted terminal transition +// (REQ-OFFICE-LOOP-LIVENESS-005) and increments office_loop_terminal_total +// by the resulting shape. run is nil when transitionRunTerminal's +// pre-fetch failed; there is nothing to classify in that case. The +// workspace label falls back to LoopUnattributedWorkspace when the +// owning agent can't be resolved, matching AC-003.8 — never dropped, +// never guessed into a real workspace's totals. +func (s *Service) recordTerminalShape(ctx context.Context, run *models.Run, status string, outcome *string) { + if run == nil { + return + } + activationInstant, activationPublished := s.repo.LoopLivenessActivation() + shape := ClassifyTerminalRun(status, outcome, run.SessionID, run.RequestedAt, activationInstant, activationPublished) + workspaceID := LoopUnattributedWorkspace + if agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID); err == nil && agent != nil { + workspaceID = agent.WorkspaceID + } + IncLoopTerminal(workspaceID, string(shape)) +} + +// recordTerminalShapesForCancelledRuns classifies and counts one +// cancelled-transition shape per row a bulk cancel (CancelRunsForTasks, +// BulkCancelRuns) actually persisted — a bulk write covers rows whose +// session_id can differ per row (a claimed run may have already launched), +// so each row is classified individually rather than assuming one shape +// for the whole batch. +func (s *Service) recordTerminalShapesForCancelledRuns(ctx context.Context, cancelled []runssqlite.CancelledRun) { + for _, row := range cancelled { + s.recordTerminalShape(ctx, &models.Run{ + AgentProfileID: row.AgentProfileID, + SessionID: row.SessionID, + RequestedAt: row.RequestedAt, + }, RunStatusCancelled, nil) + } +} + +// RecordCancelledRunTerminalShapes is recordTerminalShapesForCancelledRuns, +// exported for the dashboard package's TerminalShapeRecorder seam — a +// dashboard-driven cancellation (displaced participant) reaches the same +// counter every other cancellation path reaches, without a second, +// divergence-prone classification implementation in dashboard. +func (s *Service) RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun) { + s.recordTerminalShapesForCancelledRuns(ctx, cancelled) } // releaseTaskCheckoutForRun releases the run's task checkout. Call this diff --git a/apps/backend/internal/office/service/scheduler_runs_test.go b/apps/backend/internal/office/service/scheduler_runs_test.go index 56218742e2..20e7b883b6 100644 --- a/apps/backend/internal/office/service/scheduler_runs_test.go +++ b/apps/backend/internal/office/service/scheduler_runs_test.go @@ -105,7 +105,7 @@ func TestFinishAndFailRun(t *testing.T) { t.Fatal("expected claimed run") } - if err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil { + if _, err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -141,7 +141,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) { t.Fatal("expected claimed run") } - if err := svc.FailRun(ctx, req.ID); err != nil { + if _, err := svc.FailRun(ctx, req.ID); err != nil { t.Fatalf("fail: %v", err) } @@ -157,14 +157,19 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) { } } -// TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome is Review -// round 2's lower-priority TEST-002 (spec:2130-2132): two terminal callers -// reaching the same run row must never leave status and outcome -// disagreeing, because transitionRunTerminal's underlying FinishRun writes -// both columns in a single UPDATE statement. This drives both orderings — -// FinishRun-then-FailRun and FailRun-then-FinishRun — and asserts the row -// always ends up wholly consistent with whichever call went last. -func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) { +// TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome supersedes +// Review round 2's lower-priority TEST-002 (spec:2130-2132) under Review +// round 3's R3-1 guard: transitionRunTerminal's underlying FinishRun now +// only writes while the row is still status = 'claimed' (the same guard +// that stops a stale write from clobbering a concurrent cancel), so two +// terminal callers reaching the same row no longer race to "whichever +// wrote last" — the first write reaches a real terminal state and every +// later one is a no-op that leaves status and outcome exactly as the +// first write left them. This drives both orderings — FinishRun-then- +// FailRun and FailRun-then-FinishRun — and asserts the row stays +// consistent with whichever call went FIRST, and that the second call +// reports wrote=false. +func TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome(t *testing.T) { ctx := context.Background() queueAndClaim := func(t *testing.T, svc *service.Service) string { @@ -186,36 +191,46 @@ func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) { t.Run("FailRun after FinishRun", func(t *testing.T) { svc := newTestService(t) id := queueAndClaim(t, svc) - if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil { - t.Fatalf("finish: %v", err) + wrote, err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed) + if err != nil || !wrote { + t.Fatalf("finish: wrote=%v err=%v, want wrote=true", wrote, err) } - if err := svc.FailRun(ctx, id); err != nil { - t.Fatalf("fail: %v", err) - } - got, err := svc.GetRun(ctx, id) + wrote, err = svc.FailRun(ctx, id) if err != nil { - t.Fatalf("get run: %v", err) - } - if got.Status != models.RunStatusFailed || got.Outcome != nil { - t.Fatalf("row = {status: %q, outcome: %v}, want the later FailRun write: {failed, nil}", got.Status, got.Outcome) - } - }) - - t.Run("FinishRun after FailRun", func(t *testing.T) { - svc := newTestService(t) - id := queueAndClaim(t, svc) - if err := svc.FailRun(ctx, id); err != nil { t.Fatalf("fail: %v", err) } - if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil { - t.Fatalf("finish: %v", err) + if wrote { + t.Error("fail: wrote = true, want false (run was already finished, not claimed)") } got, err := svc.GetRun(ctx, id) if err != nil { t.Fatalf("get run: %v", err) } if got.Status != models.RunStatusFinished || got.Outcome == nil || *got.Outcome != service.RunOutcomeProcessed { - t.Fatalf("row = {status: %q, outcome: %v}, want the later FinishRun write: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed) + t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed) + } + }) + + t.Run("FinishRun after FailRun", func(t *testing.T) { + svc := newTestService(t) + id := queueAndClaim(t, svc) + wrote, err := svc.FailRun(ctx, id) + if err != nil || !wrote { + t.Fatalf("fail: wrote=%v err=%v, want wrote=true", wrote, err) + } + wrote, err = svc.FinishRun(ctx, id, service.RunOutcomeProcessed) + if err != nil { + t.Fatalf("finish: %v", err) + } + if wrote { + t.Error("finish: wrote = true, want false (run was already failed, not claimed)") + } + got, err := svc.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != models.RunStatusFailed || got.Outcome != nil { + t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {failed, nil}", got.Status, got.Outcome) } }) } diff --git a/apps/backend/internal/office/service/scheduler_staleness.go b/apps/backend/internal/office/service/scheduler_staleness.go index cc50845509..2e22da0342 100644 --- a/apps/backend/internal/office/service/scheduler_staleness.go +++ b/apps/backend/internal/office/service/scheduler_staleness.go @@ -44,7 +44,11 @@ func (si *SchedulerIntegration) evaluateRunStaleness( return false, "", nil } -// cancelStaleRun marks the run cancelled, logs the event, and releases any checkout. +// cancelStaleRun marks the run cancelled, logs the event, and releases +// any checkout. Terminal-shape recording, the OfficeRunProcessed +// broadcast, and the activity log entry only happen when the cancel +// actually applied: a run another writer already finished or failed +// must not be reported as cancelled. func (si *SchedulerIntegration) cancelStaleRun( ctx context.Context, run *models.Run, @@ -60,12 +64,18 @@ func (si *SchedulerIntegration) cancelStaleRun( si.releaseCheckoutIfNeeded(ctx, run) si.svc.clearAgentWorking(ctx, agent.ID, run.ID) - if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil { + cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason) + if err != nil { si.logger.Error("failed to cancel stale run", zap.String("run_id", run.ID), zap.Error(err)) - } else { - si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) + return } + if !cancelled { + return + } + + si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil) + si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run) si.svc.LogActivityWithRun(ctx, agent.WorkspaceID, "scheduler", "office-scheduler", diff --git a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go index 7fba442bf3..a921777287 100644 --- a/apps/backend/internal/office/service/scheduler_taskless_launch_test.go +++ b/apps/backend/internal/office/service/scheduler_taskless_launch_test.go @@ -74,6 +74,103 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) { } } +// TestSchedulerTick_TasklessRunRecordsTerminalShape is the Testing round 3 +// regression test. failTasklessRun calls repo.MarkRunFailed directly +// (not through HandleAgentFailure), so it bypassed office_loop_terminal_total +// entirely — the same bypass class Review round 1 (R1-1) fixed for the other +// three production terminal-writers, just missed here. A taskless run never +// launches, so it must classify as unlaunched_failed. +func TestSchedulerTick_TasklessRunRecordsTerminalShape(t *testing.T) { + mock := &mockTaskStarter{} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "coordinator-terminal-shape", + WorkspaceID: "ws-1", + Name: "coordinator-terminal-shape", + Role: models.AgentRoleCEO, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + service.RunSchedulerTick(svc, ctx) + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} + +// TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent is the PR +// fixup round 2 regression test (CodeRabbit). failTasklessRun used to +// call AppendRunEvent before checking MarkRunFailed's wrote result, so a +// run a concurrent writer (e.g. a cancel) already made terminal still +// picked up a "scheduler.launch" error event on its timeline — a spurious +// entry on a run whose actual outcome was decided by the other writer. +// AppendRunEvent must only fire once wrote=true confirms this call won +// the terminal-write race. +func TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent(t *testing.T) { + mock := &mockTaskStarter{} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "coordinator-taskless-race", + WorkspaceID: "ws-1", + Name: "coordinator-taskless-race", + Role: models.AgentRoleCEO, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v %v", run, err) + } + + // Another writer wins the terminal-transition race before + // failTasklessRun below runs: the row is already finished, so + // MarkRunFailed's guarded UPDATE (status = 'claimed') matches nothing. + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish (simulating the winning writer): %v", err) + } + + service.FailTasklessRunForTest(svc, ctx, run, agent, "scheduler cannot launch a taskless run") + + events, err := svc.ListRunEventsForTest(ctx, run.ID) + if err != nil { + t.Fatalf("list run events: %v", err) + } + for _, e := range events { + if e.EventType == "error" { + t.Fatalf("expected no error event appended for a lost taskless-fail race; got %+v", e) + } + } + + survivor, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if survivor.Status != service.RunStatusFinished { + t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status) + } +} + // TestSchedulerTick_TaskBoundRunStillLaunches is the regression guard // alongside the taskless-failure fix above: an ordinary task-bound run with // a wired task starter must still launch normally and stay `claimed` (not diff --git a/apps/backend/internal/office/service/service.go b/apps/backend/internal/office/service/service.go index a9dc425f07..04691673fc 100644 --- a/apps/backend/internal/office/service/service.go +++ b/apps/backend/internal/office/service/service.go @@ -53,6 +53,30 @@ type TaskStarterWithLaunchContext interface { StartTaskWithLaunchContext(ctx context.Context, taskID string, agentProfileID string, launch LaunchContext) error } +// TaskStarterWithSession optionally returns the id of the agent session +// a direct (non-routed) launch created, so the caller can persist it on +// the run row (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that does not +// implement this leaves the run's session id empty, counted as a +// without-session launch. +type TaskStarterWithSession interface { + StartTaskWithEnvReturningSession(ctx context.Context, taskID string, agentProfileID string, executorID string, + executorProfileID string, priority string, prompt string, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, env map[string]string) (sessionID string, err error) +} + +// TaskStarterWithLaunchContextSession combines TaskStarterWithLaunchContext +// and TaskStarterWithSession: a starter satisfying this carries the full +// launch context (skills included) AND returns the launched session id in +// the same call, so neither capability has to be dropped for the other. +// AC-OFFICE-LOOP-LIVENESS-002.7 requires the session id unconditionally, on +// every direct launch, regardless of whether that launch also carries +// per-run skill additions — the production adapter must satisfy this +// rather than TaskStarterWithLaunchContext alone. +type TaskStarterWithLaunchContextSession interface { + StartTaskWithLaunchContextReturningSession(ctx context.Context, taskID string, agentProfileID string, + launch LaunchContext) (sessionID string, err error) +} + // LaunchContext mirrors scheduler.LaunchContext so the office.service // package can carry the Office-built launch context (prompt, env, // workflow step, attachments, plan-mode, profile) into the routing diff --git a/apps/backend/internal/office/service/session_hop_direct_test.go b/apps/backend/internal/office/service/session_hop_direct_test.go new file mode 100644 index 0000000000..79fe6ae823 --- /dev/null +++ b/apps/backend/internal/office/service/session_hop_direct_test.go @@ -0,0 +1,139 @@ +package service_test + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// mockTaskStarterWithSession wraps mockTaskStarter and additionally +// implements service.TaskStarterWithSession, so direct-launch tests can +// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7) +// without touching the production orchestrator adapter. +type mockTaskStarterWithSession struct { + *mockTaskStarter + sessionID string +} + +func (m *mockTaskStarterWithSession) StartTaskWithEnvReturningSession( + ctx context.Context, taskID, agentProfileID, executorID, + executorProfileID string, priority string, prompt, workflowStepID string, + planMode bool, attachments []v1.MessageAttachment, env map[string]string, +) (string, error) { + if err := m.StartTaskWithEnv(ctx, taskID, agentProfileID, executorID, + executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env); err != nil { + return "", err + } + return m.sessionID, nil +} + +func directHopExpvarInt(t *testing.T, mapName, key string) int64 { + t.Helper() + v := expvar.Get(mapName) + if v == nil { + t.Fatalf("expvar map %q not registered", mapName) + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not a Map", mapName) + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("expvar %q[%q] is not an Int", mapName, key) + } + return iv.Value() +} + +// AC-OFFICE-LOOP-LIVENESS-002.7: on a direct (non-routed) launch, the +// session id the starter returns is persisted onto the run row. +func TestSchedulerTick_DirectLaunchPersistsSessionID(t *testing.T) { + mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-1"} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "direct-session-agent", + WorkspaceID: "ws-1", + Name: "direct-session-worker", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) + VALUES ('task-direct-session-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-session-1"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + before := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1") + + service.RunSchedulerTick(svc, ctx) + + if mock.callCount() != 1 { + t.Fatalf("expected 1 launch call, got %d", mock.callCount()) + } + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 1 || runs[0].SessionID != "sess-direct-1" { + t.Fatalf("session_id = %q, want sess-direct-1", runs[0].SessionID) + } + after := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1") + if after != before+1 { + t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before) + } +} + +// AC-002.8: a starter that only implements TaskStarterWithEnv (no +// session-returning seam) leaves the run's session id empty and counts +// under launch_without_session — the existing production fallback. +func TestSchedulerTick_DirectLaunchWithoutSessionSeamCountsWithoutSession(t *testing.T) { + mock := &mockTaskStarter{} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "direct-nosession-agent", + WorkspaceID: "ws-1", + Name: "direct-nosession-worker", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) + VALUES ('task-direct-nosession-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-nosession-1"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + before := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + + service.RunSchedulerTick(svc, ctx) + + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 1 || runs[0].SessionID != "" { + t.Fatalf("session_id = %q, want empty", runs[0].SessionID) + } + after := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + if after != before+1 { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/service/session_persist_failure_test.go b/apps/backend/internal/office/service/session_persist_failure_test.go new file mode 100644 index 0000000000..46759a626c --- /dev/null +++ b/apps/backend/internal/office/service/session_persist_failure_test.go @@ -0,0 +1,154 @@ +package service_test + +import ( + "context" + "fmt" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// rejectSessionIDWrites installs a trigger that aborts only an UPDATE +// setting runs.session_id to wantSessionID — the exact value the +// launch under test will produce — so it targets persistLaunchedSession's +// post-launch SetRunSessionID write without also catching the +// pre-launch runtime-context build, which writes session_id too (see +// runtime/context_builder.go) but with the run's pre-launch id. +func rejectSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) { + t.Helper() + svc.ExecSQL(t, fmt.Sprintf(` + CREATE TRIGGER reject_session_id_write + BEFORE UPDATE OF session_id ON runs + WHEN NEW.session_id = '%s' + BEGIN + SELECT RAISE(ABORT, 'session_id write rejected for test'); + END; + `, wantSessionID)) +} + +// AC-002.11: on a direct (non-routed) launch, a session persist failure +// is counted separately from a without-session launch, and does not +// fail the launch itself or overwrite the run's session_id — the agent +// is already running by the time this write happens. +func TestSchedulerTick_DirectLaunchSessionPersistFailureCountsSeparately(t *testing.T) { + mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-persist-fail"} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "direct-persist-fail-agent", + WorkspaceID: "ws-1", + Name: "direct-persist-fail-worker", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) + VALUES ('task-direct-persist-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-persist-fail-1"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + rejectSessionIDWrites(t, svc, "sess-direct-persist-fail") + + beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1") + beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + + service.RunSchedulerTick(svc, ctx) + + if mock.callCount() != 1 { + t.Fatalf("expected 1 launch call, got %d", mock.callCount()) + } + runs, err := svc.ListRuns(ctx, "ws-1") + if err != nil { + t.Fatalf("list runs: %v", err) + } + if len(runs) != 1 { + t.Fatalf("run count = %d, want 1", len(runs)) + } + if runs[0].SessionID != "" { + t.Fatalf("session_id = %q, want empty after a rejected write", runs[0].SessionID) + } + // The launch itself must still count as claimed/in-flight, not failed — + // the agent is already running despite the persist failure. + if runs[0].Status != service.RunStatusClaimed { + t.Fatalf("run status = %q, want claimed (launch succeeded despite persist failure)", runs[0].Status) + } + + afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1") + if afterFailed != beforeFailed+1 { + t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed) + } + afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + if afterWithoutSession != beforeWithoutSession { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession) + } +} + +// ignoreSessionIDWrites installs a trigger that uses RAISE(IGNORE) +// rather than RAISE(ABORT): the UPDATE setting runs.session_id to +// wantSessionID matches zero rows and returns no error, the shape a run +// row that vanished between claim and launch would produce — a +// different failure mode than rejectSessionIDWrites' hard DB error. +func ignoreSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) { + t.Helper() + svc.ExecSQL(t, fmt.Sprintf(` + CREATE TRIGGER ignore_session_id_write + BEFORE UPDATE OF session_id ON runs + WHEN NEW.session_id = '%s' + BEGIN + SELECT RAISE(IGNORE); + END; + `, wantSessionID)) +} + +// AC-002.11: on a direct (non-routed) launch, a real, non-empty session +// id whose write affects zero rows (no error) must be counted as a +// session persist failure, not miscounted as "launch yielded no session +// id" (AC-002.8's counter). +func TestSchedulerTick_DirectLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) { + mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-vanished"} + svc := newTestService(t, service.ServiceOptions{TaskStarter: mock}) + ctx := context.Background() + + agent := &models.AgentInstance{ + ID: "direct-vanished-agent", + WorkspaceID: "ws-1", + Name: "direct-vanished-worker", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at) + VALUES ('task-direct-vanished-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-vanished-1"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + ignoreSessionIDWrites(t, svc, "sess-direct-vanished") + + beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1") + beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + + service.RunSchedulerTick(svc, ctx) + + if mock.callCount() != 1 { + t.Fatalf("expected 1 launch call, got %d", mock.callCount()) + } + + afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1") + if afterFailed != beforeFailed+1 { + t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed) + } + afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1") + if afterWithoutSession != beforeWithoutSession { + t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession) + } +} diff --git a/apps/backend/internal/office/service/terminal_shape_wiring_test.go b/apps/backend/internal/office/service/terminal_shape_wiring_test.go new file mode 100644 index 0000000000..2c18c5b5ca --- /dev/null +++ b/apps/backend/internal/office/service/terminal_shape_wiring_test.go @@ -0,0 +1,95 @@ +package service_test + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +func terminalShapeExpvarInt(t *testing.T, key string) int64 { + t.Helper() + v := expvar.Get("office_loop_terminal_total") + if v == nil { + t.Fatal("expvar map office_loop_terminal_total not registered") + } + m, ok := v.(*expvar.Map) + if !ok { + t.Fatal("office_loop_terminal_total is not a Map") + } + sub := m.Get(key) + if sub == nil { + return 0 + } + iv, ok := sub.(*expvar.Int) + if !ok { + t.Fatalf("office_loop_terminal_total[%q] is not an Int", key) + } + return iv.Value() +} + +// REQ-OFFICE-LOOP-LIVENESS-005: FinishRun classifies the just-finished +// run and increments office_loop_terminal_total by shape. A run +// finished with outcome=processed and no session id classifies as +// silent_success (activation is published at repo construction in +// these in-memory test DBs, so the run's requested_at is after it). +func TestFinishRun_RecordsSilentSuccessTerminalShape(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("terminal-shape-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: run=%v err=%v", run, err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeSilentSuccess)) + before := terminalShapeExpvarInt(t, key) + + if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("silent_success delta = %d, want 1", after-before) + } +} + +// A failed run with no session id classifies as unlaunched_failed. +func TestFailRun_RecordsUnlaunchedFailedTerminalShape(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("terminal-shape-fail-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: run=%v err=%v", run, err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if _, err := svc.FailRun(ctx, run.ID); err != nil { + t.Fatalf("fail: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/service/terminal_shapes.go b/apps/backend/internal/office/service/terminal_shapes.go new file mode 100644 index 0000000000..3fd216c6ca --- /dev/null +++ b/apps/backend/internal/office/service/terminal_shapes.go @@ -0,0 +1,108 @@ +package service + +import "time" + +// TerminalShape is the derived classification of a terminal run +// (REQ-OFFICE-LOOP-LIVENESS-005). It never widens runs.status or +// runs.outcome — see docs/specs/task-delivery-ledger/spec.md, which +// owns runs.outcome. +type TerminalShape string + +const ( + // ShapePreActivation is a run predating the point this capability's + // columns were confirmed present, or read before that point was ever + // published — never a defect (AC-005.6). + ShapePreActivation TerminalShape = "pre_activation" + // ShapeLaunchedCompleted is a run that reported success and named a + // session. + ShapeLaunchedCompleted TerminalShape = "launched_completed" + // ShapeLaunchedFailed is a run that reached a terminal failure and + // named a session. + ShapeLaunchedFailed TerminalShape = "launched_failed" + // ShapeSilentSuccess is a run whose outcome asserts work happened but + // which never named a session — NFR-1 made observable (AC-005.4). + ShapeSilentSuccess TerminalShape = "silent_success" + // ShapeUnlaunchedSkipped is a legitimate no-launch skip, kept apart + // from a launch failure (AC-005.5). + ShapeUnlaunchedSkipped TerminalShape = "unlaunched_skipped" + // ShapeUnlaunchedFailed is a terminal failure that never named a + // session. + ShapeUnlaunchedFailed TerminalShape = "unlaunched_failed" + // ShapeUnclassified is every combination none of the above predicates + // claim, including a null outcome, an outcome this document does not + // enumerate (such as the historical no_agent_launched), and an + // unknown status — the row that makes the classification total + // (AC-005.2). + ShapeUnclassified TerminalShape = "unclassified" +) + +var terminalFailedStatuses = map[string]bool{ + RunStatusFailed: true, + "timed_out": true, + RunStatusCancelled: true, +} + +var skipOutcomes = map[string]bool{ + RunOutcomeIdleSkipped: true, + RunOutcomeBudgetBlocked: true, + RunOutcomeBudgetUnmeasurable: true, + RunOutcomeAgentInactive: true, + RunOutcomeTaskTreeHeld: true, +} + +// ClassifyTerminalRun classifies one terminal run (status not in +// {queued, claimed} — AC-005.9 excludes those from every terminal +// total; they are the stuck-run detector's input instead) into exactly +// one TerminalShape. Reads only the four persisted values named in its +// signature, never a free-text message (AC-005.7), and evaluates +// top-to-bottom with first match winning — the ordering is load-bearing: +// pre_activation first so the activation guard cannot be bypassed by a +// later row matching, silent_success below launched_completed so the +// session test is what separates them, and above unclassified so a null +// outcome never absorbs it. +// +// activationInstant/activationPublished carry the boot-time probe result +// (persisted separately in kandev_meta); a run requested before +// activation, or read when activation was never published, is always +// pre_activation regardless of what it would otherwise classify as. +func ClassifyTerminalRun( + status string, + outcome *string, + sessionID string, + requestedAt time.Time, + activationInstant time.Time, + activationPublished bool, +) TerminalShape { + if !activationPublished || requestedAt.Before(activationInstant) { + return ShapePreActivation + } + + outcomeVal := "" + if outcome != nil { + outcomeVal = *outcome + } + return classifyByOutcome(status, outcomeVal, sessionID != "") +} + +// classifyByOutcome applies the first-match-wins predicate table once +// activation is confirmed, split out from ClassifyTerminalRun to keep +// each function's branching within the complexity limit. +func classifyByOutcome(status, outcomeVal string, launched bool) TerminalShape { + processed := outcomeVal == RunOutcomeProcessed + failedTerminal := terminalFailedStatuses[status] + + switch { + case status == RunStatusFinished && processed && launched: + return ShapeLaunchedCompleted + case failedTerminal && launched: + return ShapeLaunchedFailed + case status == RunStatusFinished && processed && !launched: + return ShapeSilentSuccess + case status == RunStatusFinished && !launched && skipOutcomes[outcomeVal]: + return ShapeUnlaunchedSkipped + case failedTerminal && !launched: + return ShapeUnlaunchedFailed + default: + return ShapeUnclassified + } +} diff --git a/apps/backend/internal/office/service/terminal_shapes_test.go b/apps/backend/internal/office/service/terminal_shapes_test.go new file mode 100644 index 0000000000..bcde9cbd97 --- /dev/null +++ b/apps/backend/internal/office/service/terminal_shapes_test.go @@ -0,0 +1,188 @@ +package service_test + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/office/service" +) + +func strp(s string) *string { return &s } + +// crossProductCase pins one (status, outcome, session presence) cell of +// the AC-005.2 cross-product to the exact shape the spec's predicate +// table assigns it — a literal expectation, not a second computation of +// ClassifyTerminalRun's own branching, so a misclassification (not just +// an out-of-set value) fails the test. +type crossProductCase struct { + status string + outcome *string + sessionID string + want service.TerminalShape +} + +// crossProductCases enumerates all 5 statuses x 8 outcomes x 2 session +// states (80 cells) this codebase's runs.status/outcome/session_id can +// hold, each mapped to its expected shape by hand from the spec table. +func crossProductCases() []crossProductCase { + outcomeCases := []*string{ + nil, + strp(service.RunOutcomeProcessed), + strp(service.RunOutcomeIdleSkipped), + strp(service.RunOutcomeBudgetBlocked), + strp(service.RunOutcomeBudgetUnmeasurable), + strp(service.RunOutcomeAgentInactive), + strp(service.RunOutcomeTaskTreeHeld), + strp("no_agent_launched"), // legacy value; not in current code + } + + cases := []crossProductCase{ + // status=finished: processed+session decides launched_completed + // vs. silent_success; a skip outcome decides unlaunched_skipped + // only without a session; everything else is unclassified. + {"finished", nil, "", service.ShapeUnclassified}, + {"finished", nil, "sess-1", service.ShapeUnclassified}, + {"finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess}, + {"finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted}, + {"finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped}, + {"finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified}, + {"finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped}, + {"finished", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified}, + {"finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped}, + {"finished", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified}, + {"finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped}, + {"finished", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified}, + {"finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped}, + {"finished", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified}, + {"finished", strp("no_agent_launched"), "", service.ShapeUnclassified}, + {"finished", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified}, + + // status=some_unknown_status: never finished, never a recognized + // failure status, so always unclassified regardless of outcome + // or session. + {"some_unknown_status", nil, "", service.ShapeUnclassified}, + {"some_unknown_status", nil, "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeProcessed), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnclassified}, + {"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified}, + {"some_unknown_status", strp("no_agent_launched"), "", service.ShapeUnclassified}, + {"some_unknown_status", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified}, + } + + // status IN (failed, cancelled, timed_out): the spec's predicate + // table names all three under one row and never reads outcome for + // them, so every outcome yields the same pair of shapes for each — + // unlaunched_failed without a session, launched_failed with one. + for _, status := range []string{"failed", "cancelled", "timed_out"} { + for _, outcome := range outcomeCases { + cases = append(cases, + crossProductCase{status, outcome, "", service.ShapeUnlaunchedFailed}, + crossProductCase{status, outcome, "sess-1", service.ShapeLaunchedFailed}, + ) + } + } + + return cases +} + +// AC-OFFICE-LOOP-LIVENESS-005.1/.2: classification is total over the +// cross-product of every persisted status this codebase writes +// (finished, failed, cancelled, timed_out, and an unknown status — +// runs.status is not the closed RunStatus enum) against every observed +// outcome (including the legacy no_agent_launched value and NULL), +// crossed with session_id present/absent — pinned to the exact shape +// each of the 80 cells must produce, not just closed-set membership. +func TestClassifyTerminalRun_CrossProduct(t *testing.T) { + activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + requestedAt := activation.Add(time.Hour) + + cases := crossProductCases() + if len(cases) != 5*8*2 { + t.Fatalf("cross-product case count = %d, want %d (5 statuses x 8 outcomes x 2 session states)", + len(cases), 5*8*2) + } + + for _, tc := range cases { + got := service.ClassifyTerminalRun( + tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true, + ) + if got != tc.want { + t.Errorf("ClassifyTerminalRun(status=%q, outcome=%v, session=%q) = %q, want %q", + tc.status, tc.outcome, tc.sessionID, got, tc.want) + } + } +} + +func TestClassifyTerminalRun_PreActivation(t *testing.T) { + activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + // Before the activation instant: pre_activation regardless of + // otherwise-silent-success shape (AC-005.6). + requestedBefore := activation.Add(-time.Minute) + got := service.ClassifyTerminalRun( + "finished", strp(service.RunOutcomeProcessed), "", requestedBefore, activation, true, + ) + if got != service.ShapePreActivation { + t.Fatalf("got %q, want pre_activation", got) + } + + // Activation instant never published: always pre_activation. + got = service.ClassifyTerminalRun( + "finished", strp(service.RunOutcomeProcessed), "", requestedBefore.Add(10*time.Hour), activation, false, + ) + if got != service.ShapePreActivation { + t.Fatalf("got %q, want pre_activation when unpublished", got) + } +} + +func TestClassifyTerminalRun_NamedShapes(t *testing.T) { + activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + requestedAt := activation.Add(time.Hour) + + cases := []struct { + name string + status string + outcome *string + sessionID string + want service.TerminalShape + }{ + {"launched_completed", "finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted}, + {"launched_failed/failed", "failed", nil, "sess-1", service.ShapeLaunchedFailed}, + {"launched_failed/timed_out", "timed_out", nil, "sess-1", service.ShapeLaunchedFailed}, + {"launched_failed/cancelled", "cancelled", nil, "sess-1", service.ShapeLaunchedFailed}, + {"silent_success", "finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess}, + {"unlaunched_skipped/idle", "finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped}, + {"unlaunched_skipped/budget", "finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped}, + {"unlaunched_skipped/budget_unmeasurable", "finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped}, + {"unlaunched_skipped/inactive", "finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped}, + {"unlaunched_skipped/tree_held", "finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped}, + {"unlaunched_failed", "failed", nil, "", service.ShapeUnlaunchedFailed}, + {"unclassified/null_outcome_finished", "finished", nil, "", service.ShapeUnclassified}, + {"unclassified/legacy_no_agent_launched", "finished", strp("no_agent_launched"), "", service.ShapeUnclassified}, + // A skip outcome carrying a session id is not a real production + // shape (skip outcomes are pre-launch guards) but the total + // classification must still place it somewhere, and it must not + // be reported as "unlaunched" while a session is on record + // (AC-005.3) — it falls to unclassified. + {"unclassified/skip_with_session", "finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified}, + {"unclassified/unknown_status", "some_unknown_status", nil, "", service.ShapeUnclassified}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := service.ClassifyTerminalRun(tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true) + if got != tc.want { + t.Fatalf("ClassifyTerminalRun(%q, %v, %q) = %q, want %q", + tc.status, tc.outcome, tc.sessionID, got, tc.want) + } + }) + } +} diff --git a/apps/backend/internal/office/service/test_helpers.go b/apps/backend/internal/office/service/test_helpers.go index 5c0e594b1d..4e295f94c3 100644 --- a/apps/backend/internal/office/service/test_helpers.go +++ b/apps/backend/internal/office/service/test_helpers.go @@ -205,6 +205,25 @@ func AdmitRunForTest(svc *Service, ctx context.Context, run *models.Run, agent * return si.admitRun(ctx, run, agent) } +// DeferWorkspaceLookupFailureForTest exposes deferWorkspaceLookupFailure +// directly for external test packages, so its MaxRetryCount-exhausted +// lost-race path (another writer already made the run terminal before +// this call's FailRun runs) can be driven without reproducing a transient +// GetAgentFromConfig lookup error through the full scheduler pipeline. +func DeferWorkspaceLookupFailureForTest(svc *Service, ctx context.Context, run *models.Run) { + si := &SchedulerIntegration{svc: svc, logger: svc.logger} + si.deferWorkspaceLookupFailure(ctx, run) +} + +// FailTasklessRunForTest exposes failTasklessRun directly for external test +// packages, so the lost-race path (another writer already made the run +// terminal before this call runs) can be driven without the scheduler's +// atomic claim+launch pipeline standing in the way. +func FailTasklessRunForTest(svc *Service, ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string) { + si := &SchedulerIntegration{svc: svc, logger: svc.logger} + si.failTasklessRun(ctx, run, agent, msg) +} + // LogPolicyObservabilityForTest exposes logPolicyObservability for external // test packages, so the skip/degraded-admitted entries and their // per-policy-per-day dedup (AC-OFFICE-BUDGET-002.13/-004.8) can be tested diff --git a/apps/backend/internal/office/service/tree_controls.go b/apps/backend/internal/office/service/tree_controls.go index c7a9026b7a..2f9eda00a5 100644 --- a/apps/backend/internal/office/service/tree_controls.go +++ b/apps/backend/internal/office/service/tree_controls.go @@ -55,9 +55,11 @@ func (s *Service) PauseTaskTree(ctx context.Context, rootTaskID string) (*models return nil, fmt.Errorf("create pause members: %w", err) } s.cancelTaskExecutions(ctx, members, "tree_paused") - if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused"); err != nil { + cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused") + if err != nil { s.logger.Warn("cancel runs for paused tree failed", zap.Error(err)) } + s.recordTerminalShapesForCancelledRuns(ctx, cancelled) s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold) return hold, nil } @@ -99,9 +101,11 @@ func (s *Service) CancelTaskTree(ctx context.Context, rootTaskID, cancelledBy st if err := s.repo.BulkReleaseTaskCheckout(ctx, memberTaskIDs(members)); err != nil { return nil, fmt.Errorf("release task checkouts: %w", err) } - if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled"); err != nil { + cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled") + if err != nil { s.logger.Warn("cancel runs for cancelled tree failed", zap.Error(err)) } + s.recordTerminalShapesForCancelledRuns(ctx, cancelled) _ = cancelledBy s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold) return hold, nil diff --git a/apps/backend/internal/office/service/tree_controls_test.go b/apps/backend/internal/office/service/tree_controls_test.go index ae5bd09424..5707f6b44e 100644 --- a/apps/backend/internal/office/service/tree_controls_test.go +++ b/apps/backend/internal/office/service/tree_controls_test.go @@ -61,6 +61,40 @@ func TestPauseTaskTree(t *testing.T) { } } +// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns is the Testing +// round 3 regression test. PauseTaskTree cancels every member task's runs +// via repo.CancelRunsForTasks, which bypassed office_loop_terminal_total +// entirely — the same bypass class Review round 1 fixed for the +// single-run CancelRun call sites. A queued run never launched, so +// pausing its tree must classify the cancellation as unlaunched_failed. +func TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) { + canceller := &mockTaskCanceller{} + svc := newTestService(t, service.ServiceOptions{TaskCanceller: canceller}) + ctx := context.Background() + insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS") + + agent := makeAgent("tree-pause-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"root"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if _, err := svc.PauseTaskTree(ctx, "root"); err != nil { + t.Fatalf("PauseTaskTree: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} + func TestCancelAndRestoreTaskTree(t *testing.T) { svc := newTestService(t) ctx := context.Background() @@ -106,3 +140,33 @@ func TestCancelAndRestoreTaskTree(t *testing.T) { t.Fatalf("active hold after restore = %+v, want nil; cancel hold was %s", preview.ActiveHold, hold.ID) } } + +// TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns mirrors +// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns for +// CancelTaskTree's own repo.CancelRunsForTasks call site. +func TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS") + + agent := makeAgent("tree-cancel-worker", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"root"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed)) + before := terminalShapeExpvarInt(t, key) + + if _, err := svc.CancelTaskTree(ctx, "root", "user:test"); err != nil { + t.Fatalf("CancelTaskTree: %v", err) + } + + after := terminalShapeExpvarInt(t, key) + if after != before+1 { + t.Fatalf("unlaunched_failed delta = %d, want 1", after-before) + } +} diff --git a/apps/backend/internal/office/testharness/routes_office.go b/apps/backend/internal/office/testharness/routes_office.go index 444560785b..b075f374be 100644 --- a/apps/backend/internal/office/testharness/routes_office.go +++ b/apps/backend/internal/office/testharness/routes_office.go @@ -189,7 +189,7 @@ func seedAgentFailureHandler( c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - if err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil { + if _, err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil { log.Error("test harness: mark run failed", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return diff --git a/apps/backend/internal/office/wakeup/causation_id_test.go b/apps/backend/internal/office/wakeup/causation_id_test.go new file mode 100644 index 0000000000..95ba3bd652 --- /dev/null +++ b/apps/backend/internal/office/wakeup/causation_id_test.go @@ -0,0 +1,199 @@ +package wakeup_test + +import ( + "context" + "testing" + + officemodels "github.com/kandev/kandev/internal/office/models" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/office/shared" + "github.com/kandev/kandev/internal/office/wakeup" +) + +func (h *testHarness) seedWakeupWithCausation(id, source, payload, causationID string) { + h.t.Helper() + if err := h.repo.CreateWakeupRequest(context.Background(), &officesqlite.WakeupRequest{ + ID: id, AgentProfileID: h.agentID, Source: source, Payload: payload, CausationID: causationID, + }); err != nil { + h.t.Fatalf("seed wakeup: %v", err) + } +} + +func (h *testHarness) seedRunWithReasonAndCausation(id, status, reason, causationID string) { + h.t.Helper() + run := &officemodels.Run{ + ID: id, + AgentProfileID: h.agentID, + Reason: reason, + Payload: "{}", + Status: officemodels.RunStatus(status), + CoalescedCount: 1, + ContextSnapshot: `{"prior":"snapshot"}`, + CausationID: causationID, + } + if err := h.repo.CreateRun(context.Background(), run); err != nil { + h.t.Fatalf("seed run: %v", err) + } +} + +// AC-OFFICE-LOOP-LIVENESS-002.3: createFreshRun copies the causation id +// from the wakeup request onto the fresh run. +func TestDispatch_FreshRunCopiesCausationIDFromWakeup(t *testing.T) { + h := newHarness(t, wakeup.PolicyCoalesceIfActive) + h.seedWakeupWithCausation("w-1", wakeup.SourceSelf, `{"reason":"test"}`, "cause-fresh-1") + + if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil { + t.Fatalf("dispatch: %v", err) + } + got, err := h.repo.GetWakeupRequest(context.Background(), "w-1") + if err != nil { + t.Fatalf("get wakeup request: %v", err) + } + run, err := h.repo.GetRunByID(context.Background(), got.RunID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if run.CausationID != "cause-fresh-1" { + t.Fatalf("run.CausationID = %q, want cause-fresh-1", run.CausationID) + } +} + +// AC-OFFICE-LOOP-LIVENESS-002.4: coalescing touches only the wakeup +// request, never the run. The in-flight run keeps the id of the wake +// that created it; the coalesced request keeps its own — the two stay +// joinable through agent_wakeup_requests.run_id. +func TestDispatch_CoalesceLeavesRunCausationIDUntouched(t *testing.T) { + h := newHarness(t, wakeup.PolicyCoalesceIfActive) + h.seedRunWithReasonAndCausation("run-pre", "queued", shared.RunReasonHeartbeat, "cause-run-original") + h.seedWakeupWithCausation("w-1", wakeup.SourceComment, `{"task_id":"t-1"}`, "cause-wakeup-coalesced") + + if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil { + t.Fatalf("dispatch: %v", err) + } + + gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-1") + if err != nil { + t.Fatalf("get wakeup request: %v", err) + } + if gotReq.CausationID != "cause-wakeup-coalesced" { + t.Fatalf("coalesced request CausationID = %q, want unchanged cause-wakeup-coalesced", gotReq.CausationID) + } + if gotReq.RunID != "run-pre" { + t.Fatalf("coalesced request run_id = %q, want run-pre", gotReq.RunID) + } + + gotRun, err := h.repo.GetRunByID(context.Background(), "run-pre") + if err != nil { + t.Fatalf("get run: %v", err) + } + if gotRun.CausationID != "cause-run-original" { + t.Fatalf("run.CausationID = %q, want unchanged cause-run-original (coalesce must not touch it)", gotRun.CausationID) + } +} + +// AC-002.4 skip half: MarkWakeupRequestSkipped touches only the +// request; it has no run to touch, and the request's own causation id +// survives the transition. +func TestDispatch_SkipLeavesWakeupCausationIDUntouched(t *testing.T) { + h := newHarness(t, wakeup.PolicyCoalesceIfActive) + // skip_if_active requires the routine lookup to resolve that policy; + // simplest deterministic path here is a non-routine source with an + // in-flight run under the default coalesce policy is not skip — so + // drive skip via the routine lookup fallback used elsewhere in this + // package (skip_if_active policy routine). + h.dispatcher.SetRoutineLookup(fakeRoutineLookupSkip{}) + h.seedRunWithReasonAndCausation("run-inflight", "claimed", shared.RunReasonRoutineDispatchCron, "cause-inflight") + h.seedWakeupWithCausation("w-skip", wakeup.SourceRoutine, `{"routine_id":"r-skip"}`, "cause-skip-1") + + if err := h.dispatcher.Dispatch(context.Background(), "w-skip"); err != nil { + t.Fatalf("dispatch: %v", err) + } + got, err := h.repo.GetWakeupRequest(context.Background(), "w-skip") + if err != nil { + t.Fatalf("get wakeup request: %v", err) + } + if got.Status != officesqlite.WakeupStatusSkipped { + t.Fatalf("status = %q, want skipped", got.Status) + } + if got.CausationID != "cause-skip-1" { + t.Fatalf("skipped request CausationID = %q, want unchanged cause-skip-1", got.CausationID) + } +} + +// AC-002.4/AC-002.5: PromoteRunAndCoalesceWakeupIfQueued promotes the +// run's reason but must never touch causation_id — a promotion is not a +// new cause. +func TestDispatch_PromotionDoesNotChangeRunCausationID(t *testing.T) { + h := newHarness(t, wakeup.PolicyCoalesceIfActive) + h.seedRunWithReasonAndCausation("run-cron", "queued", shared.RunReasonRoutineDispatchCron, "cause-cron-original") + h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-1") + // Give the event wakeup an event-classified reason so promotion fires. + if _, err := h.repo.ExecRaw(context.Background(), + `UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`, + shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil { + t.Fatalf("seed reason: %v", err) + } + + if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil { + t.Fatalf("dispatch: %v", err) + } + run, err := h.repo.GetRunByID(context.Background(), "run-cron") + if err != nil { + t.Fatalf("get run: %v", err) + } + if run.Reason != shared.RunReasonRoutineDispatchEvent { + t.Fatalf("run.Reason = %q, want promoted to %q", run.Reason, shared.RunReasonRoutineDispatchEvent) + } + if run.CausationID != "cause-cron-original" { + t.Fatalf("run.CausationID = %q, want unchanged cause-cron-original (promotion is not a new cause)", run.CausationID) + } +} + +// AC-002.3/AC-002.4: the lost-CAS fresh-run path (event coalescing into +// an already-claimed cron run) creates its own run, and that run +// carries the *requesting* wake's causation id, not the claimed run's. +func TestDispatch_LostCASFreshRunCarriesRequestingWakeCausationID(t *testing.T) { + h := newHarness(t, wakeup.PolicyCoalesceIfActive) + h.seedRunWithReasonAndCausation("run-cron", "claimed", shared.RunReasonRoutineDispatchCron, "cause-claimed-cron") + h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-fresh") + if _, err := h.repo.ExecRaw(context.Background(), + `UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`, + shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil { + t.Fatalf("seed reason: %v", err) + } + + if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil { + t.Fatalf("dispatch: %v", err) + } + gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-event") + if err != nil { + t.Fatalf("get wakeup request: %v", err) + } + if gotReq.RunID == "" || gotReq.RunID == "run-cron" { + t.Fatalf("expected a fresh run distinct from the claimed one, got %q", gotReq.RunID) + } + freshRun, err := h.repo.GetRunByID(context.Background(), gotReq.RunID) + if err != nil { + t.Fatalf("get fresh run: %v", err) + } + if freshRun.CausationID != "cause-event-fresh" { + t.Fatalf("fresh run.CausationID = %q, want cause-event-fresh (the requesting wake's id)", freshRun.CausationID) + } + + claimedRun, err := h.repo.GetRunByID(context.Background(), "run-cron") + if err != nil { + t.Fatalf("get claimed run: %v", err) + } + if claimedRun.CausationID != "cause-claimed-cron" { + t.Fatalf("claimed run.CausationID = %q, want unchanged cause-claimed-cron", claimedRun.CausationID) + } +} + +// fakeRoutineLookupSkip resolves every routine to skip_if_active so the +// skip transition can be driven deterministically without a real +// office_routines row. +type fakeRoutineLookupSkip struct{} + +func (fakeRoutineLookupSkip) GetRoutine(_ context.Context, id string) (*officemodels.Routine, error) { + return &officemodels.Routine{ID: id, ConcurrencyPolicy: officemodels.RoutineConcurrencyPolicy(wakeup.PolicySkipIfActive)}, nil +} diff --git a/apps/backend/internal/office/wakeup/dispatcher.go b/apps/backend/internal/office/wakeup/dispatcher.go index cd3a9bd6ee..603c9d2835 100644 --- a/apps/backend/internal/office/wakeup/dispatcher.go +++ b/apps/backend/internal/office/wakeup/dispatcher.go @@ -290,6 +290,10 @@ func (d *Dispatcher) createFreshRun( CoalescedCount: 1, ContextSnapshot: payload, RequestedAt: time.Now().UTC(), + // AC-OFFICE-LOOP-LIVENESS-002.3: copied from the requesting + // wake, including on the lost-CAS fresh-run path — that run + // still carries the requesting wake's id, not a new one. + CausationID: req.CausationID, } if err := d.repo.CreateRun(ctx, run); err != nil { return fmt.Errorf("create run for wakeup %s: %w", req.ID, err) diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index b315160e3b..a849581e81 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -1102,8 +1102,8 @@ type startTaskOptions struct { func (s *Service) StartTaskWithRoute( ctx context.Context, taskID, agentProfileID string, launch executor.LaunchContext, route executor.RouteOverride, -) error { - _, err := s.startTask(ctx, taskID, agentProfileID, +) (*executor.TaskExecution, error) { + return s.startTask(ctx, taskID, agentProfileID, launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt, launch.WorkflowStepID, launch.PlanMode, false, launch.Attachments, startTaskOptions{ @@ -1111,7 +1111,6 @@ func (s *Service) StartTaskWithRoute( AdditionalSkillSlugs: append([]string(nil), launch.AdditionalSkillSlugs...), Route: &route, }) - return err } func (s *Service) prepareExplicitWorkflowStartRoute( diff --git a/apps/backend/internal/runs/repository/sqlite/base_test.go b/apps/backend/internal/runs/repository/sqlite/base_test.go index e7dcf9648c..4faa43c1d8 100644 --- a/apps/backend/internal/runs/repository/sqlite/base_test.go +++ b/apps/backend/internal/runs/repository/sqlite/base_test.go @@ -225,7 +225,7 @@ func TestReadPathsUseTheReadOnlyHandle(t *testing.T) { } // Writes must not fall back to the reader. - if err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil { + if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil { t.Errorf("FinishRun succeeded with a closed writer; the write path is not using r.db") } } diff --git a/apps/backend/internal/runs/repository/sqlite/cancel.go b/apps/backend/internal/runs/repository/sqlite/cancel.go index 7b5904f698..d0729c4d6a 100644 --- a/apps/backend/internal/runs/repository/sqlite/cancel.go +++ b/apps/backend/internal/runs/repository/sqlite/cancel.go @@ -16,54 +16,81 @@ import ( // and have its real finished_at overwritten. const cancellableRunStatuses = `status IN ('queued', 'claimed')` +// CancelledRun identifies one row CancelRunsWhere actually cancelled, +// carrying the fields a caller needs to classify the transition's +// terminal shape (office_loop_terminal_total) without a second read. +type CancelledRun struct { + ID string + AgentProfileID string + SessionID string + RequestedAt time.Time +} + // CancelRunsWhere is the single writer of the terminal cancel state on the // runs table. CancelRun, BulkCancelRuns and the office repository's -// CancelRunsForTasks are all thin selectors over it, so the guard above -// cannot be bypassed by picking a different entry point. +// CancelRunsForTasks and CancelDisplacedParticipantRun are all thin +// selectors over it, so the guard above cannot be bypassed by picking a +// different entry point. // // selector is an extra WHERE fragment ANDed onto the guard; it must be a // SQL literal owned by the calling repository method, using ? placeholders // (rebound here) for every value. Never interpolate caller-supplied data // into it. selectorArgs bind those placeholders, in order. // -// Returns the number of rows actually cancelled: 0 means every run the -// selector matched had already reached a terminal state. +// Returns the rows actually cancelled: an empty slice means every run the +// selector matched had already reached a terminal state. The guard and the +// write are the same UPDATE statement (RETURNING reads the classification +// columns off the rows it just changed), so there is no separate read a +// row's status could change behind: a run that reaches a terminal state on +// another connection is either fully visible to this statement or not, and +// either way it cannot be cancelled by it. func (r *Repository) CancelRunsWhere( ctx context.Context, cancelReason string, selector string, selectorArgs ...interface{}, -) (int64, error) { - args := make([]interface{}, 0, len(selectorArgs)+2) - args = append(args, cancelReason, time.Now().UTC()) - args = append(args, selectorArgs...) - +) ([]CancelledRun, error) { query := fmt.Sprintf(` UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE %s AND (%s) + RETURNING id, agent_profile_id, session_id, requested_at `, cancellableRunStatuses, selector) - - res, err := r.db.ExecContext(ctx, r.db.Rebind(query), args...) + args := append([]interface{}{cancelReason, time.Now().UTC()}, selectorArgs...) + rows, err := r.db.QueryxContext(ctx, r.db.Rebind(query), args...) if err != nil { - return 0, err + return nil, err } - return res.RowsAffected() + defer rows.Close() //nolint:errcheck + + var cancelled []CancelledRun + for rows.Next() { + var row CancelledRun + if err := rows.Scan(&row.ID, &row.AgentProfileID, &row.SessionID, &row.RequestedAt); err != nil { + return nil, err + } + cancelled = append(cancelled, row) + } + return cancelled, rows.Err() } // CancelRun marks a run as cancelled with an optional cancel reason. -// A run that already reached a terminal state is left untouched. -func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) error { - _, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id) - return err +// A run that already reached a terminal state is left untouched. Returns +// whether this call actually cancelled the row, so a caller counting the +// transition (office_loop_terminal_total) can tell a persisted change +// from a no-op on a run that was already terminal. +func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) (bool, error) { + rows, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id) + return len(rows) > 0, err } -// BulkCancelRuns cancels multiple runs by ID with the given reason. -// Runs in the set that already reached a terminal state are left untouched. -func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) error { +// BulkCancelRuns cancels multiple runs by ID with the given reason, +// returning the rows actually cancelled (see CancelRunsWhere). Runs in +// the set that already reached a terminal state are left untouched. +func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) ([]CancelledRun, error) { if len(ids) == 0 { - return nil + return nil, nil } placeholders := make([]string, len(ids)) args := make([]interface{}, len(ids)) @@ -72,6 +99,5 @@ func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelRea args[i] = id } selector := fmt.Sprintf(`id IN (%s)`, strings.Join(placeholders, ",")) - _, err := r.CancelRunsWhere(ctx, cancelReason, selector, args...) - return err + return r.CancelRunsWhere(ctx, cancelReason, selector, args...) } diff --git a/apps/backend/internal/runs/repository/sqlite/cancel_postgres_test.go b/apps/backend/internal/runs/repository/sqlite/cancel_postgres_test.go new file mode 100644 index 0000000000..65ac389b5c --- /dev/null +++ b/apps/backend/internal/runs/repository/sqlite/cancel_postgres_test.go @@ -0,0 +1,139 @@ +package sqlite_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/kandev/kandev/internal/office/models" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun is +// the two-connection regression for a TOCTOU CancelRunsWhere used to have: +// its cancellable-status guard was applied only to the candidate read, not +// to the write, so a run that genuinely finished on another connection +// between the two could be silently overwritten to cancelled. Real +// Postgres row locking forces the interleaving this needs: one connection +// holds an uncommitted finish on the row, CancelRunsWhere's write blocks on +// that row lock, and only proceeds once the finish commits — at which +// point it must see the fresh 'finished' status and cancel nothing. Skips +// unless KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun(t *testing.T) { + dsn := testutil.PostgresDSNFromEnv(t) + // db is the repo's own pool: OpenIsolatedPostgres pins it to exactly + // one physical connection and sets that connection's search_path to a + // fresh schema, so every statement through it (including the schema + // init below and repo.CancelRunsWhere's own transaction) lands there. + db := testutil.OpenIsolatedPostgres(t, dsn) + + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + officeRepo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init office repo: %v", err) + } + repo := officeRepo.RunsRepository() + ctx := context.Background() + + // finisherDB is a second, genuinely separate physical connection to + // the SAME schema. A pool-wide MaxOpenConns bump on a single *sqlx.DB + // would not do this safely: search_path is a per-connection session + // setting, so a new connection the pool opens under load would not + // inherit it and could silently operate against the wrong schema. + var schema string + if err := db.Get(&schema, "SELECT current_schema()"); err != nil { + t.Fatalf("read isolated schema name: %v", err) + } + finisherDB, err := sqlx.Open("pgx", dsn) + if err != nil { + t.Fatalf("open finisher connection: %v", err) + } + finisherDB.SetMaxOpenConns(1) + finisherDB.SetMaxIdleConns(1) + t.Cleanup(func() { _ = finisherDB.Close() }) + if _, err := finisherDB.Exec("SET search_path TO " + schema); err != nil { + t.Fatalf("set finisher search_path: %v", err) + } + + run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + now := time.Now().UTC() + if _, err := db.ExecContext(ctx, db.Rebind( + `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, + ), now, run.ID); err != nil { + t.Fatalf("seed claimed: %v", err) + } + + tx, err := finisherDB.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin finisher tx: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, finisherDB.Rebind( + `UPDATE runs SET status = 'finished', outcome = 'processed', finished_at = ? WHERE id = ?`, + ), now, run.ID); err != nil { + t.Fatalf("finisher update (holds the row lock, uncommitted): %v", err) + } + + var ( + wg sync.WaitGroup + cancelled []runssqlite.CancelledRun + cancelErr error + cancelDone = make(chan struct{}) + commitErr error + ) + wg.Add(1) + go func() { + defer wg.Done() + defer close(cancelDone) + cancelled, cancelErr = repo.CancelRunsWhere(ctx, "too late", `id = ?`, run.ID) + }() + + // Give CancelRunsWhere's write time to reach Postgres and block on the + // row lock the finisher's uncommitted transaction holds, before that + // transaction commits. Without this window the two goroutines could + // run in either order and the test would not exercise the lock-wait + // path this bug depends on. + select { + case <-cancelDone: + t.Fatal("CancelRunsWhere returned before the finisher committed; it never blocked on the row lock") + case <-time.After(200 * time.Millisecond): + } + + commitErr = tx.Commit() + wg.Wait() + if commitErr != nil { + t.Fatalf("finisher commit: %v", commitErr) + } + if cancelErr != nil { + t.Fatalf("cancel where: %v", cancelErr) + } + + if len(cancelled) != 0 { + t.Fatalf("cancelled = %d, want 0: a run that committed finished must not be reported cancelled", len(cancelled)) + } + if got := mustGetRunStatus(t, db, run.ID); got != "finished" { + t.Fatalf("run status = %q, want finished (the concurrent cancel must not have overwritten it)", got) + } +} + +func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string { + t.Helper() + var status string + if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil { + t.Fatalf("get run status: %v", err) + } + return status +} diff --git a/apps/backend/internal/runs/repository/sqlite/cancel_test.go b/apps/backend/internal/runs/repository/sqlite/cancel_test.go index dd3b251080..57c1dfc376 100644 --- a/apps/backend/internal/runs/repository/sqlite/cancel_test.go +++ b/apps/backend/internal/runs/repository/sqlite/cancel_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kandev/kandev/internal/office/models" + runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite" ) // TestCancelRun_CancelsQueuedAndClaimedRuns pins every column the @@ -21,9 +22,13 @@ func TestCancelRun_CancelsQueuedAndClaimedRuns(t *testing.T) { setStatus(t, repo, claimed.ID, "claimed", timePtr(now), nil) for _, id := range []string{queued.ID, claimed.ID} { - if err := repo.CancelRun(ctx, id, "user stopped the task"); err != nil { + cancelled, err := repo.CancelRun(ctx, id, "user stopped the task") + if err != nil { t.Fatalf("cancel %q: %v", id, err) } + if !cancelled { + t.Errorf("%s: cancelled = false, want true", id) + } got := mustGetRun(t, repo, id) checkString(t, id+" status", string(got.Status), "cancelled") checkStringPtr(t, id+" cancel_reason", got.CancelReason, strPtr("user stopped the task")) @@ -47,9 +52,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) { run := queueRunAt(t, repo, "run-"+status, "a1", finishedAt.Add(-time.Hour)) setStatus(t, repo, run.ID, status, nil, timePtr(finishedAt)) - if err := repo.CancelRun(ctx, run.ID, "too late"); err != nil { + cancelled, err := repo.CancelRun(ctx, run.ID, "too late") + if err != nil { t.Fatalf("cancel %q: %v", run.ID, err) } + if cancelled { + t.Errorf("%s: cancelled = true, want false (already terminal)", run.ID) + } got := mustGetRun(t, repo, run.ID) checkString(t, run.ID+" status", string(got.Status), status) if got.CancelReason != nil { @@ -64,9 +73,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) { // TestCancelRun_UnknownRunIsANoOp pins that a missing id is not an error. func TestCancelRun_UnknownRunIsANoOp(t *testing.T) { repo := newTestRepo(t) - if err := repo.CancelRun(context.Background(), "nope", "reason"); err != nil { + cancelled, err := repo.CancelRun(context.Background(), "nope", "reason") + if err != nil { t.Fatalf("cancel unknown run: %v", err) } + if cancelled { + t.Error("cancelled = true, want false (unknown run)") + } } // TestCancelRunsWhere_ReportsRowsActuallyCancelled pins the returned @@ -87,8 +100,8 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) { if err != nil { t.Fatalf("cancel where: %v", err) } - if cancelled != 2 { - t.Errorf("cancelled = %d, want 2", cancelled) + if len(cancelled) != 2 { + t.Errorf("cancelled = %d, want 2", len(cancelled)) } checkString(t, "q1", string(mustGetRun(t, repo, "q1").Status), "cancelled") checkString(t, "c1", string(mustGetRun(t, repo, "c1").Status), "cancelled") @@ -101,8 +114,71 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) { if err != nil { t.Fatalf("cancel where (second pass): %v", err) } - if cancelled != 0 { - t.Errorf("second pass cancelled = %d, want 0", cancelled) + if len(cancelled) != 0 { + t.Errorf("second pass cancelled = %d, want 0", len(cancelled)) + } +} + +// TestCancelRunsWhere_ReturnedRowsCarryClassificationFields pins the +// fields a caller needs to classify each cancelled run's loop-liveness +// terminal shape (office_loop_terminal_total) without a second read: the +// launched run's session_id must survive onto the returned row, distinct +// from the never-launched run's empty one. +func TestCancelRunsWhere_ReturnedRowsCarryClassificationFields(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + requestedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + + launched := mustCreateRun(t, repo, &models.Run{ + ID: "launched", AgentProfileID: "a1", Reason: "task_assigned", + Payload: `{}`, Status: "claimed", + }) + if err := repo.SetRunRequestedAtForTest(ctx, launched.ID, requestedAt); err != nil { + t.Fatalf("backdate requested_at: %v", err) + } + if _, err := repo.SetRunSessionID(ctx, launched.ID, "session-in-flight"); err != nil { + t.Fatalf("set session id: %v", err) + } + neverLaunched := mustCreateRun(t, repo, &models.Run{ + ID: "never-launched", AgentProfileID: "a2", Reason: "task_assigned", + Payload: `{}`, Status: "queued", + }) + if err := repo.SetRunRequestedAtForTest(ctx, neverLaunched.ID, requestedAt); err != nil { + t.Fatalf("backdate requested_at: %v", err) + } + + cancelled, err := repo.CancelRunsWhere(ctx, "reason", + `id IN (?, ?)`, launched.ID, neverLaunched.ID) + if err != nil { + t.Fatalf("cancel where: %v", err) + } + if len(cancelled) != 2 { + t.Fatalf("cancelled = %d, want 2", len(cancelled)) + } + + byID := map[string]runssqlite.CancelledRun{} + for _, row := range cancelled { + byID[row.ID] = row + } + got, ok := byID[launched.ID] + if !ok { + t.Fatalf("launched run missing from cancelled rows: %v", cancelled) + } + if got.AgentProfileID != "a1" { + t.Errorf("launched agent_profile_id = %q, want a1", got.AgentProfileID) + } + if got.SessionID != "session-in-flight" { + t.Errorf("launched session_id = %q, want session-in-flight", got.SessionID) + } + if !got.RequestedAt.Equal(requestedAt) { + t.Errorf("launched requested_at = %v, want %v", got.RequestedAt, requestedAt) + } + got, ok = byID[neverLaunched.ID] + if !ok { + t.Fatalf("never-launched run missing from cancelled rows: %v", cancelled) + } + if got.SessionID != "" { + t.Errorf("never-launched session_id = %q, want empty", got.SessionID) } } @@ -126,8 +202,8 @@ func TestCancelRunsWhere_MultiArgSelectorBindsInOrder(t *testing.T) { if err != nil { t.Fatalf("cancel where: %v", err) } - if cancelled != 1 { - t.Errorf("cancelled = %d, want 1", cancelled) + if len(cancelled) != 1 { + t.Errorf("cancelled = %d, want 1", len(cancelled)) } checkString(t, "match", string(mustGetRun(t, repo, "match").Status), "cancelled") checkString(t, "wrong-reason", string(mustGetRun(t, repo, "wrong-reason").Status), "queued") @@ -148,12 +224,15 @@ func TestBulkCancelRuns_CancelsOnlyTheListedEligibleRuns(t *testing.T) { setStatus(t, repo, finished.ID, "finished", nil, timePtr(now)) queueRunAt(t, repo, "out-of-set", "a1", now) - err := repo.BulkCancelRuns(ctx, + cancelled, err := repo.BulkCancelRuns(ctx, []string{"in-set-queued", "in-set-claimed", "in-set-finished", "missing-id"}, "workspace deleted") if err != nil { t.Fatalf("bulk cancel: %v", err) } + if len(cancelled) != 2 { + t.Errorf("cancelled = %d, want 2 (queued + claimed, not the finished run)", len(cancelled)) + } checkString(t, "in-set-queued", string(mustGetRun(t, repo, "in-set-queued").Status), "cancelled") checkString(t, "in-set-claimed", string(mustGetRun(t, repo, "in-set-claimed").Status), "cancelled") @@ -174,9 +253,13 @@ func TestBulkCancelRuns_EmptyIDListIsANoOp(t *testing.T) { queueRunAt(t, repo, "survivor", "a1", time.Now().UTC()) for _, ids := range [][]string{nil, {}} { - if err := repo.BulkCancelRuns(ctx, ids, "nothing"); err != nil { + cancelled, err := repo.BulkCancelRuns(ctx, ids, "nothing") + if err != nil { t.Fatalf("bulk cancel %v: %v", ids, err) } + if len(cancelled) != 0 { + t.Errorf("bulk cancel %v: cancelled = %d, want 0", ids, len(cancelled)) + } } got := mustGetRun(t, repo, "survivor") checkString(t, "status", string(got.Status), "queued") diff --git a/apps/backend/internal/runs/repository/sqlite/errors_test.go b/apps/backend/internal/runs/repository/sqlite/errors_test.go index 35ea307d50..31fd613cb6 100644 --- a/apps/backend/internal/runs/repository/sqlite/errors_test.go +++ b/apps/backend/internal/runs/repository/sqlite/errors_test.go @@ -116,7 +116,8 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) { return repo.UpdateRunOutputSummary(ctx, run.ID, "o", "f") }, "FinishRun": func() error { - return repo.FinishRun(ctx, run.ID, "finished", nil) + _, err := repo.FinishRun(ctx, run.ID, "finished", nil) + return err }, "ClaimRun": func() error { _, err := repo.ClaimRun(ctx, "a1") @@ -146,10 +147,12 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) { return err }, "CancelRun": func() error { - return repo.CancelRun(ctx, run.ID, "reason") + _, err := repo.CancelRun(ctx, run.ID, "reason") + return err }, "BulkCancelRuns": func() error { - return repo.BulkCancelRuns(ctx, []string{run.ID}, "reason") + _, err := repo.BulkCancelRuns(ctx, []string{run.ID}, "reason") + return err }, "AppendRunEvent": func() error { _, err := repo.AppendRunEvent(ctx, run.ID, "run.progress", "info", "{}") diff --git a/apps/backend/internal/runs/repository/sqlite/finish_run_postgres_test.go b/apps/backend/internal/runs/repository/sqlite/finish_run_postgres_test.go new file mode 100644 index 0000000000..a3f0650abe --- /dev/null +++ b/apps/backend/internal/runs/repository/sqlite/finish_run_postgres_test.go @@ -0,0 +1,122 @@ +package sqlite_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/kandev/kandev/internal/office/models" + officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun is the +// two-connection regression for Review round 3, R3-1: FinishRun's write +// used to carry no status guard at all, so a run cancelled on another +// connection between the caller's read and this write would have its +// 'cancelled' status and real finished_at silently overwritten to +// 'finished'/'failed'. Real Postgres row locking forces the interleaving +// this needs: one connection holds an uncommitted cancel on the row, +// FinishRun's write blocks on that row lock, and only proceeds once the +// cancel commits — at which point the guard (status = 'claimed') must see +// the fresh 'cancelled' status and write nothing. Skips unless +// KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) { + dsn := testutil.PostgresDSNFromEnv(t) + db := testutil.OpenIsolatedPostgres(t, dsn) + + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + officeRepo, err := officesqlite.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init office repo: %v", err) + } + repo := officeRepo.RunsRepository() + ctx := context.Background() + + var schema string + if err := db.Get(&schema, "SELECT current_schema()"); err != nil { + t.Fatalf("read isolated schema name: %v", err) + } + cancellerDB, err := sqlx.Open("pgx", dsn) + if err != nil { + t.Fatalf("open canceller connection: %v", err) + } + cancellerDB.SetMaxOpenConns(1) + cancellerDB.SetMaxIdleConns(1) + t.Cleanup(func() { _ = cancellerDB.Close() }) + if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil { + t.Fatalf("set canceller search_path: %v", err) + } + + run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"} + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + now := time.Now().UTC() + if _, err := db.ExecContext(ctx, db.Rebind( + `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, + ), now, run.ID); err != nil { + t.Fatalf("seed claimed: %v", err) + } + + tx, err := cancellerDB.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin canceller tx: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, cancellerDB.Rebind( + `UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`, + ), "too late", now, run.ID); err != nil { + t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err) + } + + var ( + wg sync.WaitGroup + finishedRun *models.Run + finishErr error + finishDone = make(chan struct{}) + commitErr error + ) + outcome := "processed" + wg.Add(1) + go func() { + defer wg.Done() + defer close(finishDone) + finishedRun, finishErr = repo.FinishRun(ctx, run.ID, "finished", &outcome) + }() + + // Give FinishRun's write time to reach Postgres and block on the row + // lock the canceller's uncommitted transaction holds, before that + // transaction commits. Without this window the two goroutines could + // run in either order and the test would not exercise the lock-wait + // path this bug depends on. + select { + case <-finishDone: + t.Fatal("FinishRun returned before the canceller committed; it never blocked on the row lock") + case <-time.After(200 * time.Millisecond): + } + + commitErr = tx.Commit() + wg.Wait() + if commitErr != nil { + t.Fatalf("canceller commit: %v", commitErr) + } + if finishErr != nil { + t.Fatalf("finish run: %v", finishErr) + } + + if finishedRun != nil { + t.Fatalf("finished run = %+v, want nil: a run that committed cancelled must not be reported finished", finishedRun) + } + if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" { + t.Fatalf("run status = %q, want cancelled (the concurrent finish must not have overwritten it)", got) + } +} diff --git a/apps/backend/internal/runs/repository/sqlite/runs.go b/apps/backend/internal/runs/repository/sqlite/runs.go index 5dc73a3b0f..c44aa584db 100644 --- a/apps/backend/internal/runs/repository/sqlite/runs.go +++ b/apps/backend/internal/runs/repository/sqlite/runs.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "strings" "time" @@ -41,13 +42,13 @@ func (r *Repository) CreateRunTx(ctx context.Context, tx *sqlx.Tx, req *models.R id, agent_profile_id, reason, payload, status, coalesced_count, idempotency_key, context_snapshot, capabilities, input_snapshot, output_summary, failure_reason, session_id, retry_count, scheduled_retry_at, - requested_at, error_message, cancel_reason, continuation_scope - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + requested_at, error_message, cancel_reason, continuation_scope, causation_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), req.ID, req.AgentProfileID, req.Reason, req.Payload, req.Status, req.CoalescedCount, req.IdempotencyKey, req.ContextSnapshot, req.Capabilities, req.InputSnapshot, req.OutputSummary, req.FailureReason, req.SessionID, req.RetryCount, req.ScheduledRetryAt, req.RequestedAt, - req.ErrorMessage, req.CancelReason, req.ContinuationScope) + req.ErrorMessage, req.CancelReason, req.ContinuationScope, req.CausationID) return err } @@ -103,6 +104,35 @@ func (r *Repository) UpdateRunRuntimeSnapshot( return err } +// SetRunSessionID persists the session id a launch produced. A no-op +// when sessionID is empty — the caller counts that as a without-session +// launch rather than clobbering whatever the column already held. +// Guarded to status = 'claimed' so a launch that loses a race against a +// concurrent cancel or terminal write cannot mutate an already-terminal +// row's session id after the fact: without the guard, a terminal-shape +// classification already recorded off the empty session id would drift +// from what the row shows on a later read. Returns whether the row was +// still claimed so the caller can distinguish a real write from a +// stale run id or a lost race. +func (r *Repository) SetRunSessionID( + ctx context.Context, runID, sessionID string, +) (bool, error) { + if sessionID == "" { + return false, nil + } + res, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE runs SET session_id = ? WHERE id = ? AND status = 'claimed' + `), sessionID, runID) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + if err != nil { + return false, err + } + return n > 0, nil +} + // UpdateRunPromptArtifacts persists the assembled prompt the agent // received and the continuation-summary content prepended at dispatch. // Called from the scheduler-integration after BuildAgentPrompt completes @@ -189,12 +219,32 @@ func (r *Repository) ClaimRun(ctx context.Context, agentInstanceID string) (*mod // outcome from a different transition. outcome is nil for the failed path // and for callers with no established semantic label (docs/specs/ // task-delivery-ledger/spec.md, "Office run outcome"). -func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) error { +// FinishRun writes the terminal status/outcome and returns the row as it +// stands immediately after that write, via the same statement (RETURNING), +// so a caller classifying the transition (office_loop_terminal_total) never +// depends on a separate read succeeding independently of the write that +// persisted it. Guarded to status = 'claimed', the same status every caller +// reaches this from (ClaimNextEligibleRun then processRun/an event +// subscriber): without the guard, a cancel that commits between the +// caller's read and this write would have its 'cancelled' status and +// finished_at overwritten by this transition. Returns (nil, nil) for an +// unknown id or a run no longer claimed (already terminal by another +// writer): zero rows changed, so there is nothing to classify. +func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) (*models.Run, error) { now := time.Now().UTC() - _, err := r.db.ExecContext(ctx, r.db.Rebind(` - UPDATE runs SET status = ?, outcome = ?, finished_at = ? WHERE id = ? - `), status, outcome, now, id) - return err + var run models.Run + err := r.db.QueryRowxContext(ctx, r.db.Rebind(` + UPDATE runs SET status = ?, outcome = ?, finished_at = ? + WHERE id = ? AND status = 'claimed' + RETURNING * + `), status, outcome, now, id).StructScan(&run) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &run, nil } // GetRunByID returns the run row for a given ID. Returns sql.ErrNoRows when unknown. diff --git a/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go b/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go index ba7036b547..ab5d2f322e 100644 --- a/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go +++ b/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go @@ -275,7 +275,7 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) { before := time.Now().UTC().Add(-time.Second) run := mustCreateRun(t, repo, fullRun()) - if err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil { + if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil { t.Fatalf("finish run: %v", err) } got := mustGetRun(t, repo, run.ID) @@ -288,6 +288,35 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) { } } +// TestFinishRun_ReturnsFullRowWithTheReaderClosed pins the property a +// terminal-shape classifier depends on: FinishRun's own RETURNING +// clause carries every field needed to classify the transition (here, +// SessionID), so a caller never needs a separate read through r.ro to +// get it. +func TestFinishRun_ReturnsFullRowWithTheReaderClosed(t *testing.T) { + repo, _, reader := newTestRepoWithHandles(t) + ctx := context.Background() + run := mustCreateRun(t, repo, fullRun()) + + if err := reader.Close(); err != nil { + t.Fatalf("close reader: %v", err) + } + + got, err := repo.FinishRun(ctx, run.ID, "finished", nil) + if err != nil { + t.Fatalf("finish run with the reader closed: %v", err) + } + if got == nil { + t.Fatal("finish run returned a nil row") + } + checkString(t, "status", string(got.Status), "finished") + checkString(t, "session_id", got.SessionID, "sess-7") + checkString(t, "agent_profile_id", got.AgentProfileID, "agent-42") + if got.FinishedAt == nil { + t.Fatalf("finished_at = nil, want a stamp") + } +} + // TestSetRunStatusForTest_WritesStatusAndBothTimestamps covers the // harness affordance used by the E2E seeder, including its NULL case. func TestSetRunStatusForTest_WritesStatusAndBothTimestamps(t *testing.T) { diff --git a/apps/backend/internal/runs/repository/sqlite/runs_session_id_test.go b/apps/backend/internal/runs/repository/sqlite/runs_session_id_test.go new file mode 100644 index 0000000000..eb68e38134 --- /dev/null +++ b/apps/backend/internal/runs/repository/sqlite/runs_session_id_test.go @@ -0,0 +1,117 @@ +package sqlite_test + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/office/models" +) + +// AC-OFFICE-LOOP-LIVENESS-002.7/.8/.10: SetRunSessionID persists the +// session id a launch produced, is a no-op on an empty id (so a caller +// can distinguish "wrote nothing" from a real write), and — given the +// single-launch-in-flight invariant — an unconditional non-empty write +// is "last non-empty wins". +func TestSetRunSessionID_PersistsNonEmptyID(t *testing.T) { + repo := newTestRepo(t) + run := &models.Run{ID: "run-session-1", AgentProfileID: "agent-1", Status: "claimed"} + mustCreateRun(t, repo, run) + + wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-abc") + if err != nil { + t.Fatalf("SetRunSessionID: %v", err) + } + if !wrote { + t.Fatal("expected wrote=true for existing run with non-empty session id") + } + + got := mustGetRun(t, repo, run.ID) + if got.SessionID != "sess-abc" { + t.Fatalf("session_id = %q, want sess-abc", got.SessionID) + } +} + +func TestSetRunSessionID_EmptyIDIsNoop(t *testing.T) { + repo := newTestRepo(t) + run := &models.Run{ID: "run-session-2", AgentProfileID: "agent-1", Status: "claimed", SessionID: "sess-existing"} + mustCreateRun(t, repo, run) + + wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "") + if err != nil { + t.Fatalf("SetRunSessionID: %v", err) + } + if wrote { + t.Fatal("expected wrote=false for empty session id") + } + + got := mustGetRun(t, repo, run.ID) + if got.SessionID != "sess-existing" { + t.Fatalf("session_id = %q, want unchanged sess-existing", got.SessionID) + } +} + +func TestSetRunSessionID_UnknownRunIsNoop(t *testing.T) { + repo := newTestRepo(t) + + wrote, err := repo.SetRunSessionID(context.Background(), "run-does-not-exist", "sess-x") + if err != nil { + t.Fatalf("SetRunSessionID: %v", err) + } + if wrote { + t.Fatal("expected wrote=false for unknown run id") + } +} + +// A cancel that lands between claim and this write already recorded a +// terminal shape off the run's session id at that instant; letting this +// write land anyway would make the persisted row disagree with the +// classification that was already counted for it. +func TestSetRunSessionID_LostRaceAgainstConcurrentCancelIsNoop(t *testing.T) { + repo := newTestRepo(t) + run := &models.Run{ID: "run-session-4", AgentProfileID: "agent-1", Status: "claimed"} + mustCreateRun(t, repo, run) + + cancelled, err := repo.CancelRun(context.Background(), run.ID, "lost_race") + if err != nil { + t.Fatalf("CancelRun: %v", err) + } + if !cancelled { + t.Fatal("expected CancelRun to cancel the still-claimed run") + } + + wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-late") + if err != nil { + t.Fatalf("SetRunSessionID: %v", err) + } + if wrote { + t.Fatal("expected wrote=false once the run is no longer claimed") + } + + got := mustGetRun(t, repo, run.ID) + if got.SessionID != "" { + t.Fatalf("session_id = %q, want unchanged empty string on the cancelled row", got.SessionID) + } + if got.Status != "cancelled" { + t.Fatalf("status = %q, want cancelled", got.Status) + } +} + +// AC-OFFICE-LOOP-LIVENESS-002.10: a later, non-empty write overwrites an +// earlier one — "last non-empty wins" for the relaunch-keeps-latest case. +func TestSetRunSessionID_LaterWriteOverwritesEarlier(t *testing.T) { + repo := newTestRepo(t) + run := &models.Run{ID: "run-session-3", AgentProfileID: "agent-1", Status: "claimed"} + mustCreateRun(t, repo, run) + + if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-first"); err != nil { + t.Fatalf("first SetRunSessionID: %v", err) + } + if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-second"); err != nil { + t.Fatalf("second SetRunSessionID: %v", err) + } + + got := mustGetRun(t, repo, run.ID) + if got.SessionID != "sess-second" { + t.Fatalf("session_id = %q, want sess-second", got.SessionID) + } +} diff --git a/docs/plans/office-loop-liveness/plan.md b/docs/plans/office-loop-liveness/plan.md new file mode 100644 index 0000000000..0b2f679595 --- /dev/null +++ b/docs/plans/office-loop-liveness/plan.md @@ -0,0 +1,75 @@ +--- +created: 2026-09-07 +status: done +requirements: + - REQ-OFFICE-LOOP-LIVENESS-001 + - REQ-OFFICE-LOOP-LIVENESS-002 + - REQ-OFFICE-LOOP-LIVENESS-003 + - REQ-OFFICE-LOOP-LIVENESS-004 + - REQ-OFFICE-LOOP-LIVENESS-005 +system_design: + - ../../specs/office/system-design/loop-liveness.md +legacy_specs: [] +--- + +# Implementation Plan: Office Loop Liveness + +## Overview + +Office's unattended loop (cron tick -> trigger claim -> routine run -> wake -> +`runs` row -> claim -> launch -> terminal) has no surface that says whether it +ran. On the instance this card was written against, the loop had been dead +since 2026-08-05 and nothing surfaced it; discovering that took a code read +plus hand-written SQL. This plan makes the loop legible without repairing it: +persist `last_run_at`, carry one causation id end to end, add counters and a +liveness verdict endpoint outside dev mode, and classify every terminal run +shape so a silent failure reads differently from quiet success. + +## Root cause + +No counter exists for routine ticks, runs launched, or taskless failures. +`/debug/vars` is dev-mode only. `routines.last_run_at` is never written, so +"never ran" and "ran correctly 326 times" are indistinguishable from any +available surface. `runs.session_id` can be silently dropped on launch, and +five distinct terminal shapes all persist `status = 'finished'`, so a +dashboard bucketing them collapses real failures into apparent success. + +## Scope + +### In scope + +- `REQ-OFFICE-LOOP-LIVENESS-001`: monotonic `last_run_at` write via a narrow + `TouchRoutineLastRun`, `UpdateRoutine` no longer writes the column. +- `REQ-OFFICE-LOOP-LIVENESS-002`: a `causation_id` minted once per wake + origin, carried `office_routine_runs` -> `agent_wakeup_requests` -> `runs`, + and a real `runs.session_id` (the Office launch seam stops discarding the + session id the orchestrator returns). +- `REQ-OFFICE-LOOP-LIVENESS-003`: per-hop counters readable without dev mode + via `GET /api/v1/office/workspaces/:wsId/loop-counters`. +- `REQ-OFFICE-LOOP-LIVENESS-004`: `GET .../loop-health` returning a verdict + (`unknown -> dead -> degraded -> not_armed -> healthy`) plus evidence and + thresholds. +- `REQ-OFFICE-LOOP-LIVENESS-005`: a total terminal-shape classification over + `(status, outcome, session_id)`, separating `silent_success` from the other + terminal shapes. + +### Out of scope + +- Repairing the loop once it is detected as dead (detection only; `runs.outcome` + is not widened). +- `office_routine_runs`' terminal transition / `SyncRunStatus` (owned by a + sibling card). +- Operator notification on a detected stall (blocked on this card, tracked + separately). + +## Work orders + +- [Task 01: Make the Office loop legible end-to-end](task-01-make-the-loop-legible.md) + +## Verification + +```bash +cd apps/backend +go test -tags fts5 -count=1 ./internal/office/... ./internal/runs/... +golangci-lint run ./internal/office/... ./internal/runs/... +``` diff --git a/docs/plans/office-loop-liveness/task-01-make-the-loop-legible.md b/docs/plans/office-loop-liveness/task-01-make-the-loop-legible.md new file mode 100644 index 0000000000..868117509b --- /dev/null +++ b/docs/plans/office-loop-liveness/task-01-make-the-loop-legible.md @@ -0,0 +1,133 @@ +--- +id: "01-make-the-loop-legible" +title: "Make the Office loop legible end-to-end" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-OFFICE-LOOP-LIVENESS-001 + - REQ-OFFICE-LOOP-LIVENESS-002 + - REQ-OFFICE-LOOP-LIVENESS-003 + - REQ-OFFICE-LOOP-LIVENESS-004 + - REQ-OFFICE-LOOP-LIVENESS-005 +acceptance_criteria: + - AC-OFFICE-LOOP-LIVENESS-001.1 + - AC-OFFICE-LOOP-LIVENESS-001.2 + - AC-OFFICE-LOOP-LIVENESS-001.3 + - AC-OFFICE-LOOP-LIVENESS-001.4 + - AC-OFFICE-LOOP-LIVENESS-001.5 + - AC-OFFICE-LOOP-LIVENESS-001.6 + - AC-OFFICE-LOOP-LIVENESS-001.7 + - AC-OFFICE-LOOP-LIVENESS-001.8 + - AC-OFFICE-LOOP-LIVENESS-002.1 + - AC-OFFICE-LOOP-LIVENESS-002.2 + - AC-OFFICE-LOOP-LIVENESS-002.3 + - AC-OFFICE-LOOP-LIVENESS-002.4 + - AC-OFFICE-LOOP-LIVENESS-002.5 + - AC-OFFICE-LOOP-LIVENESS-002.6 + - AC-OFFICE-LOOP-LIVENESS-002.7 + - AC-OFFICE-LOOP-LIVENESS-002.8 + - AC-OFFICE-LOOP-LIVENESS-002.9 + - AC-OFFICE-LOOP-LIVENESS-002.10 + - AC-OFFICE-LOOP-LIVENESS-002.11 + - AC-OFFICE-LOOP-LIVENESS-003.1 + - AC-OFFICE-LOOP-LIVENESS-003.2 + - AC-OFFICE-LOOP-LIVENESS-003.3 + - AC-OFFICE-LOOP-LIVENESS-003.4 + - AC-OFFICE-LOOP-LIVENESS-003.5 + - AC-OFFICE-LOOP-LIVENESS-003.6 + - AC-OFFICE-LOOP-LIVENESS-003.7 + - AC-OFFICE-LOOP-LIVENESS-003.8 + - AC-OFFICE-LOOP-LIVENESS-003.9 + - AC-OFFICE-LOOP-LIVENESS-004.1 + - AC-OFFICE-LOOP-LIVENESS-004.2 + - AC-OFFICE-LOOP-LIVENESS-004.3 + - AC-OFFICE-LOOP-LIVENESS-004.4 + - AC-OFFICE-LOOP-LIVENESS-004.5 + - AC-OFFICE-LOOP-LIVENESS-004.6 + - AC-OFFICE-LOOP-LIVENESS-004.7 + - AC-OFFICE-LOOP-LIVENESS-004.8 + - AC-OFFICE-LOOP-LIVENESS-004.9 + - AC-OFFICE-LOOP-LIVENESS-004.10 + - AC-OFFICE-LOOP-LIVENESS-004.11 + - AC-OFFICE-LOOP-LIVENESS-004.12 + - AC-OFFICE-LOOP-LIVENESS-004.13 + - AC-OFFICE-LOOP-LIVENESS-004.14 + - AC-OFFICE-LOOP-LIVENESS-004.15 + - AC-OFFICE-LOOP-LIVENESS-004.16 + - AC-OFFICE-LOOP-LIVENESS-004.17 + - AC-OFFICE-LOOP-LIVENESS-005.1 + - AC-OFFICE-LOOP-LIVENESS-005.2 + - AC-OFFICE-LOOP-LIVENESS-005.3 + - AC-OFFICE-LOOP-LIVENESS-005.4 + - AC-OFFICE-LOOP-LIVENESS-005.5 + - AC-OFFICE-LOOP-LIVENESS-005.6 + - AC-OFFICE-LOOP-LIVENESS-005.7 + - AC-OFFICE-LOOP-LIVENESS-005.8 + - AC-OFFICE-LOOP-LIVENESS-005.9 +system_design: + - ../../specs/office/system-design/loop-liveness.md +--- + +# Task 01: Make the Office loop legible end-to-end + +## Summary + +Persist a monotonic `last_run_at` on every routine fire, mint one +`causation_id` per wake origin and carry it (plus a real `session_id`) from +routine run to wake to `runs` row, expose per-hop counters and a liveness +verdict endpoint outside dev mode, and classify every terminal run shape so +"quiet because there is no work" reads differently from "quiet because it +silently failed." + +## In scope + +- `TouchRoutineLastRun`, a narrow monotonic-guarded write called from + `dispatchRoutineRun` before concurrency-policy evaluation; `UpdateRoutine` + stops writing `last_run_at`. +- A `causation_id` minted in `dispatchRoutineRun`, carried through the wakeup + and into the claimed `runs` row; `StartTaskWithRoute` widened to return the + session id so the Office launch seam stops discarding it. +- expvar counters per loop hop (tick evaluated, trigger fired, wake created, + run claimed, agent launched, terminal outcome by kind), reachable outside + dev mode via `GET /api/v1/office/workspaces/:wsId/loop-counters`. +- `GET /api/v1/office/workspaces/:wsId/loop-health`: a verdict + (`unknown -> dead -> degraded -> not_armed -> healthy`) with evidence lists + and thresholds, failing loud (503 naming the failing input). +- `ClassifyTerminalRun`, a function over `(status, outcome, session_id)` plus + the activation instant, separating `silent_success` from + `unlaunched_skipped`, `unlaunched_failed`, `launched_completed`, + `launched_failed`, `pre_activation`, and `unclassified`. + +## Out of scope + +- Repairing a detected-dead loop (detection only; `runs.outcome` is not + widened). +- `office_routine_runs`' terminal transition / `SyncRunStatus`. +- Operator notification on a detected stall. + +## Acceptance + +- `routines.last_run_at` advances monotonically on every dispatch, including + a skip, and is never written by `UpdateRoutine`. +- A `causation_id` minted once per wake origin is readable end to end from + `office_routine_runs` through `agent_wakeup_requests` into the claimed + `runs` row; `runs.session_id` reflects the real launched session. +- `/loop-counters` and `/loop-health` are reachable without dev mode and + without pprof enabled. +- `/loop-health` returns a verdict, its supporting evidence, and the + thresholds used, and responds 503 naming the failing input on a read + failure rather than failing closed. +- Every terminal run classifies into exactly one named shape; a legacy value + outside the current outcome enum and a pre-activation row both classify + without error. + +## Verification + +```bash +cd apps/backend +go build ./... && go vet ./... +go test -tags fts5 -count=1 ./internal/office/... ./internal/runs/... +golangci-lint run ./internal/office/... ./internal/runs/... +``` diff --git a/docs/specs/office/requirements/loop-liveness.md b/docs/specs/office/requirements/loop-liveness.md new file mode 100644 index 0000000000..1a70828b40 --- /dev/null +++ b/docs/specs/office/requirements/loop-liveness.md @@ -0,0 +1,338 @@ +--- +status: draft +system: office +created: 2026-09-07 +owners: + - kandev +--- + +# Office Loop Liveness Requirements + +## Overview + +Office's unattended loop is a chain: a cron tick evaluates due triggers, claims +one, creates a routine run, enqueues a wake, the wake becomes a `runs` row, the +scheduler claims it, an agent launches, and the run reaches a terminal state. +Every link is individually logged. **No surface answers whether the chain ran.** + +On the instance this was written against it stopped on 2026-08-05 and nothing +reported it: `last_run_at` was never written, 322 consecutive fires read as +correct coalescing, one surviving `runs.outcome = 'no_agent_launched'` names a +value no longer in the code, and every `runs` row carried `session_id = ''`. +Every surface read as healthy. + +Office owns this contract: every input is an Office primitive. It makes the loop +**legible**, not working — nothing here fires or re-arms a trigger, requeues a +run, or launches an agent. + +## Source obligations + +Inherited from `Forge/docs/kandev/office/BETA-REQUIREMENTS.md` (gaps 21, 31, §7a), +outside this repository. Restated in full; this spec, not the citation, is built: + +- **NFR-1 — no unverifiable success.** A run reporting that work was done must + carry persisted evidence an agent ran. A success claim without it is a defect to + be surfaced, not a success. +- **NFR-2 — one-identifier traceability.** From any identifier at any hop, an + operator must reach every other hop by keyed lookup on a persisted column, with + no log correlation and no free-text search. + +## Terminology + +- **Wake origin:** the first durable row a wake creates — `office_routine_runs` for + a routine fire, `agent_wakeup_requests` otherwise. +- **Causation id:** an opaque identifier minted once per wake origin and copied + forward unchanged onto every row descending from it. +- **Eligible trigger:** a row in `office_routine_triggers` with `kind = 'cron'` + and `enabled = 1`, nothing further: an *armed-ness* test, not a *due* test. + `GetDueTriggers`' extra `next_run_at` clauses shall not be reused here: they + drop every stranded trigger. +- **Overdue trigger:** an eligible trigger whose `next_run_at` is non-`NULL` and + older than `now - overdue grace`. +- **Claim instant:** `COALESCE(last_fired_at, created_at)`. Only the claim writes + `last_fired_at`, in the statement clearing `next_run_at`; `created_at` covers a + trigger that never fired, making the age total. `updated_at` shall not be used: + its other writers would re-date a dead trigger. +- **Stranded trigger:** an eligible trigger whose `next_run_at` is `NULL` and + whose claim instant is older than `now - stranded grace`. +- **Claiming trigger:** the same, but whose claim instant is *within* the grace. + Mid-claim, not dead; neither overdue nor stranded. +- **Claimable run:** a `queued` run the scheduler's claim filter would take: + `routing_blocked_status` is `NULL` and no other run for the same + `agent_profile_id` is `claimed`. Failing either, the queue is holding it back + on purpose. +- **Queue-eligible instant:** `scheduled_retry_at` when non-`NULL`, else + `requested_at`, so a backoff ages from when the run may next be claimed. +- **Stuck run:** a claimable run whose `current_route_attempt_seq` is `0` and + whose queue-eligible instant is older than `now - queued grace`; or a `claimed` + run whose `claimed_at` is `NULL` or older than `now - claimed grace`. A non-zero + sequence means the run was relaunched; see **Out of scope**. +- **Stuck instant:** the column a stuck run is aged by — `claimed_at` when + `claimed`, the queue-eligible instant when `queued`. It is the stuck list's + ordering key (`AC-OFFICE-LOOP-LIVENESS-004.9`). +- **Terminal shape:** the derived classification of a terminal run + (`REQ-OFFICE-LOOP-LIVENESS-005`). +- **Silent success:** a terminal shape asserting work happened on a run for which + no session was ever recorded. +- **Activation instant:** the timestamp published in `kandev_meta` once a boot has + positively probed that this capability's columns exist. Older rows cannot be + classified and must not be reported as defects. + +## Requirements + +### REQ-OFFICE-LOOP-LIVENESS-001: Persist routine fire recency + +**Intent:** `office_routines.last_run_at` is the field an operator reaches for to +ask "is this routine running", and nothing writes it, so a dead schedule reads +like one that never had a schedule. It records that the **schedule fired**, not +that the fire produced work: a coalesced or skipped fire still advances it. + +#### Acceptance criteria + +- **AC-OFFICE-LOOP-LIVENESS-001.1:** When an `office_routine_runs` row is created + for a routine, the system shall persist that row's dispatch instant to the + routine's `last_run_at`, whatever disposition the run later reaches. +- **AC-OFFICE-LOOP-LIVENESS-001.2:** The persisted instant shall be the routine + run's `started_at`, never the instant the write executed. +- **AC-OFFICE-LOOP-LIVENESS-001.3:** The write shall be monotonic: when the stored + value is already at or after the incoming instant, it shall not change. +- **AC-OFFICE-LOOP-LIVENESS-001.4:** When two dispatches for the same routine + commit concurrently, the stored value shall equal the greater of the two + instants, whichever order they commit in. +- **AC-OFFICE-LOOP-LIVENESS-001.5:** The write shall touch only `last_run_at` and + `updated_at`, and read-modify-write no other routine column. +- **AC-OFFICE-LOOP-LIVENESS-001.6:** When the write matches zero rows, the system + shall treat that as success and shall not retry. +- **AC-OFFICE-LOOP-LIVENESS-001.7:** When the write returns an error, the system + shall complete the dispatch anyway and record the failure as a counted, logged + event. +- **AC-OFFICE-LOOP-LIVENESS-001.8:** No other routine write shall change + `last_run_at`. The existing multi-column routine update that carries the field + shall stop writing it, so a caller holding a stale routine snapshot cannot move + the value backwards or clear it: monotonicity is a property of the column, not + of one statement. + +### REQ-OFFICE-LOOP-LIVENESS-002: Correlate a wake to its run and its session + +**Intent:** **NFR-2** ([Source obligations](#source-obligations)) requires a wake +→ run → session → process chain traceable by one id. It breaks twice today: no row +carries an id shared with its predecessor, and the launch drops the session id it +is handed. + +#### Acceptance criteria + +- **AC-OFFICE-LOOP-LIVENESS-002.1:** Every wake origin shall mint exactly one + causation id and persist it on the row it creates. +- **AC-OFFICE-LOOP-LIVENESS-002.2:** A wakeup request created for a routine fire + shall carry that fire's causation id. +- **AC-OFFICE-LOOP-LIVENESS-002.3:** A run created from a wakeup request shall + carry that request's causation id. +- **AC-OFFICE-LOOP-LIVENESS-002.4:** When a wakeup request is coalesced or skipped + against an in-flight run, the run's causation id shall not change; the request + keeps its own, and the two remain joinable through the request's run id. +- **AC-OFFICE-LOOP-LIVENESS-002.5:** When a run is requeued, retried, parked or + lifted, its causation id shall not change. +- **AC-OFFICE-LOOP-LIVENESS-002.6:** An empty causation id shall mean "not + correlated". No reader shall group, join, or aggregate two rows because both + carry an empty causation id. +- **AC-OFFICE-LOOP-LIVENESS-002.7:** When a launch creates an agent session for a + run, the run shall record that session's id before the launch call returns to the + scheduler, on both the direct and the provider-routed launch path. +- **AC-OFFICE-LOOP-LIVENESS-002.8:** When a launch reports success but yields no + session id, the run's session id shall be left empty and the condition counted, + never defaulted to a placeholder. +- **AC-OFFICE-LOOP-LIVENESS-002.9:** A heavy routine fire, which creates a task + instead of a wakeup request, shall record its causation id and its linked task id + on the routine run, so the fire reaches the sessions its task produces by the + keyed path `office_routine_runs.linked_task_id` → `tasks.id` → + `task_sessions.task_id`. +- **AC-OFFICE-LOOP-LIVENESS-002.10:** When one run is launched more than once — a + run requeued after a post-start provider fallback and relaunched against the same + row — the recorded session id shall be the one from the launch holding the + greatest `runs.claimed_at`, and the launch write shall never replace a recorded + session id with an empty one. This binds the launch write only: the requeue + preceding a relaunch already clears the column, which this capability does not + change. +- **AC-OFFICE-LOOP-LIVENESS-002.11:** When a launch yields a non-empty session id + but persisting it to the run fails, the system shall not fail the launch and + shall not retry the write; it shall leave the run's session id at whatever value + it already held, never writing the unpersisted one, and shall record the failure + as a counted, logged event carrying the run id and the unstored session id. That + counter shall be distinct from the one counting launches that yielded no session + id. + +### REQ-OFFICE-LOOP-LIVENESS-003: Count every hop of the loop, outside dev mode + +**Intent:** Office's only counters cover provider routing and budget claims, and +`/debug/vars` is mounted only under dev mode or pprof, so none is reachable on a +normal install. Per-hop counts are what show which link stopped. + +#### Acceptance criteria + +- **AC-OFFICE-LOOP-LIVENESS-003.1:** The system shall count, at minimum: cron + routine ticks evaluated, eligible triggers claimed, routine runs created, wakeup + requests created, runs claimed, agent launches, and terminal run transitions. +- **AC-OFFICE-LOOP-LIVENESS-003.2:** Routine runs shall be counted by + `(source, disposition)` and terminal runs by terminal shape, so a fire that + coalesced, one skipped and one launched are separately countable. +- **AC-OFFICE-LOOP-LIVENESS-003.3:** Every counter whose event is attributable to + a workspace shall carry it as a label. +- **AC-OFFICE-LOOP-LIVENESS-003.4:** Counters shall be readable with dev mode and + pprof both disabled. +- **AC-OFFICE-LOOP-LIVENESS-003.5:** A workspace-scoped reader shall receive only + that workspace's labelled counter values, and no other workspace's in any form. +- **AC-OFFICE-LOOP-LIVENESS-003.6:** The system shall publish the instant of the + most recent cron loop tick and the instant the process started. +- **AC-OFFICE-LOOP-LIVENESS-003.7:** Counters shall continue to be published through + the existing expvar mechanism, so `/debug/vars` keeps working. +- **AC-OFFICE-LOOP-LIVENESS-003.8:** When an event's workspace cannot be + resolved, the counter shall record it under an explicit unattributed label + rather than dropping it or guessing. Its value shall not be added into any + workspace's own values, shall be returned only in the process-scoped part of the + response, identical for every reader, and shall not be reachable only through a + dev-mode surface. +- **AC-OFFICE-LOOP-LIVENESS-003.9:** A counter naming a persisted state change + shall be incremented exactly once per change, at the site that persists it, and + a change that fails to persist shall not increment it. A counter naming an event + that persists nothing — a tick, a failed write, a launch that yielded no session, + an unservable read — shall be incremented exactly once at the site that observes + it, and shall be separable from the counters of persisted changes. + +### REQ-OFFICE-LOOP-LIVENESS-004: One request answers whether the loop is alive + +**Intent:** "Was an eligible routine due, claimed, launched and completed inside +its SLO?" today needs manual reconstruction. One workspace-scoped request must +answer it with a verdict, its evidence and its thresholds. + +#### Acceptance criteria + +- **AC-OFFICE-LOOP-LIVENESS-004.1:** The system shall expose a workspace-scoped + read returning one verdict from a closed set: `unknown`, `dead`, `degraded`, + `not_armed`, `healthy`. +- **AC-OFFICE-LOOP-LIVENESS-004.2:** The verdict shall be decided by that exact + precedence order, first match winning. +- **AC-OFFICE-LOOP-LIVENESS-004.3:** The verdict shall be `unknown` when the + activation instant has not been published. +- **AC-OFFICE-LOOP-LIVENESS-004.4:** The verdict shall be `dead` when the + workspace has at least one overdue or stranded trigger. +- **AC-OFFICE-LOOP-LIVENESS-004.5:** The verdict shall be `degraded` when no `dead` + condition holds and the workspace has at least one stuck run, or at least one + silent success whose terminal instant falls inside the evaluation window. The + window binds silent successes only: stuck runs are never terminal, carry no + terminal instant, and shall be reported however long they have been stuck. +- **AC-OFFICE-LOOP-LIVENESS-004.6:** The verdict shall be `not_armed` when no + `dead` or `degraded` condition holds and the workspace has no eligible trigger. +- **AC-OFFICE-LOOP-LIVENESS-004.7:** The response shall carry the evidence for its + verdict: the offending trigger rows, the stuck runs, the silent-success runs, + and the terminal-shape counts for the window. Overdue and stranded triggers form + one list, each row naming the condition it matched; so do queued-stuck and + claimed-stuck runs. Every condition that can decide `dead` or `degraded` shall + be evidenced by the rows that caused it, never by a count alone. +- **AC-OFFICE-LOOP-LIVENESS-004.8:** Every threshold and the evaluation window + used to reach the verdict shall be returned alongside it. +- **AC-OFFICE-LOOP-LIVENESS-004.9:** Each returned list shall be ordered + most-urgent-first by named columns with a named tiebreak, shall be capped, and + shall report the cap and the untruncated total when it truncates. +- **AC-OFFICE-LOOP-LIVENESS-004.10:** When any input needed for the verdict cannot + be read, the system shall fail loudly rather than return one, recording which + input failed. +- **AC-OFFICE-LOOP-LIVENESS-004.11:** The read shall be side-effect free: it shall + not fire, claim, re-arm, requeue, cancel or launch anything, and shall write no + table. +- **AC-OFFICE-LOOP-LIVENESS-004.12:** Two reads issued concurrently for the same + workspace shall each return an internally consistent verdict, and neither shall + affect the other. Internally consistent means each evidence list, its cap and + its total come from a single statement, and every threshold is applied to the + one captured instant. It does not mean the lists share a snapshot: the read + takes no transaction. +- **AC-OFFICE-LOOP-LIVENESS-004.13:** A workspace with no routines, runs or agents + shall return `not_armed` with empty evidence lists, never an error or null. +- **AC-OFFICE-LOOP-LIVENESS-004.14:** The read shall accept no caller-supplied + parameter other than the workspace, so no caller can widen the window, raise the + cap or soften a threshold; every threshold shall be evaluated against one + instant captured once per request. +- **AC-OFFICE-LOOP-LIVENESS-004.15:** A claiming trigger shall not be reported as + stranded or overdue and shall appear in no evidence list — excluded from the + verdict, not returned under a third label. +- **AC-OFFICE-LOOP-LIVENESS-004.16:** The evaluation window shall be applied to a + terminal run's finish instant, or its request instant when that is absent, so no + terminal run escapes the window by recording no finish instant. +- **AC-OFFICE-LOOP-LIVENESS-004.17:** A queued run that is not claimable shall + appear in no evidence list and shall decide no verdict, and a queued run's age + shall be measured from its queue-eligible instant. A run the queue is holding + back, and one whose backoff has not expired, are deferrals rather than stalls. + +### REQ-OFFICE-LOOP-LIVENESS-005: Separate the terminal shapes + +**Intent:** Six terminal dispositions write `status = 'finished'`, a terminal +failure writes `status = 'failed'` with a null outcome, and a legacy value no +enum contains survives in production data. Naming each ending apart makes +"quiet because there is no work" and "quiet because it silently fails" different +readings. + +#### Acceptance criteria + +- **AC-OFFICE-LOOP-LIVENESS-005.1:** The system shall classify every terminal run + into exactly one shape from a closed set. +- **AC-OFFICE-LOOP-LIVENESS-005.2:** The classification shall be total: every + combination of persisted status, outcome and session id shall map to one shape, + including any status or outcome value this document does not enumerate. +- **AC-OFFICE-LOOP-LIVENESS-005.3:** A run that ended without an agent having + launched shall be distinguishable from one that launched. +- **AC-OFFICE-LOOP-LIVENESS-005.4:** A run whose outcome asserts work happened but + which never recorded a session shall be classified as a silent success and shall + be separately countable — **NFR-1** made observable. +- **AC-OFFICE-LOOP-LIVENESS-005.5:** A legitimate no-launch skip shall be + classified distinctly from a launch failure. +- **AC-OFFICE-LOOP-LIVENESS-005.6:** A run predating the activation instant shall + be classified as pre-activation, never as a silent success. +- **AC-OFFICE-LOOP-LIVENESS-005.7:** Classification shall read only persisted + columns, never matching free-text messages. +- **AC-OFFICE-LOOP-LIVENESS-005.8:** Classification shall not change `runs.status` + or `runs.outcome`, nor alter what any existing consumer reports. +- **AC-OFFICE-LOOP-LIVENESS-005.9:** A run that has not reached a terminal status + shall be given no shape and appear in no terminal-shape total; it is the + stuck-run detector's input. + +## Out of scope + +Each exclusion below is a decision, not an omission. + +- **Repairing anything this capability detects.** No overdue trigger fired, no + stranded trigger re-armed, no stuck run requeued or cancelled, no agent launched + — the separation `REQ-OFFICE-STALL-VISIBILITY-003` also makes. The stranded + trigger is a real defect; its fix is a scheduling change owned elsewhere. +- **Reconciling `eligible trigger` with routine status.** The cron path selects on + `enabled` alone and never reads `office_routines.status`, so a paused routine + with an enabled trigger still fires. Eligibility tracks that behaviour; aligning + the two is the scheduler's card. +- **Widening `runs.outcome`.** `docs/specs/task-delivery-ledger/spec.md` owns that + column; this capability derives its vocabulary on top of it. +- **The terminal transition of `office_routine_runs`.** `status` stays + `task_created` forever and `completed_at` is written on every status change. + Both belong to card `032fadb8`, which owns `SyncRunStatus`. +- **A single-id chain for the heavy routine path.** `runs` carries no task id, so + `AC-OFFICE-LOOP-LIVENESS-002.9` delivers a two-key join rather than one id end + to end. Closing it needs the workflow engine to carry a causation id through + task creation. +- **Recording a causation id on `task_sessions`.** The run names its session, so + one id reaches it through the run. +- **Runs whose workspace cannot be resolved.** No foreign key makes + `runs.agent_profile_id` name a live profile, so such a run reaches no workspace + and decides no verdict. It counts under `AC-OFFICE-LOOP-LIVENESS-003.8`'s + unattributed label rather than being dropped. +- **Queue latency for a run already launched once.** A post-start fallback and a + stale-claim recovery both return a run to `queued` with `claimed_at` cleared, + and `runs` has no column dating that return. The fallback is excluded by the + attempt-sequence test; the recovery is not, so such a run is aged from its + original request and its reported wait overstates. It is genuinely unclaimed, + so it is reported rather than hidden. Closing this needs a requeue instant on + `runs`, owned elsewhere. +- **Notifying anybody** (card `5470c372`, which this unblocks); **WIP limits, + causation depth caps and self-trigger suppression** (card `7dbc0b94`, which + consumes this id); **retention** (card `9a7aa679`). +- **A global, unauthenticated metrics endpoint.** Every read added here is + workspace-scoped and goes through the existing Office authorization guard. +- **Any frontend.** No screen renders the verdict, the counters or `last_run_at`. +- **Correcting the historical `no_agent_launched` rows.** They are classified, not + rewritten; this capability performs no data migration. diff --git a/docs/specs/office/system-design/loop-liveness.md b/docs/specs/office/system-design/loop-liveness.md new file mode 100644 index 0000000000..e76ed287f3 --- /dev/null +++ b/docs/specs/office/system-design/loop-liveness.md @@ -0,0 +1,581 @@ +--- +status: draft +system: office +requirements: + - REQ-OFFICE-LOOP-LIVENESS-001 + - REQ-OFFICE-LOOP-LIVENESS-002 + - REQ-OFFICE-LOOP-LIVENESS-003 + - REQ-OFFICE-LOOP-LIVENESS-004 + - REQ-OFFICE-LOOP-LIVENESS-005 +--- + +# Office Loop Liveness System Design + +## Purpose and boundaries + +This design adds one write to the routine dispatch path, one identifier carried +across three Office tables, one set of counters, and two read-only workspace +endpoints. It changes no scheduling decision. + +Adjacent contracts read and constrained but not owned: + +- `internal/scheduler/cron` — the shared 30-second cron loop, `RoutinesHandler`. +- `internal/office/routines` — dispatch, catch-up, concurrency policy. +- `internal/office/wakeup` — the dispatcher's coalesce/skip/fresh-run branches. +- `internal/office/service` — `SchedulerIntegration`'s 5-second drain, claim, + guards, terminal transitions. +- `internal/runs/repository/sqlite` — the shared `runs` table, `FinishRun`, and + `ClaimNextEligibleRun`, whose filter the stuck-run predicate mirrors. +- `docs/specs/task-delivery-ledger/spec.md` — owns `runs.outcome`. Not modified. + +## Prior art + +Two external legs (the compiled wiki, and a cross-vendor comparison) were attempted +and both were UNAVAILABLE; receipts are in the card's plan, and nothing is claimed +here about what any vendor ships. + +**In-repository prior art, which was read.** Existing patterns are adopted, and one +is deliberately departed from: + +- `docs/specs/office/requirements/stall-visibility.md` is the closest sibling: + detection-only, `expvar.NewMap` counters with a `k=v;k=v` label model, and a skip + counter labelled by reason. This capability copies all three and **departs on + failure direction**; see [Failure](#failure). +- `docs/specs/task-delivery-ledger/spec.md` owns `runs.outcome`, so this design + derives a classification on top rather than widening it. Its + `telemetry.run_outcome.activated_at` key, written only after a positive schema + probe, is the pattern reused for the activation instant. +- `internal/office/service/wake_metrics.go` states the argument this generalizes: + "production cannot otherwise tell a working sweep (candidates found, nothing to + do) from a dead one (handler not ticking at all)." + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-OFFICE-LOOP-LIVENESS-001` | [Routine fire recency](#routine-fire-recency) | +| `REQ-OFFICE-LOOP-LIVENESS-002` | [Causation identity](#causation-identity) | +| `REQ-OFFICE-LOOP-LIVENESS-003` | [Counters](#counters), [The counter read](#the-counter-read) | +| `REQ-OFFICE-LOOP-LIVENESS-004` | [The liveness read](#the-liveness-read) | +| `REQ-OFFICE-LOOP-LIVENESS-005` | [Terminal shapes](#terminal-shapes) | + +## Persistence + +Three additive columns and three partial indexes, applied with the existing +`r.migrate.Apply` idiom in `internal/office/repository/sqlite`. No table is +created, no column is dropped, no value is backfilled. + +| Table | Column | Type | Meaning | +| --- | --- | --- | --- | +| `office_routine_runs` | `causation_id` | TEXT NOT NULL DEFAULT `''` | The id minted by this fire. | +| `agent_wakeup_requests` | `causation_id` | TEXT NOT NULL DEFAULT `''` | Copied from the fire, or minted when the wake has no routine origin. | +| `runs` | `causation_id` | TEXT NOT NULL DEFAULT `''` | Copied from the wakeup request that created the run. | +| `office_routines` | — | — | No new column. `last_run_at` already exists. | + +`''` rather than `NULL` because every existing row in all three tables predates +this feature and the reader must treat "legacy" and "uncorrelated" identically. + +An index is added on the `causation_id` of **all three** tables, each **partial to +`causation_id != ''`**, on both dialects. Partial because the legacy rows would +otherwise collapse into one bucket a lookup could accidentally join. All three +rather than `runs` alone because NFR-2 is symmetric: an operator walks *backwards* +to the originating row as often as forwards, and an unindexed backward hop is a +table scan wearing the words "direct lookup". + +### Activation + +`telemetry.office_loop_liveness.activated_at` is written into `kandev_meta` by +`persistence.WriteMetaKeyIfAbsent` at boot, only after a positive `columnExists` +probe of all three `causation_id` columns — the same shape as `activateRunOutcome`, +for the same reason: the migration runner swallows failures at `WARN`, so the probe +is all that stands between a failed migration and a consumer believing the +mechanism is live. + +The instant matters beyond bookkeeping. Every `outcome = 'processed'` row on the +reference instance has `session_id = ''` because nothing wrote it; classifying them +as silent successes would report dozens of defects on the day this ships and train +the operator to ignore the number. A run whose `requested_at` precedes the +activation instant is therefore `pre_activation`, so `silent_success` counts only +rows that could have been correlated. + +## Routine fire recency + +One new repository method, deliberately narrow: + +```go +TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error +``` + +```sql +UPDATE office_routines +SET last_run_at = ?, updated_at = ? +WHERE id = ? AND (last_run_at IS NULL OR last_run_at < ?) +``` + +It is **not** `UpdateRoutine`, which writes eleven columns from an in-memory +struct: using it here would turn a dispatch into a read-modify-write that clobbers +a concurrent edit from the UI or a config sync (AC-001.5). + +**The other writers, surveyed**, because a monotonic statement secures the +statement, not the column. `last_run_at` is written by the insert, by +`TouchRoutineLastRun`, and by `UpdateRoutine`, which carries it unconditionally +from `routine.LastRunAt`. That third writer defeats REQ-001: a caller holding a +routine loaded before a fire, or one that never populated the field, silently puts +a stale value or `NULL` back over a fresh instant and restores the "reads as never +ran" state. `UpdateRoutine` therefore **drops `last_run_at` from its column list** +(AC-001.8); the field stays on the struct for reads, and +`UpdateRoutineConfigFields` beside it already excludes the same column for the same +stale-snapshot reason. `last_run_at` then moves only forward, and only from a fire. + +The `last_run_at < ?` guard makes the write monotonic, hence idempotent (AC-001.3) +and order-independent under concurrency (AC-001.4): whichever of the catch-up +dispatch and a manual fire commits first, the surviving value is the greater +instant, because the earlier fails the predicate. A replay is a zero-row no-op, not +an error (AC-001.6). + +The call site is `dispatchRoutineRun`, immediately after `CreateRoutineRun` returns +and **before** `applyConcurrencyPolicy`, so a coalesced or skipped fire advances the +column (AC-001.1). The value passed is `*run.StartedAt` — the `now` also written +into the routine-run row, so the two are the same instant by construction +(AC-001.2). The existing `routine.LastRunAt = &now`, which mutates a struct nobody +persists, is deleted. Errors are logged and counted, never returned (AC-001.7): a +routine that fired must not read as having failed to fire because a bookkeeping +write lost a lock. + +## Causation identity + +### Minting + +A causation id is a UUIDv4 string, minted at exactly one place per origin: + +| Origin | Minted in | Persisted first on | +| --- | --- | --- | +| cron routine fire | `dispatchRoutineRun` | `office_routine_runs.causation_id` | +| manual / webhook routine fire | `dispatchRoutineRun` | `office_routine_runs.causation_id` | +| every other wake | the `agent_wakeup_requests` writer | `agent_wakeup_requests.causation_id` | + +`dispatchRoutineRun` is a single funnel for all three routine sources, so one mint +site covers them (AC-002.1). The lightweight path copies the id onto the +`WakeupRequest` it builds (AC-002.2); `createFreshRun` copies it from the request +onto the run (AC-002.3). + +The routines path is today the **only** production writer of +`agent_wakeup_requests`, so the third row is a rule for a future writer, not a +second implementation site now. Any new wake source mints at its own origin: +inheriting an id from an unrelated wake would make two causes look like one. + +### What must not change it + +The dispatcher has three branches and only one creates a run: + +- `MarkWakeupRequestCoalesced` and `MarkWakeupRequestSkipped` touch the request, + never the run. The in-flight run keeps the id of the wake that created it, the + coalesced request keeps its own, and the two stay joinable through + `agent_wakeup_requests.run_id` (AC-002.4). First writer wins: a run whose + identity changed under a reader would break the one thing the id is for. +- `PromoteRunAndCoalesceWakeupIfQueued` promotes the run's `reason` and must not + touch `causation_id` — a promotion is not a new cause. +- `createFreshRun` is the only writer, including where promotion lost the CAS race + and the event gets its own run: that run carries the *requesting* wake's id. + +Retry, park and lift (`ScheduleRetry`, `LiftParkedRuns`) mutate the existing row +and never re-create it, so AC-002.5 holds with no code change and is asserted by +test. + +### Empty is not a key + +`''` means "not correlated". A reader must never `GROUP BY causation_id` or join on +it without excluding `''` (AC-002.6). Every query this design adds carries the +exclusion in SQL rather than trusting a caller to remember it, and the partial +index makes the exclusion the cheap path. + +### The session hop + +`runs.session_id` is `NOT NULL DEFAULT ''` with exactly two writers today. +`ContextBuilder.BuildAndPersist` writes it from `run.Payload["session_id"]`, empty +for every Office-created run. `RequeueRunForNextCandidate` (`run_routing.go`) +*clears* it, in the same statement that returns the run to `queued` and nulls +`claimed_at` and `finished_at` — easy to miss, and it bounds what AC-002.10 can +promise. + +The session id that does exist is returned by the orchestrator and thrown away at +the Office seam: `orchestrator.Service.StartTask` returns a +`*executor.TaskExecution` whose `SessionID` is "the database ID of the agent +session", and both `schedulerTaskStarterAdapter.StartTask` and +`orchestrator.Service.StartTaskWithRoute` discard it. + +The fix widens the Office-facing seam, using the optional-interface idiom the +package already uses for `TaskStarterWithEnv`: + +```go +type TaskStarterWithSession interface { + StartTaskReturningSession(...) (sessionID string, err error) +} +``` + +with the routed path widened symmetrically. On success the scheduler persists the +returned id onto the run before `launchAgent` returns (AC-002.7). The production +adapter is **required** to satisfy the interface — no silent fallback — pinned by a +wiring completeness test, the device `TestOfficeRouteGroupMountsScopeGuard` uses. + +An empty returned session id is left empty and counted as +`office_loop_launch_without_session_total` (AC-002.8), never replaced with the run +id or any other placeholder: a placeholder would make a broken hop look correlated. + +The *write itself* can also fail with a real session id in hand, and that is the +dangerous case: the row it leaves behind — `outcome = 'processed'`, +`session_id = ''` — is byte-identical to a genuine `silent_success`, so a feature +built to detect that defect would manufacture it. Hence AC-002.11: do not fail the +launch, because the agent is already running; do not retry, because a failure of a +single-row `UPDATE` by primary key means the database is unavailable and a retry +loop would block the 5-second drain; and count it under a **distinct** counter, +`office_loop_session_persist_failed_total`. The column keeps *whatever it already +held*, because on a relaunch it may already name a live session. The log line +carries the run id and the unstored session id. The two counters are not merged: +the stored rows are identical, so the counters are the only place the two causes +stay apart. + +One run can be launched twice: `failTasklessRun`'s doc comment records the +post-start-fallback case, and the requeue reuses the same row. The write is **last +non-empty wins** (AC-002.10), ordered by `runs.claimed_at` — a *total* order, not a +race: a run is launched only while it holds a claim, and a relaunch requires a +return to `queued` and a fresh claim, so two launches are never in flight at once. +The guard is asymmetric on purpose — + +```sql +UPDATE runs SET session_id = ? WHERE id = ? AND ? != '' +``` + +— so the launch write never replaces a known link with an empty one. + +Its reach is narrow. `RequeueRunForNextCandidate` clears `session_id` on the way +back to `queued`, so the guard stops an empty write clobbering a good id *within one +claim* and cannot preserve one across a requeue. AC-002.10 is scoped to the launch +write accordingly, and a relaunch yielding no session id leaves the run +uncorrelated, counted under AC-002.8. + +### The heavy path + +`materialiseHeavyRoutineRun` writes `causation_id` and `linked_task_id` on the +routine run (AC-002.9). The task is then started through the orchestrator, not +queued through the `runs` table this design instruments — `runs` carries +`agent_profile_id` and no task id, so there is no causation hop to add there. The +reachable path is `office_routine_runs.linked_task_id` → `tasks.id` → +`task_sessions.task_id`, a two-key join, declared as such in the requirements' +**Out of scope**. + +## Counters + +`expvar.NewMap` with the `k1=v1;k2=v2` label convention already used by +`internal/office/scheduler/metrics_vars.go` and +`internal/orchestrator/office_stall_metrics.go`, in a new +`internal/office/service/loop_metrics.go`. + +| Counter | Labels | +| --- | --- | +| `office_loop_cron_tick_total` | — | +| `office_loop_trigger_claimed_total` | `workspace` | +| `office_loop_routine_run_total` | `workspace`, `source`, `disposition` | +| `office_loop_wakeup_created_total` | `workspace`, `source` | +| `office_loop_run_claimed_total` | `workspace` | +| `office_loop_launch_total` | `workspace` | +| `office_loop_launch_without_session_total` | `workspace` | +| `office_loop_session_persist_failed_total` | `workspace` | +| `office_loop_terminal_total` | `workspace`, `shape` | +| `office_loop_last_run_at_write_failed_total` | `workspace` | +| `office_loop_liveness_degraded_total` | `workspace`, `reason` | + +Two published instants deviate from the "counters only" note in `metrics_vars.go`: +`office_loop_cron_tick_at` and `office_loop_process_started_at` never decrease, so +they are not the drifting gauges that note warns about, and without them a flat +counter is ambiguous between a restarted process and a stopped loop (AC-003.6). + +AC-003.9's two increment rules split this table. The six that **name a persisted +change** (`trigger_claimed`, `routine_run`, `wakeup_created`, `run_claimed`, +`launch`, `terminal`) increment at the persisting site after the write succeeds, so +each counts rows that exist rather than attempts. The five that **name an event +persisting nothing** (`cron_tick`, `launch_without_session`, +`session_persist_failed`, `last_run_at_write_failed`, `liveness_degraded`) +increment at the observing site — three exist precisely to count a failure, so +gating them on a successful write would silence the signal they carry. + +These count **transitions, not entities**: a run requeued after a provider fallback +reaches a terminal state twice and contributes two `office_loop_terminal_total` +events, so no consumer may read the counter as a distinct-run count. Deduplicating +would need per-run state in the metric, which the label-cardinality rule forbids: +cardinality is bounded by the install's Office workspaces, and no label carries a +run, task, agent or routine id. + +When an event's workspace cannot be resolved — a failed agent row read, an orphaned +`agent_profile_id` — the counter records `workspace=_unattributed` rather than +dropping the event or guessing (AC-003.8); dropping it would make a partly-broken +loop look quieter than a healthy one. It is never added into any workspace's +values, but nor is it hidden behind `/debug/vars`: it is returned in the +**process-scoped** block below. Every workspace-labelled value is filtered to the +requesting workspace before serialization (AC-003.5). + +### The counter read + +Expvar publication keeps `/debug/vars` working (AC-003.7), but that route is +registered only under `p.devMode` (`internal/backendapp/helpers.go`) and is an +unfiltered global dump that could not satisfy AC-003.5 even where mounted. Hence: + +`GET /api/v1/office/workspaces/:wsId/loop-counters` + +Mounted on the Office route group beside the liveness read, inheriting +`AgentAuthMiddleware` and `officeWorkspaceScopeMiddleware`; `:wsId` is its only id, +so it needs no new `officeParamScopeResolvers` entry and leaves +`TestOfficeRouteScopeCompleteness` passing. Expvar remains the single source of +truth — this route is a filtered projection, not a second accounting. + +The response has two blocks, and that split is the isolation boundary: + +- **`workspace`** — every counter above carrying a `workspace` label, filtered to + the requesting workspace, other labels (`source`, `disposition`, `shape`, + `reason`) preserved. Zero-filling is at *counter-name* granularity: every name in + the table appears with a zero total even if unobserved, so "no events" is + distinguishable from "key absent", while secondary-label breakdowns appear only + as observed — their cross-product would invent combinations the code never emits. +- **`process`** — `office_loop_cron_tick_total`, the two instants, and the + `_unattributed` totals: identical for every reader, no workspace's data + (AC-003.8). + +Reading parses the `k1=v1;k2=v2` keys back out of each `expvar.Map` and keeps only +entries whose `workspace` matches. The parse is total — a key that does not split +into the expected labels lands in a malformed bucket in `process` rather than being +dropped, on the same argument as `_unattributed`. + +Counters are process-lifetime totals and reset on restart, which is what +`office_loop_process_started_at` is for. The route takes no query parameters +(AC-004.14). + +## Terminal shapes + +A function over three persisted columns plus the activation instant. It +writes nothing (AC-005.8) and reads no free text (AC-005.7). + +It is defined only over **terminal** runs, where terminal means "not `queued` and +not `claimed`" — an exclusion rather than an enumeration, so a status added later is +classified rather than silently skipped. Those two have no shape and are excluded +from every terminal total (AC-005.9); they are the stuck-run detector's input. +`runs` has no `running` status — `claimed` is the executing state. + +AC-005.2's totality ranges over the values actually written to `runs.status`, not +the Go `RunStatus` enum, which omits `cancelled` (written by +`internal/runs/repository/sqlite/cancel.go`). No writer of `timed_out` exists; it +appears below only as a defensive read. The cross-product test therefore enumerates +`finished`, `failed`, `cancelled`, `timed_out` and an unknown value. + +Evaluated top to bottom, first match winning. The final row makes it total +(AC-005.2), which absorbs the historical `no_agent_launched` value without a +migration. + +| Shape | Predicate | +| --- | --- | +| `pre_activation` | `requested_at` < activation instant, or activation instant unpublished | +| `launched_completed` | `status = 'finished'` and `outcome = 'processed'` and `session_id != ''` | +| `launched_failed` | `status IN ('failed','timed_out','cancelled')` and `session_id != ''` | +| `silent_success` | `status = 'finished'` and `outcome = 'processed'` and `session_id = ''` | +| `unlaunched_skipped` | `status = 'finished'` and `session_id = ''` and `outcome IN ('idle_skipped','budget_blocked','budget_unmeasurable','agent_inactive','task_tree_held')` | +| `unlaunched_failed` | `status IN ('failed','timed_out','cancelled')` and `session_id = ''` | +| `unclassified` | anything else, including `outcome IS NULL` and unknown values such as `no_agent_launched` | + +`silent_success` is the NFR-1 shape (no unverifiable success — see the requirements' +[Source obligations](../requirements/loop-liveness.md#source-obligations)): an +outcome asserting work happened on a run that never named a session (AC-005.4). +`unlaunched_skipped` is the legitimate no-launch, kept apart from +`unlaunched_failed` (AC-005.5). + +The ordering is load-bearing: `pre_activation` is first so the activation guard +cannot be bypassed by a later row matching; `silent_success` sits below +`launched_completed` so the session test is what separates them, and above +`unclassified` so a null outcome never absorbs it. + +`unlaunched_skipped` carries `session_id = ''` for the same reason +`unlaunched_failed` does: without it a `finished` run with a skip outcome *and* a +recorded session would be labelled "unlaunched", contradicting AC-005.3. That cell +should never occur — the skip outcomes are pre-launch guards — but the cross-product +test reaches it, and it falls to `unclassified`. + +## The liveness read + +`GET /api/v1/office/workspaces/:wsId/loop-health` + +Mounted on the Office route group, so it inherits `AgentAuthMiddleware` and +`officeWorkspaceScopeMiddleware`; `:wsId` is the only id it names, so it needs no new +`officeParamScopeResolvers` entry and satisfies `TestOfficeRouteScopeCompleteness` +as written. When the Office feature is off the group is not mounted and the route is +a 404. + +### Thresholds + +Named constants, all returned in the response (AC-004.8): + +| Constant | Value | Why | +| --- | --- | --- | +| `officeLoopTriggerOverdueGrace` | 3m | Six ticks of the 30s cron loop — far above jitter, and below the 5-minute coordinator routine's period, so a missed period is visible before the next is due. It measures the *loop*, not the routine's period, so it holds for a daily cron too. | +| `officeLoopTriggerStrandedGrace` | 5m | A claim clears `next_run_at` and the re-arm follows within one tick. Five minutes is ten ticks; past that the `NULL` is not a race, it is permanent. | +| `officeLoopRunQueuedGrace` | 2m | Applied only to a **claimable** run, so it measures the drain, not the queue: the drain ticks every 5s and takes 10 runs a tick, so two minutes without claiming a run the filter accepts means it is dead or starved. A run waiting on a claimed sibling is excluded from the predicate, not aged by it. | +| `officeLoopRunClaimedGrace` | 10m | Detection must precede recovery, so this is deliberately **not** `staleClaimedRunAge` (30m), the *recovery* threshold: sharing it would let a future tuning of recovery silently retune detection. | +| `officeLoopEvaluationWindow` | 24h | Terminal-shape counts and silent successes cover `COALESCE(finished_at, requested_at) >= now − 24h`, lower bound inclusive. The `COALESCE` is load-bearing: `finished_at` is nullable, so filtering on it alone would let a terminal run recording no finish instant vanish from every total (AC-004.16); `requested_at` is `NOT NULL`, so the fallback is total. It does **not** bind stuck runs. | +| `officeLoopEvidenceCap` | 50 | Per list. | + +### Queries + +Workspace scoping follows the existing join, `runs → agent_profiles.workspace_id` +(`ListRuns`); triggers scope through +`office_routine_triggers → office_routines.workspace_id`. That run join is inner and +no foreign key backs it, so a run naming a deleted profile reaches no workspace; the +requirements exclude it by name and route it to the unattributed counter. + +The trigger predicate is written out rather than reusing `GetDueTriggers`, whose +`next_run_at IS NOT NULL AND next_run_at <= ?` clauses are fatal here: every +stranded trigger has `next_run_at IS NULL` and would be filtered out, deleting the +`dead` path, while a trigger due later today would read as `not_armed`. Eligibility +is `kind = 'cron' AND enabled = 1`; `next_run_at` then splits eligible triggers into +overdue, stranded and claiming. + +The stuck-run predicates name their clocks, because `runs` has no single "waiting +since" column. Claimed age is `claimed_at`, treating `NULL` as stuck. Queued age is +the **queue-eligible instant**, `COALESCE(scheduled_retry_at, requested_at)`, +restricted to `current_route_attempt_seq = 0` — the restriction is not defensive: +`RequeueRunForNextCandidate` nulls `claimed_at` but leaves `requested_at` at the +first request, so without it every post-start provider fallback would read as stuck +the moment it requeued. + +**The queued predicate mirrors `ClaimNextEligibleRun`** (AC-004.17): a `queued` run +is stuck only if the scheduler would claim it *now* and has not, so it carries that +filter's exclusions — a future `scheduled_retry_at` (the `COALESCE` handles it: a +future instant is never older than `now − grace`), `routing_blocked_status IS NOT +NULL`, and `NOT EXISTS (SELECT 1 FROM runs s WHERE s.agent_profile_id = +r.agent_profile_id AND s.status = 'claimed')`. Without that last clause the detector +reports the normal steady state of a busy agent: sessions routinely outlast two +minutes, so every sibling queued behind one crosses the grace and the workspace +reads `degraded` forever — the false alarm that teaches an operator to ignore the +verdict. A parked run is the same story: `parkRunMaxAttempts` parks *for* a human. +What survives the filter is a run nothing is holding back that was not claimed. + +Neither predicate is windowed (AC-004.5) — `officeLoopEvaluationWindow` binds +terminal rows, and a run stuck longer than the window is the one an operator most +needs to see. + +Ordering is by named columns with a named tiebreak (AC-004.9): + +- triggers — **one list carrying both overdue and stranded rows**, each naming its + condition (AC-004.7): `ORDER BY (next_run_at IS NULL) DESC, next_run_at ASC, + id ASC` — stranded first, because they are permanently dead rather than merely + late. One list rather than two because the ordering is what ranks them against + each other, and two lists would leave "which is worse" to the reader. +- stuck runs — also one list, each row naming its condition, ordered by the + **stuck instant** (`claimed_at` when claimed, `COALESCE(scheduled_retry_at, + requested_at)` when queued): `ORDER BY (status = 'claimed') DESC, + (stuck_instant IS NULL) DESC, stuck_instant ASC, id ASC`. Two clocks cannot share + one column, so ranking by `requested_at` alone would put a long-dead claimed run + below a merely old queued one. Claimed first because such a run holds its + profile's only claim slot and blocks every queued sibling behind it; `NULL` first + because a claimed row with no `claimed_at` has no age at all. The `(expr) DESC` + idiom is the trigger list's, and sorts identically on both dialects unlike + `NULLS FIRST`. +- silent successes: `ORDER BY COALESCE(finished_at, requested_at) DESC, id ASC` — + newest first, because a silent success is only actionable while the session it + failed to name might still be found. + +**Silent successes are an evidence list, not just a count** (AC-004.7): they can +decide `degraded` on their own, and "degraded, one silent success" without naming +the run sends the reader back to hand-written SQL. Same cap and total treatment as +the other lists. + +Each list is capped at `officeLoopEvidenceCap`, and the response carries the cap and +the untruncated `total`. The total comes from `COUNT(*) OVER ()` in the **same +statement** as the capped page, not a second query: two statements outside a +transaction can straddle a write, and a list whose own total contradicts it is not +the "internally consistent" AC-004.12 requires. Consistency is deliberately scoped +to one list — the read takes no transaction, so different lists may observe the row +set at different instants; what is forbidden is a list disagreeing with itself. + +A trigger with `next_run_at IS NULL` whose claim instant is *inside* the stranded +grace is a **claiming trigger** (AC-004.15), excluded from every evidence list +rather than given a third label: a trigger caught mid-claim is the loop working, +and every tick would otherwise produce one. The stranded predicate's own age test +implements this; there is no separate branch. + +The route takes no query parameters (AC-004.14): a caller able to widen the window +or soften a threshold could make any workspace report `healthy`, and a monitor must +not be able to negotiate a verdict. `now` is captured once at the top of the +handler and passed to every predicate, so the trigger, run and terminal reads +cannot disagree about the instant they were evaluated against; it is returned with +the verdict. + +### Verdict + +Precedence `unknown` → `dead` → `degraded` → `not_armed` → `healthy`, first match +winning (AC-004.2). `dead` and `degraded` outrank `not_armed` because a workspace +with no cron trigger can still have stuck runs from event-driven wakes, and +reporting that as "nothing scheduled" would hide them. + +### Failure + +Fail loud: any unreadable input returns `503` with the failing input named, and +increments `office_loop_liveness_degraded_total{reason}` over the enumerated set +`activation_read_failed`, `trigger_read_failed`, `run_read_failed`, +`terminal_read_failed`. No partial verdict is ever returned. + +This inverts `stall-visibility.md`'s fail-closed rule on purpose: that detector +fails closed because a false alert trains operators to ignore it, whereas here a +partial answer rendering as `healthy` is the exact lie the feature removes, and the +caller is a monitor that can act on a `503`. + +## Security + +No new trust boundary. Both routes go through the existing Office authorization +guard. Their responses carry routine, trigger, run, causation and session +identifiers plus timestamps and counts — no prompt text, agent output, error body +or cost figure. Cross-workspace values are limited to the process facts named in +[The counter read](#the-counter-read) (AC-003.5). + +## Testing + +Go tests only; no user-visible surface changes, so no Playwright work. + +- `internal/office/repository/sqlite` — monotonic `TouchRoutineLastRun` including + the equal-instant and older-instant no-ops, the zero-row case, and the migration + on both dialects; and `UpdateRoutine` leaving `last_run_at` untouched (AC-001.8), + asserted by touching, then updating from a struct whose `LastRunAt` is stale and + again from one where it is `nil`, and reading the fresh instant back both times. +- `internal/office/routines` — the fire advances `last_run_at` on the coalesced + and skipped paths, mints one causation id per fire, and survives a + `TouchRoutineLastRun` error. +- `internal/office/wakeup` — causation id survives coalesce, skip, promotion, and + the lost-CAS fresh-run path. +- `internal/office/service` — terminal-shape classification table-driven and + asserted total over the cross-product of `finished`, `failed`, `cancelled`, + `timed_out` and an unknown status against every outcome, including `NULL` and + the legacy value, with a session-bearing skip landing in `unclassified`; + session id persisted on both launch paths; a failing session + write leaves the column at whatever it already held, does not fail the launch, + and increments `office_loop_session_persist_failed_total` rather than the + without-session counter; relaunch keeps the greater-`claimed_at` id and an empty + relaunch does not clear it; the wiring completeness test for the + session-returning seam. +- `internal/office/dashboard` — verdict precedence per branch, threshold echo, + ordering and truncation, the silent-success evidence list, a claiming trigger + appearing in no list, a terminal run with `NULL finished_at` still inside the + window, a stuck run reported regardless of age, empty-workspace shape, and a + `503` per degraded reason. For AC-004.17, one case per exclusion, each an + otherwise-stuck queued run that must **not** be reported and must leave the + workspace `healthy`: a future `scheduled_retry_at`, a non-`NULL` + `routing_blocked_status`, and a claimed sibling on the same `agent_profile_id`; + plus the positive control with no sibling claimed, which **is** reported, and a + run whose `scheduled_retry_at` has just passed being aged from it rather than + from `requested_at`. For the ordering, a mixed list asserting a claimed-stuck run + outranks an older queued-stuck one and a `NULL` `claimed_at` row sorts first. For + the counter read: workspace-labelled values + filtered to the requesting workspace with another workspace's key absent, an + unobserved counter returned as zero rather than omitted, `_unattributed` and the + process instants present in the `process` block and absent from `workspace`, and + a malformed key counted into the malformed bucket rather than dropped. + +## Related decisions + +None. This design reuses the counter convention, the activation-key idiom, the +optional-interface seam, and the workspace scope guard Office already has.