feat(office): make the unattended loop legible end-to-end (#3613)

Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com>
Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
Henry
2026-09-13 17:03:35 +08:00
committed by GitHub
parent 0ffa5e6489
commit fbb673db0f
101 changed files with 9038 additions and 319 deletions
@@ -260,6 +260,7 @@ func (a *routineWakeupAdapter) CreateWakeupRequest(
Reason: req.Reason,
Payload: req.Payload,
RequestedAt: req.RequestedAt,
CausationID: req.CausationID,
}
if req.IdempotencyKey != "" {
row.IdempotencyKey = sql.NullString{String: req.IdempotencyKey, Valid: true}
+5
View File
@@ -12,6 +12,7 @@ import (
officemodels "github.com/kandev/kandev/internal/office/models"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
officeroutines "github.com/kandev/kandev/internal/office/routines"
officeservice "github.com/kandev/kandev/internal/office/service"
schedulercron "github.com/kandev/kandev/internal/scheduler/cron"
tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite"
workflowrepo "github.com/kandev/kandev/internal/workflow/repository"
@@ -48,6 +49,10 @@ func startCronScheduler(
routines := schedulercron.NewRoutinesHandler(routineTicker, nil, log)
loop := schedulercron.NewLoop(schedulercron.DefaultTickInterval, log,
heartbeat, budget, routines, officeRecovery, parentWakeReconciler)
// AC-OFFICE-LOOP-LIVENESS-003.6: published once, at boot, so a flat
// office_loop_cron_tick_total is distinguishable from a restarted
// process versus a genuinely stopped loop.
officeservice.RecordLoopProcessStarted(time.Now().UTC().Format(time.RFC3339))
loop.Start(ctx)
log.Info("phase 5 cron loop started",
zap.Duration("interval", schedulercron.DefaultTickInterval))
+132 -12
View File
@@ -106,6 +106,7 @@ import (
officeskills "github.com/kandev/kandev/internal/office/skills"
officewakeup "github.com/kandev/kandev/internal/office/wakeup"
orchexecutor "github.com/kandev/kandev/internal/orchestrator/executor"
v1 "github.com/kandev/kandev/pkg/api/v1"
// Runs queue (Phase 3 of task-model-unification)
runsscheduler "github.com/kandev/kandev/internal/runs/scheduler"
@@ -1474,6 +1475,10 @@ func wireOfficeSvcsDependencies(
orchestratorSvc.SetOfficeTaskStatusUpdater(services.OfficeSvcs.Dashboard)
// Wire the office service as the retry canceller for task reassignment.
services.OfficeSvcs.Dashboard.SetRetryCanceller(services.Office)
// Wire the office service as the terminal-shape recorder so a
// dashboard-driven displaced-run cancellation still counts toward
// office_loop_terminal_total.
services.OfficeSvcs.Dashboard.SetTerminalShapeRecorder(services.Office)
// Wire the office service as the task canceller for status→cancelled hard-cancels.
services.OfficeSvcs.Dashboard.SetTaskCanceller(services.Office)
// Route the Office "No parent" mutation through the canonical task detach
@@ -1573,6 +1578,13 @@ type schedulerTaskStarterAdapter struct {
orch *orchestrator.Service
}
// The production routed-launch adapter is required to satisfy
// TaskStarterWithSession — no silent fallback to a session-less launch
// (AC-OFFICE-LOOP-LIVENESS-002.7). A compile-time assertion pins this
// harder than a runtime wiring test: dropping the method fails the
// build, not just a test run.
var _ officescheduler.TaskStarterWithSession = (*schedulerTaskStarterAdapter)(nil)
// StartTask starts a task on the orchestrator with the given launch parameters.
func (a *schedulerTaskStarterAdapter) StartTask(
ctx context.Context,
@@ -1594,6 +1606,32 @@ func (a *schedulerTaskStarterAdapter) StartTaskWithRoute(
launch officescheduler.LaunchContext,
route officescheduler.RouteOverride,
) error {
_, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route)
return err
}
// StartTaskWithRouteReturningSession implements
// officescheduler.TaskStarterWithSession so the routed launch path can
// persist the session id it started (AC-OFFICE-LOOP-LIVENESS-002.7).
func (a *schedulerTaskStarterAdapter) StartTaskWithRouteReturningSession(
ctx context.Context,
taskID, agentProfileID string,
launch officescheduler.LaunchContext,
route officescheduler.RouteOverride,
) (string, error) {
execution, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route)
if err != nil || execution == nil {
return "", err
}
return execution.SessionID, nil
}
func (a *schedulerTaskStarterAdapter) startTaskWithRoute(
ctx context.Context,
taskID, agentProfileID string,
launch officescheduler.LaunchContext,
route officescheduler.RouteOverride,
) (*orchexecutor.TaskExecution, error) {
return a.orch.StartTaskWithRoute(ctx, taskID, agentProfileID,
orchexecutor.LaunchContext{
ExecutorID: launch.ExecutorID,
@@ -2069,19 +2107,101 @@ func backfillAgentDefaultSkills(
}
}
// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in the
// officeservice.TaskStarterWithLaunchContextFunc adapter. Extracted from
// initOfficeServices to keep that function under the funlen cap.
// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in an
// adapter that also implements officeservice.TaskStarterWithLaunchContextSession,
// so the direct (non-routed) launch path forwards per-run skill additions and
// persists the session id the orchestrator's *executor.TaskExecution already
// carries (AC-OFFICE-LOOP-LIVENESS-002.7) instead of discarding it. Extracted
// from initOfficeServices to keep that function under the funlen cap.
func newOfficeTaskStarter(orchestratorSvc *orchestrator.Service) officeservice.TaskStarter {
return officeservice.TaskStarterWithLaunchContextFunc(
func(ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext) error {
_, err := orchestratorSvc.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
launch.WorkflowStepID, launch.PlanMode, false, launch.Attachments, launch.Env,
launch.AdditionalSkillSlugs)
return err
},
)
return &officeOrchestratorTaskStarter{orch: orchestratorSvc}
}
// officeOrchestratorTaskStarter satisfies officeservice.TaskStarter,
// TaskStarterWithEnv, TaskStarterWithLaunchContext, TaskStarterWithSession,
// and TaskStarterWithLaunchContextSession against the orchestrator service's
// direct launch path.
type officeOrchestratorTaskStarter struct {
orch *orchestrator.Service
}
// The production direct-launch adapter is required to satisfy
// TaskStarterWithLaunchContextSession for the same reason as the routed
// adapter above (AC-OFFICE-LOOP-LIVENESS-002.7): the session id must be
// recorded unconditionally, whether or not the launch also carries skills.
var _ officeservice.TaskStarterWithLaunchContextSession = (*officeOrchestratorTaskStarter)(nil)
// StartTask implements officeservice.TaskStarter.
func (a *officeOrchestratorTaskStarter) StartTask(
ctx context.Context, taskID, agentProfileID, executorID,
executorProfileID string, priority string, prompt, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment,
) error {
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, nil, nil)
return err
}
// StartTaskWithEnv implements officeservice.TaskStarterWithEnv.
func (a *officeOrchestratorTaskStarter) StartTaskWithEnv(
ctx context.Context, taskID, agentProfileID, executorID,
executorProfileID string, priority string, prompt, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
) error {
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil)
return err
}
// StartTaskWithEnvReturningSession implements
// officeservice.TaskStarterWithSession.
func (a *officeOrchestratorTaskStarter) StartTaskWithEnvReturningSession(
ctx context.Context, taskID, agentProfileID, executorID,
executorProfileID string, priority string, prompt, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
) (string, error) {
execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil)
if err != nil || execution == nil {
return "", err
}
return execution.SessionID, nil
}
// StartTaskWithLaunchContext implements officeservice.TaskStarterWithLaunchContext.
func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContext(
ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext,
) error {
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env,
launch.AdditionalSkillSlugs)
return err
}
// StartTaskWithLaunchContextReturningSession implements
// officeservice.TaskStarterWithLaunchContextSession.
func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContextReturningSession(
ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext,
) (string, error) {
execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env,
launch.AdditionalSkillSlugs)
if err != nil || execution == nil {
return "", err
}
return execution.SessionID, nil
}
func (a *officeOrchestratorTaskStarter) startTaskWithEnvAndSkills(
ctx context.Context, taskID, agentProfileID, executorID,
executorProfileID string, priority string, prompt, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment, env map[string]string, additionalSkillSlugs []string,
) (*orchexecutor.TaskExecution, error) {
return a.orch.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
executorID, executorProfileID, priority, prompt,
workflowStepID, planMode, false, attachments, env, additionalSkillSlugs)
}
// newAgentAuth wraps officeagents.NewAgentAuth with a dev-mode warning when
@@ -68,6 +68,8 @@
{"file": "internal/office/repository/sqlite/dashboard.go", "symbol": "QueryRunActivity", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
{"file": "internal/office/repository/sqlite/labels.go", "symbol": "GetOrCreateLabel", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"},
{"file": "internal/office/repository/sqlite/labels.go", "symbol": "AddLabelToTask", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"},
{"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "CountEligibleTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"},
{"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "ListOverdueOrStrandedTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"},
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "hasFTSTable", "rule": "sqlite-catalog", "reason": "SQLite-only FTS inspection"},
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTask", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTaskForRun", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
@@ -44,6 +44,7 @@ type Handler struct {
gitMgr *configloader.GitManager
runDetail RunDetailRepo
agentSummary AgentSummaryRepository
loopHealth LoopHealthRepo
handoff *taskservice.HandoffService
guard ActiveSourceChecker
logger *logger.Logger
@@ -76,6 +77,9 @@ func NewHandler(svc *DashboardService, labelRepo labelFetcher, gitMgr *configloa
if r, ok := labelRepo.(AgentSummaryRepository); ok {
h.agentSummary = r
}
if r, ok := labelRepo.(LoopHealthRepo); ok {
h.loopHealth = r
}
return h
}
@@ -127,6 +131,8 @@ func RegisterRoutes(api *gin.RouterGroup, svc *DashboardService, labelRepo label
api.GET("/workspaces/:wsId/routing/preview", h.getWorkspaceRoutingPreview)
api.GET("/runs/:id/attempts", h.listRunAttempts)
api.GET("/agents/:id/route", h.getAgentRoute)
registerLoopHealthRoutes(api, h)
}
// -- Dashboard --
@@ -0,0 +1,64 @@
package dashboard
import (
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/kandev/kandev/internal/office/service"
)
// registerLoopHealthRoutes mounts the two REQ-OFFICE-LOOP-LIVENESS
// reads on the Office route group. Both inherit AgentAuthMiddleware
// and officeWorkspaceScopeMiddleware from the group they're mounted
// on; :wsId is their only id, so neither needs a
// officeParamScopeResolvers entry.
func registerLoopHealthRoutes(api *gin.RouterGroup, h *Handler) {
api.GET("/workspaces/:wsId/loop-health", h.getLoopHealth)
api.GET("/workspaces/:wsId/loop-counters", h.getLoopCounters)
}
// getLoopHealth serves GET /workspaces/:wsId/loop-health
// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once here and
// threaded through every predicate (AC-004.14). Any unreadable input
// fails the whole read with 503 rather than a partial verdict
// (AC-004.10), incrementing office_loop_liveness_degraded_total by
// the failing input's reason.
func (h *Handler) getLoopHealth(c *gin.Context) {
if h.loopHealth == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: "loop health not configured"})
return
}
wsID := c.Param("wsId")
now := time.Now().UTC()
resp, err := EvaluateLoopHealth(c.Request.Context(), h.loopHealth, wsID, now)
if err != nil {
h.respondLoopHealthError(c, wsID, err)
return
}
c.JSON(http.StatusOK, resp)
}
func (h *Handler) respondLoopHealthError(c *gin.Context, workspaceID string, err error) {
var degraded *LoopHealthDegradedError
reason := ReasonRunReadFailed
if errors.As(err, &degraded) {
reason = degraded.Reason
}
service.IncLoopLivenessDegraded(workspaceID, reason)
c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: err.Error(), "reason": reason})
}
// getLoopCounters serves GET /workspaces/:wsId/loop-counters
// (REQ-OFFICE-LOOP-LIVENESS-003). Reads process-wide expvar state, so
// it never fails or 503s — an empty workspace simply reads all zeros.
func (h *Handler) getLoopCounters(c *gin.Context) {
wsID := c.Param("wsId")
c.JSON(http.StatusOK, ReadLoopCounters(wsID))
}
// LoopHealthRepoImpl returns the wired loop-health repo (may be nil).
// Exposed for tests that need to confirm the wiring.
func (h *Handler) LoopHealthRepoImpl() LoopHealthRepo { return h.loopHealth }
@@ -0,0 +1,161 @@
package dashboard
import (
"expvar"
"strings"
"github.com/kandev/kandev/internal/office/service"
)
// loopCounterMapNames lists every office_loop_* expvar.Map counter
// name (REQ-OFFICE-LOOP-LIVENESS-003, "Counters" table). Zero-filled
// at counter-name granularity (AC-003.8's malformed-key note and the
// design's "Zero-filling is at counter-name granularity" rule): a
// name absent from a reader's response would be indistinguishable
// from "never observed".
var loopCounterMapNames = []string{
"office_loop_trigger_claimed_total",
"office_loop_routine_run_total",
"office_loop_wakeup_created_total",
"office_loop_run_claimed_total",
"office_loop_launch_total",
"office_loop_launch_without_session_total",
"office_loop_session_persist_failed_total",
"office_loop_terminal_total",
"office_loop_last_run_at_write_failed_total",
"office_loop_liveness_degraded_total",
}
// LoopCountersResponse is the GET .../loop-counters payload. workspace
// carries every counter above labelled to the requesting workspace
// (secondary labels like source/disposition/shape/reason preserved,
// but only as observed); process carries the cron-tick counter, the
// two published instants, and the _unattributed totals — identical
// for every reader, no workspace's data (AC-003.5, AC-003.8).
type LoopCountersResponse struct {
Workspace map[string]map[string]int64 `json:"workspace"`
Process LoopCountersProcessDTO `json:"process"`
}
// LoopCountersProcessDTO is the process-scoped block of the counter
// read: never filtered by workspace, identical for every caller.
type LoopCountersProcessDTO struct {
CronTickTotal int64 `json:"cron_tick_total"`
CronTickAt string `json:"cron_tick_at"`
ProcessStartedAt string `json:"process_started_at"`
Unattributed map[string]map[string]int64 `json:"unattributed"`
Malformed map[string]map[string]int64 `json:"malformed"`
}
// ReadLoopCounters parses every office_loop_* expvar.Map back into a
// workspace-filtered, zero-filled response (AC-003.4, AC-003.5,
// AC-003.7). A key that fails to parse into the expected "k=v;k=v"
// label shape lands in the process block's malformed bucket rather
// than being dropped or attributed to any workspace, the same
// treatment AC-003.8 gives an unresolved workspace.
func ReadLoopCounters(workspaceID string) LoopCountersResponse {
resp := LoopCountersResponse{
Workspace: make(map[string]map[string]int64, len(loopCounterMapNames)),
Process: LoopCountersProcessDTO{
Unattributed: make(map[string]map[string]int64),
Malformed: make(map[string]map[string]int64),
},
}
for _, name := range loopCounterMapNames {
resp.Workspace[name] = map[string]int64{}
}
if v := expvar.Get("office_loop_cron_tick_total"); v != nil {
if m, ok := v.(*expvar.Map); ok {
resp.Process.CronTickTotal = expvarMapTotal(m)
}
}
if v := expvar.Get("office_loop_cron_tick_at"); v != nil {
resp.Process.CronTickAt = v.String()
resp.Process.CronTickAt = strings.Trim(resp.Process.CronTickAt, `"`)
}
if v := expvar.Get("office_loop_process_started_at"); v != nil {
resp.Process.ProcessStartedAt = strings.Trim(v.String(), `"`)
}
for _, name := range loopCounterMapNames {
v := expvar.Get(name)
if v == nil {
continue
}
m, ok := v.(*expvar.Map)
if !ok {
continue
}
partitionLoopCounterMap(&resp, name, m, workspaceID)
}
return resp
}
// partitionLoopCounterMap walks one counter's labelled entries and
// routes each into the workspace block, the process unattributed
// bucket, or the process malformed bucket.
func partitionLoopCounterMap(resp *LoopCountersResponse, mapName string, m *expvar.Map, workspaceID string) {
m.Do(func(kv expvar.KeyValue) {
iv, ok := kv.Value.(*expvar.Int)
if !ok {
return
}
labels, ok := parseLoopCounterKey(kv.Key)
if !ok {
addLoopCounterBucket(resp.Process.Malformed, mapName, kv.Key, iv.Value())
return
}
ws, hasWorkspace := labels["workspace"]
switch {
case hasWorkspace && ws == workspaceID:
resp.Workspace[mapName][kv.Key] = iv.Value()
case hasWorkspace && ws == service.LoopUnattributedWorkspace:
addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value())
case hasWorkspace:
// A different workspace's value — filtered out entirely
// (AC-003.5): not returned in any form to this reader.
default:
// No workspace label at all (process-only counter reached
// via this generic loop, or a future label-less counter) —
// treat as unattributed rather than silently dropped.
addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value())
}
})
}
func addLoopCounterBucket(bucket map[string]map[string]int64, mapName, key string, value int64) {
if bucket[mapName] == nil {
bucket[mapName] = map[string]int64{}
}
bucket[mapName][key] = value
}
// parseLoopCounterKey parses the "k1=v1;k2=v2" expvar map key back
// into labels. Total parse (AC-003.8's argument extended to this
// read): any segment that doesn't split into exactly one "=" fails
// the whole key rather than silently dropping that label.
func parseLoopCounterKey(key string) (map[string]string, bool) {
if key == "" {
return map[string]string{}, true
}
labels := make(map[string]string)
for _, part := range strings.Split(key, ";") {
kv := strings.SplitN(part, "=", 2)
if len(kv) != 2 || kv[0] == "" {
return nil, false
}
labels[kv[0]] = kv[1]
}
return labels, true
}
func expvarMapTotal(m *expvar.Map) int64 {
var total int64
m.Do(func(kv expvar.KeyValue) {
if iv, ok := kv.Value.(*expvar.Int); ok {
total += iv.Value()
}
})
return total
}
@@ -0,0 +1,75 @@
package dashboard
import (
"expvar"
"testing"
"github.com/kandev/kandev/internal/office/service"
)
func TestReadLoopCounters_FiltersToRequestingWorkspace(t *testing.T) {
wsA := "ws-counters-a"
wsB := "ws-counters-b"
service.IncLoopRunClaimed(wsA)
service.IncLoopRunClaimed(wsB)
resp := ReadLoopCounters(wsA)
got := resp.Workspace["office_loop_run_claimed_total"]
if _, ok := got[service.LoopMetricLabel("workspace", wsB)]; ok {
t.Errorf("workspace block contains another workspace's key: %+v", got)
}
if v, ok := got[service.LoopMetricLabel("workspace", wsA)]; !ok || v < 1 {
t.Errorf("workspace block missing own key, got %+v", got)
}
}
func TestReadLoopCounters_UnobservedCounterIsZeroFilledNotOmitted(t *testing.T) {
resp := ReadLoopCounters("ws-counters-never-seen")
for _, name := range loopCounterMapNames {
if _, ok := resp.Workspace[name]; !ok {
t.Errorf("counter %q missing from workspace block, want zero-filled empty map", name)
}
}
}
func TestReadLoopCounters_UnattributedAndProcessInstantsPresent(t *testing.T) {
service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace)
service.IncLoopCronTick("2026-05-01T12:00:00Z")
service.RecordLoopProcessStarted("2026-05-01T11:00:00Z")
resp := ReadLoopCounters("ws-counters-some-other-workspace")
unattr := resp.Process.Unattributed["office_loop_trigger_claimed_total"]
if _, ok := unattr[service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; !ok {
t.Errorf("unattributed bucket missing entry, got %+v", unattr)
}
if _, ok := resp.Workspace["office_loop_trigger_claimed_total"][service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; ok {
t.Errorf("unattributed key leaked into workspace block: %+v", resp.Workspace["office_loop_trigger_claimed_total"])
}
if resp.Process.CronTickTotal < 1 {
t.Errorf("cron tick total = %d, want >= 1", resp.Process.CronTickTotal)
}
if resp.Process.CronTickAt == "" {
t.Error("cron tick at is empty, want the published instant")
}
if resp.Process.ProcessStartedAt == "" {
t.Error("process started at is empty, want the published instant")
}
}
func TestReadLoopCounters_MalformedKeyCountsIntoMalformedBucket(t *testing.T) {
m := expvar.Get("office_loop_run_claimed_total").(*expvar.Map)
m.Add("not-a-valid-label", 1)
resp := ReadLoopCounters("ws-counters-malformed-probe")
malformed := resp.Process.Malformed["office_loop_run_claimed_total"]
if v, ok := malformed["not-a-valid-label"]; !ok || v < 1 {
t.Errorf("malformed bucket missing key, got %+v", malformed)
}
if _, ok := resp.Workspace["office_loop_run_claimed_total"]["not-a-valid-label"]; ok {
t.Error("malformed key leaked into workspace block")
}
}
@@ -0,0 +1,347 @@
package dashboard
import (
"context"
"time"
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/service"
)
// Thresholds (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.8 — echoed verbatim
// in every response so a caller never has to hardcode them).
const (
officeLoopTriggerOverdueGrace = 3 * time.Minute
officeLoopTriggerStrandedGrace = 5 * time.Minute
officeLoopRunQueuedGrace = 2 * time.Minute
officeLoopRunClaimedGrace = 10 * time.Minute
officeLoopEvaluationWindow = 24 * time.Hour
officeLoopEvidenceCap = 50
)
// Verdict values, closed set (AC-004.1), precedence order
// unknown -> dead -> degraded -> not_armed -> healthy (AC-004.2).
const (
VerdictUnknown = "unknown"
VerdictDead = "dead"
VerdictDegraded = "degraded"
VerdictNotArmed = "not_armed"
VerdictHealthy = "healthy"
)
// Degraded reasons (AC-004.10), the enumerated set
// office_loop_liveness_degraded_total{reason} is labelled with. Per
// OPERATOR DECISION F31, the silent-success evidence read and the
// terminal-shape count read both map to ReasonTerminalReadFailed —
// both ultimately answer "what shape did this run end in", so a
// caller diagnosing a 503 reads one reason for either query rather
// than a distinction that doesn't help them act.
const (
ReasonActivationReadFailed = "activation_read_failed"
ReasonTriggerReadFailed = "trigger_read_failed"
ReasonRunReadFailed = "run_read_failed"
ReasonTerminalReadFailed = "terminal_read_failed"
)
// LoopHealthDegradedError names which input could not be read
// (AC-004.10). The HTTP layer maps this to a 503 and increments
// office_loop_liveness_degraded_total{reason}.
type LoopHealthDegradedError struct {
Reason string
Err error
}
func (e *LoopHealthDegradedError) Error() string {
return "loop health degraded (" + e.Reason + "): " + e.Err.Error()
}
func (e *LoopHealthDegradedError) Unwrap() error { return e.Err }
// LoopHealthRepo is the persistence surface EvaluateLoopHealth needs.
// Implemented by *sqlite.Repository.
type LoopHealthRepo interface {
LoopLivenessActivationChecked() (at time.Time, published bool, err error)
CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error)
ListOverdueOrStrandedTriggers(
ctx context.Context, workspaceID string, now time.Time,
overdueGrace, strandedGrace time.Duration, limit int,
) ([]sqlite.TriggerHealthRow, int, error)
ListStuckRuns(
ctx context.Context, workspaceID string, now time.Time,
queuedGrace, claimedGrace time.Duration, limit int,
) ([]sqlite.StuckRunRow, int, error)
ListSilentSuccesses(
ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int,
) ([]sqlite.SilentSuccessRow, int, error)
ListTerminalRunsInWindow(
ctx context.Context, workspaceID string, windowStart time.Time,
) ([]sqlite.TerminalRunShapeInputRow, error)
}
// TriggerEvidenceDTO is one overdue or stranded trigger row (AC-004.7).
type TriggerEvidenceDTO struct {
TriggerID string `json:"trigger_id"`
RoutineID string `json:"routine_id"`
RoutineName string `json:"routine_name"`
Condition string `json:"condition"`
NextRunAt *time.Time `json:"next_run_at,omitempty"`
LastFiredAt *time.Time `json:"last_fired_at,omitempty"`
}
// StuckRunEvidenceDTO is one claimed-stuck or queued-stuck run row.
type StuckRunEvidenceDTO struct {
RunID string `json:"run_id"`
AgentProfileID string `json:"agent_profile_id"`
Status string `json:"status"`
Condition string `json:"condition"`
StuckSince *time.Time `json:"stuck_since,omitempty"`
}
// SilentSuccessEvidenceDTO is one finished/processed/sessionless run.
type SilentSuccessEvidenceDTO struct {
RunID string `json:"run_id"`
RequestedAt time.Time `json:"requested_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
}
// EvidenceListDTO wraps a capped list with its untruncated total
// (AC-004.9): "capped" is true exactly when total > len(rows).
type triggerEvidenceListDTO struct {
Rows []TriggerEvidenceDTO `json:"rows"`
Total int `json:"total"`
Cap int `json:"cap"`
Capped bool `json:"capped"`
}
type stuckRunEvidenceListDTO struct {
Rows []StuckRunEvidenceDTO `json:"rows"`
Total int `json:"total"`
Cap int `json:"cap"`
Capped bool `json:"capped"`
}
type silentSuccessEvidenceListDTO struct {
Rows []SilentSuccessEvidenceDTO `json:"rows"`
Total int `json:"total"`
Cap int `json:"cap"`
Capped bool `json:"capped"`
}
// LoopHealthThresholdsDTO echoes every threshold and the evaluation
// window used to reach the verdict (AC-004.8), in seconds so the
// response carries no ambiguous duration string.
type LoopHealthThresholdsDTO struct {
TriggerOverdueGraceSeconds int `json:"trigger_overdue_grace_seconds"`
TriggerStrandedGraceSeconds int `json:"trigger_stranded_grace_seconds"`
RunQueuedGraceSeconds int `json:"run_queued_grace_seconds"`
RunClaimedGraceSeconds int `json:"run_claimed_grace_seconds"`
EvaluationWindowSeconds int `json:"evaluation_window_seconds"`
EvidenceCap int `json:"evidence_cap"`
}
// LoopHealthResponse is the GET .../loop-health payload.
type LoopHealthResponse struct {
Verdict string `json:"verdict"`
EvaluatedAt time.Time `json:"evaluated_at"`
Thresholds LoopHealthThresholdsDTO `json:"thresholds"`
EligibleTriggerCount int `json:"eligible_trigger_count"`
Triggers triggerEvidenceListDTO `json:"triggers"`
StuckRuns stuckRunEvidenceListDTO `json:"stuck_runs"`
SilentSuccesses silentSuccessEvidenceListDTO `json:"silent_successes"`
TerminalShapeCounts map[string]int `json:"terminal_shape_counts"`
EvaluationWindowStart time.Time `json:"evaluation_window_start"`
}
// allTerminalShapes lists every TerminalShape so the response
// zero-fills unobserved shapes rather than omitting their key,
// mirroring the counter read's zero-fill-by-name rule.
var allTerminalShapes = []service.TerminalShape{
service.ShapePreActivation,
service.ShapeLaunchedCompleted,
service.ShapeLaunchedFailed,
service.ShapeSilentSuccess,
service.ShapeUnlaunchedSkipped,
service.ShapeUnlaunchedFailed,
service.ShapeUnclassified,
}
// EvaluateLoopHealth answers "is the loop alive" for one workspace
// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once by the caller
// and passed to every predicate so the trigger, run and terminal reads
// cannot disagree about the instant they were evaluated against
// (AC-004.14). Side-effect free (AC-004.11): every call is a read.
//
// On any unreadable input, returns a *LoopHealthDegradedError naming
// the failing input rather than a partial verdict (AC-004.10).
func EvaluateLoopHealth(
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
) (*LoopHealthResponse, error) {
thresholds := LoopHealthThresholdsDTO{
TriggerOverdueGraceSeconds: int(officeLoopTriggerOverdueGrace.Seconds()),
TriggerStrandedGraceSeconds: int(officeLoopTriggerStrandedGrace.Seconds()),
RunQueuedGraceSeconds: int(officeLoopRunQueuedGrace.Seconds()),
RunClaimedGraceSeconds: int(officeLoopRunClaimedGrace.Seconds()),
EvaluationWindowSeconds: int(officeLoopEvaluationWindow.Seconds()),
EvidenceCap: officeLoopEvidenceCap,
}
windowStart := now.Add(-officeLoopEvaluationWindow)
activationAt, published, err := repo.LoopLivenessActivationChecked()
if err != nil {
return nil, &LoopHealthDegradedError{Reason: ReasonActivationReadFailed, Err: err}
}
if !published {
return &LoopHealthResponse{
Verdict: VerdictUnknown,
EvaluatedAt: now,
Thresholds: thresholds,
Triggers: triggerEvidenceListDTO{Rows: []TriggerEvidenceDTO{}, Cap: officeLoopEvidenceCap},
StuckRuns: stuckRunEvidenceListDTO{Rows: []StuckRunEvidenceDTO{}, Cap: officeLoopEvidenceCap},
SilentSuccesses: silentSuccessEvidenceListDTO{Rows: []SilentSuccessEvidenceDTO{}, Cap: officeLoopEvidenceCap},
TerminalShapeCounts: zeroFilledShapeCounts(),
EvaluationWindowStart: windowStart,
}, nil
}
triggerEvidence, eligibleCount, err := evaluateLoopHealthTriggers(ctx, repo, workspaceID, now)
if err != nil {
return nil, err
}
stuckEvidence, err := evaluateLoopHealthStuckRuns(ctx, repo, workspaceID, now)
if err != nil {
return nil, err
}
silentEvidence, err := evaluateLoopHealthSilentSuccesses(ctx, repo, workspaceID, windowStart, activationAt)
if err != nil {
return nil, err
}
shapeCounts, err := evaluateLoopHealthTerminalShapes(ctx, repo, workspaceID, windowStart, activationAt, published)
if err != nil {
return nil, err
}
verdict := decideLoopHealthVerdict(triggerEvidence.Total, eligibleCount, stuckEvidence.Total, silentEvidence.Total)
return &LoopHealthResponse{
Verdict: verdict,
EvaluatedAt: now,
Thresholds: thresholds,
EligibleTriggerCount: eligibleCount,
Triggers: triggerEvidence,
StuckRuns: stuckEvidence,
SilentSuccesses: silentEvidence,
TerminalShapeCounts: shapeCounts,
EvaluationWindowStart: windowStart,
}, nil
}
// decideLoopHealthVerdict applies the AC-004.2 precedence
// unknown -> dead -> degraded -> not_armed -> healthy. unknown is
// decided by the caller before evidence is even read, so this
// function only sees the four remaining branches.
func decideLoopHealthVerdict(triggerTotal, eligibleCount, stuckTotal, silentTotal int) string {
switch {
case triggerTotal > 0:
return VerdictDead
case stuckTotal > 0 || silentTotal > 0:
return VerdictDegraded
case eligibleCount == 0:
return VerdictNotArmed
default:
return VerdictHealthy
}
}
func evaluateLoopHealthTriggers(
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
) (triggerEvidenceListDTO, int, error) {
eligibleCount, err := repo.CountEligibleTriggers(ctx, workspaceID)
if err != nil {
return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err}
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, workspaceID, now, officeLoopTriggerOverdueGrace, officeLoopTriggerStrandedGrace, officeLoopEvidenceCap,
)
if err != nil {
return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err}
}
dtos := make([]TriggerEvidenceDTO, 0, len(rows))
for _, r := range rows {
dtos = append(dtos, TriggerEvidenceDTO{
TriggerID: r.TriggerID,
RoutineID: r.RoutineID,
RoutineName: r.RoutineName,
Condition: r.Condition,
NextRunAt: r.NextRunAt,
LastFiredAt: r.LastFiredAt,
})
}
return triggerEvidenceListDTO{
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
}, eligibleCount, nil
}
func evaluateLoopHealthStuckRuns(
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
) (stuckRunEvidenceListDTO, error) {
rows, total, err := repo.ListStuckRuns(
ctx, workspaceID, now, officeLoopRunQueuedGrace, officeLoopRunClaimedGrace, officeLoopEvidenceCap,
)
if err != nil {
return stuckRunEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonRunReadFailed, Err: err}
}
dtos := make([]StuckRunEvidenceDTO, 0, len(rows))
for _, r := range rows {
dtos = append(dtos, StuckRunEvidenceDTO{
RunID: r.RunID, AgentProfileID: r.AgentProfileID, Status: r.Status,
Condition: r.Condition, StuckSince: r.StuckSince,
})
}
return stuckRunEvidenceListDTO{
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
}, nil
}
// evaluateLoopHealthSilentSuccesses reports ReasonTerminalReadFailed
// on failure, not ReasonRunReadFailed — OPERATOR DECISION F31.
func evaluateLoopHealthSilentSuccesses(
ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart, activationAt time.Time,
) (silentSuccessEvidenceListDTO, error) {
rows, total, err := repo.ListSilentSuccesses(ctx, workspaceID, windowStart, activationAt, officeLoopEvidenceCap)
if err != nil {
return silentSuccessEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err}
}
dtos := make([]SilentSuccessEvidenceDTO, 0, len(rows))
for _, r := range rows {
dtos = append(dtos, SilentSuccessEvidenceDTO{
RunID: r.RunID, RequestedAt: r.RequestedAt, FinishedAt: r.FinishedAt,
})
}
return silentSuccessEvidenceListDTO{
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
}, nil
}
func evaluateLoopHealthTerminalShapes(
ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart time.Time,
activationAt time.Time, published bool,
) (map[string]int, error) {
rows, err := repo.ListTerminalRunsInWindow(ctx, workspaceID, windowStart)
if err != nil {
return nil, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err}
}
counts := zeroFilledShapeCounts()
for _, r := range rows {
shape := service.ClassifyTerminalRun(r.Status, r.Outcome, r.SessionID, r.RequestedAt, activationAt, published)
counts[string(shape)]++
}
return counts, nil
}
func zeroFilledShapeCounts() map[string]int {
counts := make(map[string]int, len(allTerminalShapes))
for _, s := range allTerminalShapes {
counts[string(s)] = 0
}
return counts
}
@@ -0,0 +1,103 @@
package dashboard_test
import (
"encoding/json"
"expvar"
"net/http"
"net/http/httptest"
"testing"
"github.com/kandev/kandev/internal/office/service"
)
// degradedExpvarInt reads one office_loop_liveness_degraded_total cell
// by its full label key, mirroring the terminalShapeExpvarInt helper
// pattern used for the same expvar.Map shape elsewhere in this suite.
func degradedExpvarInt(t *testing.T, key string) int64 {
t.Helper()
v := expvar.Get("office_loop_liveness_degraded_total")
if v == nil {
t.Fatal("expvar map office_loop_liveness_degraded_total not registered")
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatal("office_loop_liveness_degraded_total is not a Map")
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("office_loop_liveness_degraded_total[%q] is not an Int", key)
}
return iv.Value()
}
// AC-003.4: the loop-health and loop-counters routes are reachable
// with no dev-mode/pprof env var set — newTestDeps's router mounts
// them with no such gate, matching production registration.
func TestGetLoopHealth_ReachableWithoutDevMode(t *testing.T) {
deps := newTestDeps(t)
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-health", nil)
w := httptest.NewRecorder()
deps.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
}
}
func TestGetLoopCounters_ReachableWithoutDevMode(t *testing.T) {
deps := newTestDeps(t)
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-counters", nil)
w := httptest.NewRecorder()
deps.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
}
}
// AC-004.10: a read failure through the real, database-backed
// LoopHealthRepo (not the pure-Go fake used by loop_health_test.go)
// must reach the HTTP layer as a 503 carrying a reason field, and
// must increment office_loop_liveness_degraded_total by that reason
// — exercising the full handler wiring, not just EvaluateLoopHealth
// in isolation.
func TestGetLoopHealth_ReadFailureRespondsServiceUnavailableAndIncrementsCounter(t *testing.T) {
deps := newTestDeps(t)
wsID := "ws-loop-health-db-closed"
key := service.LoopMetricLabel("workspace", wsID, "reason", "activation_read_failed")
before := degradedExpvarInt(t, key)
if err := deps.db.Close(); err != nil {
t.Fatalf("close db: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/"+wsID+"/loop-health", nil)
w := httptest.NewRecorder()
deps.router.ServeHTTP(w, req)
if w.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503: %s", w.Code, w.Body.String())
}
var body map[string]any
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
t.Fatalf("decode body: %v", err)
}
if body["reason"] != "activation_read_failed" {
t.Errorf("reason = %v, want activation_read_failed", body["reason"])
}
if _, ok := body["error"]; !ok {
t.Errorf("body = %+v, want an error field", body)
}
after := degradedExpvarInt(t, key)
if after != before+1 {
t.Errorf("degraded counter delta = %d, want 1", after-before)
}
}
@@ -0,0 +1,390 @@
package dashboard
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/office/repository/sqlite"
)
// fakeLoopHealthRepo is a hand-rolled LoopHealthRepo for exercising
// EvaluateLoopHealth's verdict precedence and failure-reason mapping
// without a database.
type fakeLoopHealthRepo struct {
activationAt time.Time
activationPublished bool
activationErr error
eligibleCount int
eligibleErr error
triggerRows []sqlite.TriggerHealthRow
triggerTotal int
triggerErr error
stuckRows []sqlite.StuckRunRow
stuckTotal int
stuckErr error
silentRows []sqlite.SilentSuccessRow
silentTotal int
silentErr error
terminalRows []sqlite.TerminalRunShapeInputRow
terminalErr error
}
func (f *fakeLoopHealthRepo) LoopLivenessActivationChecked() (time.Time, bool, error) {
return f.activationAt, f.activationPublished, f.activationErr
}
func (f *fakeLoopHealthRepo) CountEligibleTriggers(context.Context, string) (int, error) {
return f.eligibleCount, f.eligibleErr
}
func (f *fakeLoopHealthRepo) ListOverdueOrStrandedTriggers(
context.Context, string, time.Time, time.Duration, time.Duration, int,
) ([]sqlite.TriggerHealthRow, int, error) {
return f.triggerRows, f.triggerTotal, f.triggerErr
}
func (f *fakeLoopHealthRepo) ListStuckRuns(
context.Context, string, time.Time, time.Duration, time.Duration, int,
) ([]sqlite.StuckRunRow, int, error) {
return f.stuckRows, f.stuckTotal, f.stuckErr
}
func (f *fakeLoopHealthRepo) ListSilentSuccesses(
context.Context, string, time.Time, time.Time, int,
) ([]sqlite.SilentSuccessRow, int, error) {
return f.silentRows, f.silentTotal, f.silentErr
}
func (f *fakeLoopHealthRepo) ListTerminalRunsInWindow(
context.Context, string, time.Time,
) ([]sqlite.TerminalRunShapeInputRow, error) {
return f.terminalRows, f.terminalErr
}
func baseFakeRepo(now time.Time) *fakeLoopHealthRepo {
return &fakeLoopHealthRepo{
activationAt: now.Add(-48 * time.Hour),
activationPublished: true,
eligibleCount: 1,
}
}
func TestEvaluateLoopHealth_UnpublishedActivationIsUnknown(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.activationPublished = false
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictUnknown {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown)
}
if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 {
t.Errorf("expected empty evidence lists when unknown, got %+v", resp)
}
for shape, count := range resp.TerminalShapeCounts {
if count != 0 {
t.Errorf("shape %q = %d, want 0 (zero-filled)", shape, count)
}
}
}
func TestEvaluateLoopHealth_OverdueTriggerIsDead(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}}
repo.triggerTotal = 1
// Even with a stuck run also present, dead outranks degraded.
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1"}}
repo.stuckTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictDead {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDead)
}
}
func TestEvaluateLoopHealth_StuckRunIsDegraded(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}}
repo.stuckTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictDegraded {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
}
}
func TestEvaluateLoopHealth_SilentSuccessIsDegraded(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.silentRows = []sqlite.SilentSuccessRow{{RunID: "r1", RequestedAt: now.Add(-time.Hour)}}
repo.silentTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictDegraded {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
}
}
func TestEvaluateLoopHealth_NoEligibleTriggerIsNotArmed(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.eligibleCount = 0
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-empty", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictNotArmed {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictNotArmed)
}
if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 {
t.Errorf("expected empty evidence for a workspace with nothing wrong, got %+v", resp)
}
}
func TestEvaluateLoopHealth_NothingWrongIsHealthy(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictHealthy {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictHealthy)
}
}
func TestEvaluateLoopHealth_EchoesThresholds(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
want := LoopHealthThresholdsDTO{
TriggerOverdueGraceSeconds: 180,
TriggerStrandedGraceSeconds: 300,
RunQueuedGraceSeconds: 120,
RunClaimedGraceSeconds: 600,
EvaluationWindowSeconds: 86400,
EvidenceCap: 50,
}
if resp.Thresholds != want {
t.Errorf("thresholds = %+v, want %+v", resp.Thresholds, want)
}
wantWindowStart := now.Add(-24 * time.Hour)
if !resp.EvaluationWindowStart.Equal(wantWindowStart) {
t.Errorf("window start = %v, want %v", resp.EvaluationWindowStart, wantWindowStart)
}
}
func TestEvaluateLoopHealth_ClassifiesTerminalRunsIntoShapeCounts(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
processed := "processed"
repo.terminalRows = []sqlite.TerminalRunShapeInputRow{
{Status: "finished", Outcome: &processed, SessionID: "s1", RequestedAt: now.Add(-time.Hour)},
{Status: "finished", Outcome: &processed, SessionID: "", RequestedAt: now.Add(-time.Hour)},
}
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.TerminalShapeCounts["launched_completed"] != 1 {
t.Errorf("launched_completed = %d, want 1", resp.TerminalShapeCounts["launched_completed"])
}
if resp.TerminalShapeCounts["silent_success"] != 1 {
t.Errorf("silent_success = %d, want 1", resp.TerminalShapeCounts["silent_success"])
}
}
func TestEvaluateLoopHealth_ActivationReadFailureDegrades(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.activationErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonActivationReadFailed)
}
func TestEvaluateLoopHealth_TriggerReadFailureDegrades(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.triggerErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonTriggerReadFailed)
}
func TestEvaluateLoopHealth_EligibleTriggerCountReadFailureDegrades(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.eligibleErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonTriggerReadFailed)
}
func TestEvaluateLoopHealth_StuckRunReadFailureDegrades(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.stuckErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonRunReadFailed)
}
// OPERATOR DECISION F31: both the silent-success read and the
// terminal-shape read map to ReasonTerminalReadFailed.
func TestEvaluateLoopHealth_SilentSuccessReadFailureDegradesWithTerminalReason(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.silentErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonTerminalReadFailed)
}
func TestEvaluateLoopHealth_TerminalShapeReadFailureDegradesWithTerminalReason(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.terminalErr = errors.New("boom")
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
assertDegradedReason(t, err, ReasonTerminalReadFailed)
}
// AC-004.2: unknown outranks dead. An unpublished activation must win
// even when overdue-trigger evidence (the dead condition) is also
// present in the same read.
func TestEvaluateLoopHealth_UnknownOutranksDead(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.activationPublished = false
repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}}
repo.triggerTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictUnknown {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown)
}
}
// AC-004.2: degraded outranks not_armed. Zero eligible triggers (the
// not_armed condition) must not suppress a stuck run already in
// flight from before the workspace went unarmed.
func TestEvaluateLoopHealth_DegradedOutranksNotArmed(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
repo.eligibleCount = 0
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}}
repo.stuckTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if resp.Verdict != VerdictDegraded {
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
}
}
// AC-004.7: evidence rows carry the offending row's identifying data,
// not just a nonzero count that flips the verdict.
func TestEvaluateLoopHealth_EvidenceRowsArePopulated(t *testing.T) {
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
repo := baseFakeRepo(now)
nextRunAt := now.Add(time.Hour)
repo.triggerRows = []sqlite.TriggerHealthRow{{
TriggerID: "trig-1", RoutineID: "rout-1", RoutineName: "Nightly sync",
Condition: "overdue", NextRunAt: &nextRunAt,
}}
repo.triggerTotal = 1
stuckSince := now.Add(-20 * time.Minute)
repo.stuckRows = []sqlite.StuckRunRow{{
RunID: "run-1", AgentProfileID: "agent-1", Status: "claimed",
Condition: "claimed_stuck", StuckSince: &stuckSince,
}}
repo.stuckTotal = 1
repo.silentRows = []sqlite.SilentSuccessRow{{
RunID: "run-2", RequestedAt: now.Add(-time.Hour),
}}
repo.silentTotal = 1
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
if err != nil {
t.Fatalf("evaluate: %v", err)
}
if len(resp.Triggers.Rows) != 1 {
t.Fatalf("trigger rows = %d, want 1", len(resp.Triggers.Rows))
}
gotTrigger := resp.Triggers.Rows[0]
if gotTrigger.TriggerID != "trig-1" || gotTrigger.RoutineID != "rout-1" ||
gotTrigger.RoutineName != "Nightly sync" || gotTrigger.Condition != "overdue" {
t.Errorf("trigger row = %+v, want trig-1/rout-1/Nightly sync/overdue", gotTrigger)
}
if gotTrigger.NextRunAt == nil || !gotTrigger.NextRunAt.Equal(nextRunAt) {
t.Errorf("trigger row next_run_at = %v, want %v", gotTrigger.NextRunAt, nextRunAt)
}
if len(resp.StuckRuns.Rows) != 1 {
t.Fatalf("stuck run rows = %d, want 1", len(resp.StuckRuns.Rows))
}
gotStuck := resp.StuckRuns.Rows[0]
if gotStuck.RunID != "run-1" || gotStuck.AgentProfileID != "agent-1" ||
gotStuck.Status != "claimed" || gotStuck.Condition != "claimed_stuck" {
t.Errorf("stuck run row = %+v, want run-1/agent-1/claimed/claimed_stuck", gotStuck)
}
if gotStuck.StuckSince == nil || !gotStuck.StuckSince.Equal(stuckSince) {
t.Errorf("stuck run row stuck_since = %v, want %v", gotStuck.StuckSince, stuckSince)
}
if len(resp.SilentSuccesses.Rows) != 1 {
t.Fatalf("silent success rows = %d, want 1", len(resp.SilentSuccesses.Rows))
}
if got := resp.SilentSuccesses.Rows[0]; got.RunID != "run-2" {
t.Errorf("silent success row = %+v, want run-2", got)
}
}
func assertDegradedReason(t *testing.T, err error, wantReason string) {
t.Helper()
if err == nil {
t.Fatal("expected an error, got nil")
}
var degraded *LoopHealthDegradedError
if !errors.As(err, &degraded) {
t.Fatalf("error = %v, want *LoopHealthDegradedError", err)
}
if degraded.Reason != wantReason {
t.Errorf("reason = %q, want %q", degraded.Reason, wantReason)
}
}
@@ -12,6 +12,7 @@ import (
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routing"
"github.com/kandev/kandev/internal/office/shared"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
taskmodels "github.com/kandev/kandev/internal/task/models"
workflowmodels "github.com/kandev/kandev/internal/workflow/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
@@ -99,8 +100,12 @@ type Repository interface {
GetTaskWorkflowStepID(ctx context.Context, taskID string) (string, error)
// CancelDisplacedParticipantRun cancels the run(s) the step-entry
// fan-out queued for agentProfileID at (taskID, stepID). Used after a
// claim displaces an agent from a role.
CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error)
// claim displaces an agent from a role. Returns the rows actually
// cancelled so the caller can record each one's loop-liveness
// terminal shape.
CancelDisplacedParticipantRun(
ctx context.Context, taskID, stepID, agentProfileID string,
) ([]runssqlite.CancelledRun, error)
IsTaskWorkflowStepTerminal(ctx context.Context, taskID string) (terminal, hasStep bool, err error)
}
@@ -152,6 +157,17 @@ type RetryCanceller interface {
CancelPendingRetriesForTask(ctx context.Context, taskID string) error
}
// TerminalShapeRecorder classifies and counts a loop-liveness terminal
// shape (office_loop_terminal_total, REQ-OFFICE-LOOP-LIVENESS-005) for
// runs the dashboard package cancelled directly. Mirrors the RetryCanceller
// / TaskCanceller pattern above: the classification logic (activation
// instant, workspace resolution) is stateful and already lives once on
// *office/service.Service, so this seam reuses it instead of a second,
// divergence-prone copy in dashboard.
type TerminalShapeRecorder interface {
RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun)
}
// ProjectBudgetEvaluator evaluates project-scoped budget policies for a
// destination project. Wired to costs.CostService.EvaluateProjectBudget so
// reassigning a task into a project re-checks that project's policies —
@@ -376,34 +392,35 @@ type WorkspaceSettings struct {
// DashboardService provides dashboard, inbox, activity, run, and task-search business logic.
type DashboardService struct {
repo Repository
logger *logger.Logger
activity shared.ActivityLogger
agents shared.AgentReader
costs shared.CostChecker
permissions shared.PermissionLister // optional; nil means no permission items in inbox
settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable
governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable
eb bus.EventBus // optional; nil means no events are published
retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign
taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled
taskDetacher TaskDetacher // optional; canonical empty-parent mutation
taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated
sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal
reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations
engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine
approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs
skillLister SkillLister // optional; nil means skill_count is always 0
routineLister RoutineLister // optional; nil means routine_count is always 0
failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries
failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources
markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint
decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints
routingProvider RoutingProvider // optional; nil disables /routing endpoints (503)
attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses
runResolver RunResolver // optional; nil means status-change activity rows have no run_id
assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization
projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies
repo Repository
logger *logger.Logger
activity shared.ActivityLogger
agents shared.AgentReader
costs shared.CostChecker
permissions shared.PermissionLister // optional; nil means no permission items in inbox
settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable
governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable
eb bus.EventBus // optional; nil means no events are published
retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign
terminalShapeRecorder TerminalShapeRecorder // optional; nil means displaced-run cancellations aren't counted in office_loop_terminal_total
taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled
taskDetacher TaskDetacher // optional; canonical empty-parent mutation
taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated
sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal
reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations
engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine
approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs
skillLister SkillLister // optional; nil means skill_count is always 0
routineLister RoutineLister // optional; nil means routine_count is always 0
failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries
failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources
markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint
decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints
routingProvider RoutingProvider // optional; nil disables /routing endpoints (503)
attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses
runResolver RunResolver // optional; nil means status-change activity rows have no run_id
assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization
projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies
// officeSessionIdentity gates RecordAgentDecision's use of the caller's
// own session id. Defaults false (zero value); set via
// SetOfficeSessionIdentity, wired from features.officeSessionIdentity.
@@ -516,6 +533,13 @@ func (s *DashboardService) SetRetryCanceller(c RetryCanceller) {
s.retryCanceller = c
}
// SetTerminalShapeRecorder wires the seam used to count a loop-liveness
// terminal shape for a run this package cancelled directly (a displaced
// participant's queued run).
func (s *DashboardService) SetTerminalShapeRecorder(r TerminalShapeRecorder) {
s.terminalShapeRecorder = r
}
// SetRunResolver wires the seam used to attribute status-change activity
// rows back to the office run that produced them. Optional; when unset,
// rows are logged with an empty run_id.
@@ -501,12 +501,17 @@ func (s *DashboardService) cancelDisplacedRun(ctx context.Context, taskID, stepI
if displacedAgentID == "" {
return
}
if _, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID); err != nil {
cancelled, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID)
if err != nil {
s.logger.Warn("cancel displaced participant run failed",
zap.String("task_id", taskID),
zap.String("step_id", stepID),
zap.String("agent_profile_id", displacedAgentID),
zap.Error(err))
return
}
if s.terminalShapeRecorder != nil {
s.terminalShapeRecorder.RecordCancelledRunTerminalShapes(ctx, cancelled)
}
}
@@ -0,0 +1,139 @@
package dashboard_test
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/kandev/kandev/internal/office/dashboard"
"github.com/kandev/kandev/internal/office/models"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
)
// recordingTerminalShapeRecorder captures RecordCancelledRunTerminalShapes
// calls so tests can verify a displaced-participant cancellation reaches
// office_loop_terminal_total's classification path.
type recordingTerminalShapeRecorder struct {
calls [][]runssqlite.CancelledRun
}
func (r *recordingTerminalShapeRecorder) RecordCancelledRunTerminalShapes(
_ context.Context, cancelled []runssqlite.CancelledRun,
) {
r.calls = append(r.calls, cancelled)
}
var _ dashboard.TerminalShapeRecorder = (*recordingTerminalShapeRecorder)(nil)
// TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape is
// the Testing round 3 regression test. cancelDisplacedRun calls
// repo.CancelDisplacedParticipantRun directly, which bypassed
// office_loop_terminal_total entirely — the same bypass class Review
// round 1 fixed for the single-run CancelRun call sites. The setup mirrors
// TestAddTaskReviewer_ClaimingAutoSeatCancelsDisplacedRunAndLogsActivity;
// this test additionally asserts the terminal-shape recorder is invoked
// with the displaced run's cancelled row.
func TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape(t *testing.T) {
deps := newTestDeps(t)
recorder := &recordingTerminalShapeRecorder{}
deps.svc.SetTerminalShapeRecorder(recorder)
insertTestTask(t, deps.db, "claim-shape1", "ws-claim-shape", "C", "todo", 2)
stepID := "step-claim-shape1"
if _, err := deps.db.Exec(`
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape', '', datetime('now'), datetime('now'))
`); err != nil {
t.Fatalf("seed claiming agent profile: %v", err)
}
if _, err := deps.db.Exec(`
INSERT INTO workflow_step_participants
(id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance)
VALUES ('auto-seat-claim-shape1', ?, 'claim-shape1', 'reviewer', 'agent-auto', 1, 0, 'auto')
`, stepID); err != nil {
t.Fatalf("seed auto seat: %v", err)
}
run := &models.Run{
AgentProfileID: "agent-auto",
Reason: "task_assigned",
Payload: `{"task_id":"claim-shape1","workflow_step_id":"` + stepID + `"}`,
Status: models.RunStatusQueued,
CoalescedCount: 1,
}
if err := deps.repo.CreateRun(context.Background(), run); err != nil {
t.Fatalf("seed displaced run: %v", err)
}
body := `{"agent_profile_id":"agent-claiming"}`
req := httptest.NewRequest(http.MethodPost,
"/api/v1/office/tasks/claim-shape1/reviewers", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
deps.router.ServeHTTP(w, req)
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String())
}
if len(recorder.calls) != 1 {
t.Fatalf("terminal shape recorder calls = %d, want 1", len(recorder.calls))
}
if len(recorder.calls[0]) != 1 || recorder.calls[0][0].ID != run.ID {
t.Fatalf("recorded cancelled rows = %+v, want [%s]", recorder.calls[0], run.ID)
}
}
// TestSetTerminalShapeRecorder_NilTolerated keeps the wiring optional: when
// no recorder is registered, the displaced-run cancellation itself must
// still succeed (dashboard tests run without the office service wired in).
func TestSetTerminalShapeRecorder_NilTolerated(t *testing.T) {
deps := newTestDeps(t)
insertTestTask(t, deps.db, "claim-shape-nil", "ws-claim-shape-nil", "C", "todo", 2)
stepID := "step-claim-shape-nil"
if _, err := deps.db.Exec(`
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape-nil', '', datetime('now'), datetime('now'))
`); err != nil {
t.Fatalf("seed claiming agent profile: %v", err)
}
if _, err := deps.db.Exec(`
INSERT INTO workflow_step_participants
(id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance)
VALUES ('auto-seat-claim-shape-nil', ?, 'claim-shape-nil', 'reviewer', 'agent-auto', 1, 0, 'auto')
`, stepID); err != nil {
t.Fatalf("seed auto seat: %v", err)
}
run := &models.Run{
AgentProfileID: "agent-auto",
Reason: "task_assigned",
Payload: `{"task_id":"claim-shape-nil","workflow_step_id":"` + stepID + `"}`,
Status: models.RunStatusQueued,
CoalescedCount: 1,
}
if err := deps.repo.CreateRun(context.Background(), run); err != nil {
t.Fatalf("seed displaced run: %v", err)
}
body := `{"agent_profile_id":"agent-claiming"}`
req := httptest.NewRequest(http.MethodPost,
"/api/v1/office/tasks/claim-shape-nil/reviewers", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
deps.router.ServeHTTP(w, req)
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String())
}
gotRun, err := deps.repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get displaced run: %v", err)
}
if gotRun.Status != "cancelled" {
t.Fatalf("displaced run status = %q, want cancelled", gotRun.Status)
}
}
@@ -134,7 +134,7 @@ func TestReconcileAgentWorkingStatus_ClearsTerminalOwner(t *testing.T) {
if _, err := repo.MarkAgentWorking(ctx, agent.ID, claimed.ID); err != nil {
t.Fatalf("mark agent working: %v", err)
}
if err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil {
if _, err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil {
t.Fatalf("finish run: %v", err)
}
@@ -485,6 +485,13 @@ type Run struct {
// EarliestRetryAt is the earliest moment a parked run should be re-
// resolved. Set only when at least one degraded route is auto-retryable.
EarliestRetryAt *time.Time `json:"earliest_retry_at,omitempty" db:"earliest_retry_at"`
// CausationID is copied from the agent_wakeup_requests row that
// created this run (REQ-OFFICE-LOOP-LIVENESS-002). "" means
// uncorrelated — either a legacy pre-migration row or a run created
// off a wake that never carried an id. Never a join/group-by key
// without excluding "" first.
CausationID string `json:"causation_id,omitempty" db:"causation_id"`
}
// RouteAttempt records one provider attempt inside a Run. Each fallback
@@ -598,6 +605,11 @@ type RoutineRun struct {
StartedAt *time.Time `json:"started_at" db:"started_at"`
CompletedAt *time.Time `json:"completed_at" db:"completed_at"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
// CausationID is minted once per fire in dispatchRoutineRun
// (REQ-OFFICE-LOOP-LIVENESS-002) — the origin id every wakeup
// request and run this fire produces carries forward. "" only for
// rows written before this feature.
CausationID string `json:"causation_id,omitempty" db:"causation_id"`
}
// ApprovalType constants for approval request types.
@@ -226,7 +226,7 @@ func TestMarkAgentWorking_SuccessorTakeover(t *testing.T) {
// Run A leaves 'claimed' (finishes) but its own clearAgentWorking has not
// run yet — the un-transacted window handleAgentCompleted leaves open.
if err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil {
if _, err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil {
t.Fatalf("finish run A: %v", err)
}
@@ -127,6 +127,7 @@ func (r *Repository) initSchema() error {
return fmt.Errorf("required office migration: %w", err)
}
r.activateRunOutcome()
r.activateLoopLiveness()
return nil
}
@@ -413,6 +414,9 @@ func (r *Repository) createRunTables() error {
-- re-deriving it against a context_snapshot a coalesced wakeup
-- may have since patched.
continuation_scope TEXT NOT NULL DEFAULT '',
-- causation_id (office-loop-liveness): copied from the wakeup
-- request that created this run. '' means uncorrelated.
causation_id TEXT NOT NULL DEFAULT '',
requested_at TIMESTAMP NOT NULL,
claimed_at TIMESTAMP,
finished_at TIMESTAMP
@@ -492,6 +496,7 @@ func (r *Repository) createRoutineTables() error {
started_at TIMESTAMP,
completed_at TIMESTAMP,
created_at TIMESTAMP NOT NULL,
causation_id TEXT NOT NULL DEFAULT '',
FOREIGN KEY (routine_id) REFERENCES office_routines(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_office_routine_runs_active_fingerprint
@@ -733,7 +738,8 @@ func (r *Repository) createAgentWakeupRequestTable() error {
run_id TEXT NOT NULL DEFAULT '',
requested_at TIMESTAMP NOT NULL,
claimed_at TIMESTAMP,
finished_at TIMESTAMP
finished_at TIMESTAMP,
causation_id TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_wakeup_agent_status ON agent_wakeup_requests(agent_profile_id, status);
CREATE UNIQUE INDEX IF NOT EXISTS idx_wakeup_idempotency ON agent_wakeup_requests(idempotency_key)
@@ -62,6 +62,7 @@ func (r *Repository) runMigrations() error {
r.migrateParentWakeReceiptColumns()
r.migrate.Apply("task_workspace_groups.ownership_generation",
`ALTER TABLE task_workspace_groups ADD COLUMN ownership_generation INTEGER NOT NULL DEFAULT 1`)
r.migrateLoopLivenessCausationID()
if err := r.migrate.Err(); err != nil {
return err
}
@@ -83,6 +84,35 @@ func (r *Repository) backfillRoutineTriggerTimezones() error {
return nil
}
// migrateLoopLivenessCausationID adds the causation_id correlation
// column to all three rows in the wake-to-run chain
// (REQ-OFFICE-LOOP-LIVENESS-002): the routine fire, the wakeup request
// it produces, and the run the wakeup request produces. "" rather than
// NULL, because every existing row predates this feature and a reader
// must treat "legacy" and "uncorrelated" identically. Each column gets
// its own partial index (excluding "") so a backward correlation walk
// from a run to its originating fire is a direct lookup on every
// dialect, not a table scan — NFR-2 is symmetric with the forward walk.
func (r *Repository) migrateLoopLivenessCausationID() {
_ = r.migrate.Apply("office_routine_runs.causation_id",
`ALTER TABLE office_routine_runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
_ = r.migrate.Apply("idx_office_routine_runs_causation_id",
`CREATE INDEX IF NOT EXISTS idx_office_routine_runs_causation_id
ON office_routine_runs(causation_id) WHERE causation_id != ''`)
_ = r.migrate.Apply("agent_wakeup_requests.causation_id",
`ALTER TABLE agent_wakeup_requests ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
_ = r.migrate.Apply("idx_agent_wakeup_requests_causation_id",
`CREATE INDEX IF NOT EXISTS idx_agent_wakeup_requests_causation_id
ON agent_wakeup_requests(causation_id) WHERE causation_id != ''`)
_ = r.migrate.Apply("runs.causation_id",
`ALTER TABLE runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
_ = r.migrate.Apply("idx_runs_causation_id",
`CREATE INDEX IF NOT EXISTS idx_runs_causation_id
ON runs(causation_id) WHERE causation_id != ''`)
}
// migrateContinuationScope adds runs.continuation_scope for databases
// created before WO-16's claim-time scope persistence. Existing rows receive
// a scope from their stored context snapshot so queued or claimed taskless
@@ -266,20 +266,33 @@ func (r *Repository) SetWorkspaceAgentFailureThreshold(
return err
}
// MarkRunFailed sets a run row to status=failed with a verbatim
// error message and stamps finished_at. Idempotent: re-marking is a
// no-op when the run is already in the same state.
// MarkRunFailed sets a run row to status=failed with a verbatim error
// message and stamps finished_at. Guarded to status = 'claimed'
// (Review round 3, R3-1): every caller reaches this after reading a
// claimed run, so without the guard a cancel that commits in between
// would have its 'cancelled' status and finished_at overwritten by
// this write. Returns wrote=false — the same "nothing changed" signal
// runs/repository/sqlite.FinishRun gives its callers — when the row
// was no longer claimed, whether because it was already marked failed
// or because it reached a different terminal state first.
func (r *Repository) MarkRunFailed(
ctx context.Context, runID, errorMessage string,
) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
) (bool, error) {
res, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE runs
SET status = 'failed',
error_message = ?,
finished_at = COALESCE(finished_at, ?)
WHERE id = ?
WHERE id = ? AND status = 'claimed'
`), errorMessage, time.Now().UTC(), runID)
return err
if err != nil {
return false, err
}
n, err := res.RowsAffected()
if err != nil {
return false, err
}
return n > 0, nil
}
// DismissInboxItem records a per-user dismissal. Idempotent — a second
@@ -225,9 +225,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
"2026-01-02 09:00:00", "2026-01-02 10:00:00", "2026-01-02 11:00:00", "{}")
seedSummaryRun(t, repo, "mr-other", "agent-a", "queued", "2026-01-03 09:00:00", "", "", "{}")
if err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded"); err != nil {
wrote, err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded")
if err != nil {
t.Fatalf("MarkRunFailed: %v", err)
}
if !wrote {
t.Error("wrote = false, want true (mr-open was claimed)")
}
run, err := repo.GetRun(ctx, "mr-open")
if err != nil {
t.Fatalf("GetRun: %v", err)
@@ -242,10 +246,21 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
t.Error("FinishedAt is nil, want it stamped")
}
// Re-marking is idempotent and must not move an existing finished_at.
if err := repo.MarkRunFailed(ctx, "mr-done", "later failure"); err != nil {
// Review round 3, R3-1: MarkRunFailed is guarded to status = 'claimed',
// so a run already in a different terminal state (here, 'finished') is
// left untouched — a guarded no-op, not a re-stamp. This is a
// deliberate change from the pre-guard "idempotent re-mark" contract:
// unconditionally re-marking an already-terminal run is exactly the
// TOCTOU bug this guard exists to close (a concurrent cancel/finish
// must never have its real terminal state overwritten by a stale
// failure write).
wrote, err = repo.MarkRunFailed(ctx, "mr-done", "later failure")
if err != nil {
t.Fatalf("MarkRunFailed (already finished): %v", err)
}
if wrote {
t.Error("wrote = true, want false (mr-done was already finished, not claimed)")
}
run, err = repo.GetRun(ctx, "mr-done")
if err != nil {
t.Fatalf("GetRun: %v", err)
@@ -253,8 +268,11 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
if run.FinishedAt == nil || run.FinishedAt.Format("2006-01-02 15:04:05") != "2026-01-02 11:00:00" {
t.Errorf("FinishedAt = %v, want the original 2026-01-02 11:00:00", run.FinishedAt)
}
if run.ErrorMessage != "later failure" {
t.Errorf("ErrorMessage = %q, want it overwritten", run.ErrorMessage)
if run.ErrorMessage != "" {
t.Errorf("ErrorMessage = %q, want it left untouched (guarded no-op)", run.ErrorMessage)
}
if run.Status != "finished" {
t.Errorf("Status = %q, want finished (unchanged)", run.Status)
}
// An untouched run keeps its status.
@@ -266,9 +284,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
}
// Marking a missing run is a silent no-op, not an error.
if err := repo.MarkRunFailed(ctx, "no-such-run", "x"); err != nil {
wrote, err = repo.MarkRunFailed(ctx, "no-such-run", "x")
if err != nil {
t.Errorf("MarkRunFailed(missing) = %v, want nil", err)
}
if wrote {
t.Error("wrote = true, want false (no such run)")
}
}
func TestInboxDismissals_UpsertQueryAndList(t *testing.T) {
@@ -0,0 +1,301 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"time"
"github.com/kandev/kandev/internal/db/dialect"
)
// TriggerHealthRow is one overdue or stranded cron trigger
// (REQ-OFFICE-LOOP-LIVENESS-004). Condition names which predicate
// matched so a single evidence list can carry both kinds (AC-004.7).
type TriggerHealthRow struct {
TriggerID string `db:"id"`
RoutineID string `db:"routine_id"`
RoutineName string `db:"routine_name"`
NextRunAt *time.Time `db:"next_run_at"`
LastFiredAt *time.Time `db:"last_fired_at"`
Condition string `db:"-"`
}
// CountEligibleTriggers reports how many enabled cron triggers exist
// for the workspace, regardless of due state — the input to the
// not_armed verdict (AC-004.6): zero means no trigger is scheduled at
// all, not that none happen to be due right now.
func (r *Repository) CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error) {
var count int
err := r.ro.GetContext(ctx, &count, r.ro.Rebind(`
SELECT COUNT(*) FROM office_routine_triggers t
JOIN office_routines rt ON rt.id = t.routine_id
WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ?
`), workspaceID)
return count, err
}
// ListOverdueOrStrandedTriggers returns eligible cron triggers that are
// overdue (armed but past grace) or stranded (claimed, or never fired
// at all, and not re-armed past grace), capped at limit, with the
// untruncated total from the same statement (AC-004.9, AC-004.12). The
// stranded clock is the claim instant COALESCE(last_fired_at,
// created_at), not last_fired_at alone: a trigger that has never fired
// (e.g. a malformed cron_expression that never got a next_run_at)
// ages from its own creation, not from the epoch. A trigger claimed
// less than strandedGrace ago is a claiming trigger — excluded here
// rather than given a third label (AC-004.15).
func (r *Repository) ListOverdueOrStrandedTriggers(
ctx context.Context, workspaceID string, now time.Time,
overdueGrace, strandedGrace time.Duration, limit int,
) ([]TriggerHealthRow, int, error) {
overdueBefore := now.Add(-overdueGrace)
strandedBefore := now.Add(-strandedGrace)
type row struct {
TriggerHealthRow
Total int `db:"total"`
}
var rows []row
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
SELECT t.id AS id, t.routine_id AS routine_id, rt.name AS routine_name,
t.next_run_at AS next_run_at, t.last_fired_at AS last_fired_at,
COUNT(*) OVER () AS total
FROM office_routine_triggers t
JOIN office_routines rt ON rt.id = t.routine_id
WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ?
AND (
(t.next_run_at IS NOT NULL AND t.next_run_at <= ?)
OR (t.next_run_at IS NULL AND COALESCE(t.last_fired_at, t.created_at) <= ?)
)
ORDER BY (t.next_run_at IS NULL) DESC, t.next_run_at ASC, t.id ASC
LIMIT ?
`), workspaceID, overdueBefore, strandedBefore, limit)
if err != nil {
return nil, 0, err
}
out := make([]TriggerHealthRow, 0, len(rows))
total := 0
for _, rr := range rows {
hr := rr.TriggerHealthRow
if hr.NextRunAt == nil {
hr.Condition = "stranded"
} else {
hr.Condition = "overdue"
}
out = append(out, hr)
total = rr.Total
}
return out, total, nil
}
// StuckRunRow is one queued or claimed run the scheduler should have
// progressed but has not (REQ-OFFICE-LOOP-LIVENESS-004). StuckSince is
// the clock the row aged from: claimed_at for a claimed run,
// COALESCE(scheduled_retry_at, requested_at) for a queued one.
type StuckRunRow struct {
RunID string `db:"id"`
AgentProfileID string `db:"agent_profile_id"`
Status string `db:"status"`
StuckSince *time.Time `db:"stuck_since"`
Condition string `db:"-"`
}
// stuckRunsUnionSQL is the shared claimed+queued candidate set behind
// ListStuckRuns' stuck_since column, unchanged by dialect: both source
// expressions (`claimed_at`, `COALESCE(scheduled_retry_at, requested_at)`)
// are plain TIMESTAMP-typed columns on both engines.
const stuckRunsUnionSQL = `
SELECT r.id, r.agent_profile_id, r.status, r.claimed_at AS stuck_since
FROM runs r
JOIN agent_profiles a ON a.id = r.agent_profile_id
WHERE a.workspace_id = ? AND r.status = 'claimed'
AND (r.claimed_at IS NULL OR r.claimed_at <= ?)
UNION ALL
SELECT r.id, r.agent_profile_id, r.status,
COALESCE(r.scheduled_retry_at, r.requested_at) AS stuck_since
FROM runs r
JOIN agent_profiles a ON a.id = r.agent_profile_id
WHERE a.workspace_id = ? AND r.status = 'queued'
AND r.current_route_attempt_seq = 0
AND r.routing_blocked_status IS NULL
AND COALESCE(r.scheduled_retry_at, r.requested_at) <= ?
AND NOT EXISTS (
SELECT 1 FROM runs s
WHERE s.agent_profile_id = r.agent_profile_id AND s.status = 'claimed'
)
`
// stuckSinceColumnExpr and stuckSinceLayout keep stuck_since portable
// across dialects. stuck_since is a computed UNION ALL column, so it
// carries no declared type for the driver to auto-parse into
// time.Time (unlike a direct table column read): on SQLite, strftime()
// normalizes it to a fixed ISO-8601 UTC string that the scan target
// (sql.NullString) parses by hand below. Postgres has no strftime;
// selecting the raw TIMESTAMP column there returns a native time.Time
// that database/sql's Scan already formats into an RFC3339Nano string
// on assignment to *sql.NullString, so one scan type and one dialect
// switch cover both engines without a second query or row type.
func stuckSinceColumnExpr(driver string) string {
if dialect.IsPostgres(driver) {
return "w.stuck_since"
}
return "strftime('%Y-%m-%dT%H:%M:%fZ', w.stuck_since)"
}
func stuckSinceLayout(driver string) string {
if dialect.IsPostgres(driver) {
return time.RFC3339Nano
}
return "2006-01-02T15:04:05.000Z"
}
// ListStuckRuns returns claimed-stuck and queued-stuck runs in one
// list, each row naming which condition it matched, ordered claimed
// first then by stuck instant (NULL first), capped with the same-
// statement untruncated total (AC-004.9, AC-004.12). Neither predicate
// is windowed (AC-004.5): a run stuck longer than
// officeLoopEvaluationWindow is the one most worth surfacing.
//
// The queued half mirrors ClaimNextEligibleRun's own filter
// (AC-004.17) as an accepted limitation: it restricts to
// current_route_attempt_seq = 0, so a routed run that has dispatched
// at least once and requeued reads as silent rather than stuck. That
// gap is intentional (OPERATOR DECISION F29/F30) — fixing it here
// would let this read disagree with what the scheduler would actually
// claim next.
func (r *Repository) ListStuckRuns(
ctx context.Context, workspaceID string, now time.Time,
queuedGrace, claimedGrace time.Duration, limit int,
) ([]StuckRunRow, int, error) {
claimedBefore := now.Add(-claimedGrace)
queuedBefore := now.Add(-queuedGrace)
driver := r.ro.DriverName()
type row struct {
ID string `db:"id"`
AgentProfileID string `db:"agent_profile_id"`
Status string `db:"status"`
StuckSince sql.NullString `db:"stuck_since"`
Total int `db:"total"`
}
var rows []row
query := fmt.Sprintf(`
SELECT w.id AS id, w.agent_profile_id AS agent_profile_id, w.status AS status,
%s AS stuck_since,
COUNT(*) OVER () AS total
FROM (%s) w
ORDER BY (w.status = 'claimed') DESC, (w.stuck_since IS NULL) DESC, w.stuck_since ASC, w.id ASC
LIMIT ?
`, stuckSinceColumnExpr(driver), stuckRunsUnionSQL)
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(query),
workspaceID, claimedBefore, workspaceID, queuedBefore, limit)
if err != nil {
return nil, 0, err
}
layout := stuckSinceLayout(driver)
out := make([]StuckRunRow, 0, len(rows))
total := 0
for _, rr := range rows {
hr := StuckRunRow{RunID: rr.ID, AgentProfileID: rr.AgentProfileID, Status: rr.Status}
if rr.StuckSince.Valid {
parsed, perr := time.Parse(layout, rr.StuckSince.String)
if perr != nil {
return nil, 0, fmt.Errorf("parse stuck_since %q: %w", rr.StuckSince.String, perr)
}
hr.StuckSince = &parsed
}
if hr.Status == "claimed" {
hr.Condition = "claimed_stuck"
} else {
hr.Condition = "queued_stuck"
}
out = append(out, hr)
total = rr.Total
}
return out, total, nil
}
// SilentSuccessRow is one finished run that asserted success but never
// named a session (AC-005.4 made an evidence row).
type SilentSuccessRow struct {
RunID string `db:"id"`
RequestedAt time.Time `db:"requested_at"`
FinishedAt *time.Time `db:"finished_at"`
}
// ListSilentSuccesses returns finished/processed/sessionless runs whose
// COALESCE(finished_at, requested_at) falls in [windowStart, now),
// requested at or after the activation instant (AC-005.6: a run
// predating activation is pre_activation, never a silent success),
// newest first, capped with the same-statement untruncated total
// (AC-004.7, AC-004.9, AC-004.16).
func (r *Repository) ListSilentSuccesses(
ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int,
) ([]SilentSuccessRow, int, error) {
type row struct {
SilentSuccessRow
Total int `db:"total"`
}
var rows []row
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
SELECT r.id AS id, r.requested_at AS requested_at, r.finished_at AS finished_at,
COUNT(*) OVER () AS total
FROM runs r
JOIN agent_profiles a ON a.id = r.agent_profile_id
WHERE a.workspace_id = ? AND r.status = 'finished' AND r.outcome = 'processed'
AND (r.session_id IS NULL OR r.session_id = '')
AND r.requested_at >= ?
AND COALESCE(r.finished_at, r.requested_at) >= ?
ORDER BY COALESCE(r.finished_at, r.requested_at) DESC, r.id ASC
LIMIT ?
`), workspaceID, activationAt, windowStart, limit)
if err != nil {
return nil, 0, err
}
out := make([]SilentSuccessRow, 0, len(rows))
total := 0
for _, rr := range rows {
out = append(out, rr.SilentSuccessRow)
total = rr.Total
}
return out, total, nil
}
// TerminalRunShapeInputRow carries exactly the columns
// service.ClassifyTerminalRun needs, for every terminal run (not
// queued, not claimed) in the evaluation window (AC-004.16). The
// caller classifies and buckets; this method reads no free text
// (AC-005.7) and is side-effect free.
type TerminalRunShapeInputRow struct {
Status string `db:"status"`
Outcome *string `db:"outcome"`
SessionID string `db:"session_id"`
RequestedAt time.Time `db:"requested_at"`
FinishedAt *time.Time `db:"finished_at"`
}
// ListTerminalRunsInWindow returns every terminal run for the
// workspace whose COALESCE(finished_at, requested_at) falls in
// [windowStart, now). Unbounded by officeLoopEvidenceCap: this feeds
// the terminal-shape counts, not an evidence list.
func (r *Repository) ListTerminalRunsInWindow(
ctx context.Context, workspaceID string, windowStart time.Time,
) ([]TerminalRunShapeInputRow, error) {
var rows []TerminalRunShapeInputRow
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
SELECT r.status AS status, r.outcome AS outcome, r.session_id AS session_id,
r.requested_at AS requested_at, r.finished_at AS finished_at
FROM runs r
JOIN agent_profiles a ON a.id = r.agent_profile_id
WHERE a.workspace_id = ? AND r.status NOT IN ('queued', 'claimed')
AND COALESCE(r.finished_at, r.requested_at) >= ?
`), workspaceID, windowStart)
if err != nil {
return nil, err
}
if rows == nil {
rows = []TerminalRunShapeInputRow{}
}
return rows, nil
}
@@ -0,0 +1,86 @@
package sqlite_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
)
// AC-004.15 / Claim-instant terminology (Review round 3, R3-3): a
// trigger that has never fired (last_fired_at IS NULL, e.g. a
// malformed cron_expression that never got a next_run_at) is still
// claiming, not stranded, until strandedGrace has elapsed since its
// own created_at — the spec's claim instant is
// COALESCE(last_fired_at, created_at), not last_fired_at alone. The
// old predicate treated any never-fired trigger as immediately
// eligible for stranded, regardless of how recently it was created.
func TestListOverdueOrStrandedTriggers_NeverFiredTriggerNotStrandedBeforeCreatedGrace(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
routine := seedCronTrigger(t, repo, "ws-a", "NeverFired")
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "",
Timezone: "UTC", Enabled: true, NextRunAt: nil,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger: %v", err)
}
// Created 1 minute before now: well inside the 5-minute stranded
// grace, never fired, never armed with a next_run_at.
createdAt := now.Add(-1 * time.Minute)
if _, err := repo.ExecRaw(ctx,
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`,
createdAt, trigger.ID); err != nil {
t.Fatalf("seed never-fired state: %v", err)
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 || len(rows) != 0 {
t.Fatalf("got %d rows, total %d, want 0/0 (a trigger created 1m ago is still claiming, not stranded)", len(rows), total)
}
}
func TestListOverdueOrStrandedTriggers_NeverFiredTriggerStrandedAfterCreatedGrace(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
routine := seedCronTrigger(t, repo, "ws-a", "NeverFiredStale")
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "",
Timezone: "UTC", Enabled: true, NextRunAt: nil,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger: %v", err)
}
// Created 6 minutes before now: past the 5-minute stranded grace,
// never fired, never armed.
createdAt := now.Add(-6 * time.Minute)
if _, err := repo.ExecRaw(ctx,
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`,
createdAt, trigger.ID); err != nil {
t.Fatalf("seed never-fired stale state: %v", err)
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1 (a trigger created 6m ago that never fired must be stranded)", len(rows), total)
}
if rows[0].Condition != "stranded" {
t.Errorf("condition = %q, want stranded", rows[0].Condition)
}
}
@@ -0,0 +1,169 @@
package sqlite_test
import (
"context"
"testing"
"time"
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/repository/sqlite"
)
func seedFinishedRun(
t *testing.T, repo *sqlite.Repository, db *sqlx.DB,
agentID, outcome, sessionID string, requestedAt time.Time, finishedAt *time.Time,
) string {
t.Helper()
ctx := context.Background()
run := &models.Run{
AgentProfileID: agentID, Reason: "task_assigned", Payload: "{}",
Status: "queued", CoalescedCount: 1, SessionID: sessionID,
}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
if _, err := db.Exec(
`UPDATE runs SET status = 'finished', outcome = ?, requested_at = ?, finished_at = ? WHERE id = ?`,
outcome, requestedAt, finishedAt, run.ID,
); err != nil {
t.Fatalf("finish run: %v", err)
}
return run.ID
}
func TestListSilentSuccesses_ReturnsFinishedProcessedSessionlessRun(t *testing.T) {
repo, db := newTestRepoWithDB(t)
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
finishedAt := now.Add(-time.Hour)
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt)
windowStart := now.Add(-24 * time.Hour)
activationAt := now.Add(-48 * time.Hour)
rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
}
}
// AC-005.6, applied to the AC-004.5/.7 evidence list: a legacy run
// requested before the activation instant is pre_activation, never a
// silent success, even when it is otherwise indistinguishable from one
// and falls inside the evaluation window.
func TestListSilentSuccesses_PreActivationRunExcluded(t *testing.T) {
repo, db := newTestRepoWithDB(t)
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
finishedAt := now.Add(-time.Hour)
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt)
windowStart := now.Add(-24 * time.Hour)
activationAt := now.Add(-time.Minute) // activated after this run was requested
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (run predates activation, must not read as silent success)", total)
}
}
// AC-004.16: a terminal run with a NULL finished_at still counts
// within the window, aged from requested_at via COALESCE.
func TestListSilentSuccesses_NullFinishedAtStillInsideWindow(t *testing.T) {
repo, db := newTestRepoWithDB(t)
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-time.Hour), nil)
windowStart := now.Add(-24 * time.Hour)
activationAt := now.Add(-48 * time.Hour)
rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1 (NULL finished_at falls back to requested_at)", len(rows), total)
}
if rows[0].FinishedAt != nil {
t.Errorf("FinishedAt = %v, want nil", rows[0].FinishedAt)
}
}
func TestListSilentSuccesses_LaunchedRunIsNotSilent(t *testing.T) {
repo, db := newTestRepoWithDB(t)
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
finishedAt := now.Add(-time.Hour)
seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt)
windowStart := now.Add(-24 * time.Hour)
activationAt := now.Add(-48 * time.Hour)
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (a run with a session id is not silent)", total)
}
}
func TestListSilentSuccesses_OutsideWindowExcluded(t *testing.T) {
repo, db := newTestRepoWithDB(t)
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
finishedAt := now.Add(-30 * time.Hour)
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-31*time.Hour), &finishedAt)
windowStart := now.Add(-24 * time.Hour)
activationAt := now.Add(-48 * time.Hour)
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (outside the 24h evaluation window)", total)
}
}
func TestListTerminalRunsInWindow_ExcludesQueuedAndClaimed(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, queued); err != nil {
t.Fatalf("create queued: %v", err)
}
claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, claimed); err != nil {
t.Fatalf("create claimed: %v", err)
}
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, claimed.ID); err != nil {
t.Fatalf("claim: %v", err)
}
finishedAt := now.Add(-time.Hour)
seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt)
rows, err := repo.ListTerminalRunsInWindow(ctx, "ws-a", now.Add(-24*time.Hour))
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 1 {
t.Fatalf("got %d rows, want 1 (queued/claimed excluded)", len(rows))
}
if rows[0].Status != "finished" {
t.Errorf("status = %q, want finished", rows[0].Status)
}
}
@@ -0,0 +1,105 @@
package sqlite_test
import (
"context"
"testing"
"time"
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/repository/sqlite"
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/testutil"
)
// TestPostgresListStuckRuns is the PostgreSQL twin of the ListStuckRuns cases
// in loop_health_stuck_runs_test.go. ListStuckRuns' stuck_since column is
// computed by a UNION ALL over two different source columns, which loses its
// declared type on read — the original fix normalized it with SQLite-only
// strftime(), a function Postgres does not have, so this exercised the same
// query shape (a claimed-stuck and a queued-stuck row, one list, ordered) end
// to end against a real Postgres backend. Skips unless
// KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresListStuckRuns(t *testing.T) {
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
t.Fatalf("init settings store: %v", err)
}
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init task repo: %v", err)
}
repo, err := sqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init office repo: %v", err)
}
ctx := context.Background()
// Postgres timestamp columns store microsecond precision; truncate
// before the round-trip so the later Equal() comparisons aren't
// comparing against nanosecond digits the column never persisted.
now := time.Now().UTC().Truncate(time.Microsecond)
// Two distinct agent profiles: sharing one would put the queued run
// behind a claimed sibling on the same agent_profile_id, excluding
// it under AC-OFFICE-LOOP-LIVENESS-004.17 (a busy agent's steady
// state, not a stall) — this test wants both rows reported.
// agent_profiles.agent_id carries a foreign key to agents(id),
// enforced on Postgres (not on SQLite's default connection).
for _, agentID := range []string{"agent-a", "agent-b"} {
if _, err := db.ExecContext(ctx, db.Rebind(
`INSERT INTO agents (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)`,
), agentID, agentID, now, now); err != nil {
t.Fatalf("seed agent %s: %v", agentID, err)
}
if _, err := db.ExecContext(ctx, db.Rebind(`
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`), agentID, agentID, agentID, agentID, "ws-pg-stuck", "developer", now, now); err != nil {
t.Fatalf("seed agent profile %s: %v", agentID, err)
}
}
claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, claimed); err != nil {
t.Fatalf("create claimed run: %v", err)
}
longAgo := now.Add(-72 * time.Hour)
if _, err := db.ExecContext(ctx, db.Rebind(
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
), longAgo, claimed.ID); err != nil {
t.Fatalf("seed claimed state: %v", err)
}
queued := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, queued); err != nil {
t.Fatalf("create queued run: %v", err)
}
old := now.Add(-10 * time.Minute)
if _, err := db.ExecContext(ctx, db.Rebind(
`UPDATE runs SET requested_at = ? WHERE id = ?`,
), old, queued.ID); err != nil {
t.Fatalf("backdate requested_at: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-pg-stuck", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("ListStuckRuns: %v", err)
}
if total != 2 || len(rows) != 2 {
t.Fatalf("got %d rows, total %d, want 2/2", len(rows), total)
}
// Claimed-stuck outranks queued-stuck regardless of relative age
// (loop_health.go's ORDER BY (status = 'claimed') DESC ...).
if rows[0].RunID != claimed.ID || rows[0].Condition != "claimed_stuck" {
t.Errorf("rows[0] = %+v, want claimed_stuck for %s", rows[0], claimed.ID)
}
if rows[0].StuckSince == nil || !rows[0].StuckSince.Equal(longAgo) {
t.Errorf("rows[0].StuckSince = %v, want %v", rows[0].StuckSince, longAgo)
}
if rows[1].RunID != queued.ID || rows[1].Condition != "queued_stuck" {
t.Errorf("rows[1] = %+v, want queued_stuck for %s", rows[1], queued.ID)
}
if rows[1].StuckSince == nil || !rows[1].StuckSince.Equal(old) {
t.Errorf("rows[1].StuckSince = %v, want %v", rows[1].StuckSince, old)
}
}
@@ -0,0 +1,284 @@
package sqlite_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
)
func TestListStuckRuns_ClaimedStuckReportedRegardlessOfAge(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
// Claimed a very long time ago — the predicate is not windowed
// (AC-004.5): a run stuck longer than the evaluation window is
// still reported.
longAgo := now.Add(-72 * time.Hour)
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, longAgo, run.ID); err != nil {
t.Fatalf("seed claimed state: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
}
if rows[0].Condition != "claimed_stuck" {
t.Errorf("condition = %q, want claimed_stuck", rows[0].Condition)
}
}
func TestListStuckRuns_QueuedStuckPositiveControl(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
old := now.Add(-10 * time.Minute)
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil {
t.Fatalf("backdate requested_at: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1 (positive control must be reported)", len(rows), total)
}
if rows[0].Condition != "queued_stuck" {
t.Errorf("condition = %q, want queued_stuck", rows[0].Condition)
}
}
// AC-004.17 exclusion 1: a future scheduled_retry_at is a deferral,
// not a stall, even though requested_at is old — the COALESCE handles
// this without a separate branch.
func TestListStuckRuns_ExcludesFutureScheduledRetry(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
old := now.Add(-10 * time.Minute)
future := now.Add(5 * time.Minute)
if _, err := db.Exec(
`UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`,
old, future, run.ID,
); err != nil {
t.Fatalf("seed future retry: %v", err)
}
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (future scheduled_retry_at is a deferral)", total)
}
}
// AC-004.17 exclusion 2: a non-NULL routing_blocked_status is a parked
// run — parked for a human, not stuck.
func TestListStuckRuns_ExcludesRoutingBlockedRun(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
old := now.Add(-10 * time.Minute)
if _, err := db.Exec(
`UPDATE runs SET requested_at = ?, routing_blocked_status = 'waiting_for_provider_capacity' WHERE id = ?`,
old, run.ID,
); err != nil {
t.Fatalf("seed routing blocked: %v", err)
}
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (routing-blocked run is parked, not stuck)", total)
}
}
// AC-004.17 exclusion 3: a claimed sibling on the same agent_profile_id
// means the queue is correctly holding this run back — a busy agent's
// steady state, not a stall.
func TestListStuckRuns_ExcludesRunWithClaimedSibling(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
sibling := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, sibling); err != nil {
t.Fatalf("create sibling: %v", err)
}
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, sibling.ID); err != nil {
t.Fatalf("claim sibling: %v", err)
}
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
old := now.Add(-10 * time.Minute)
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil {
t.Fatalf("backdate requested_at: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
// The claimed sibling itself is not "stuck" either (claimed_at is
// fresh), so the workspace should report nothing at all.
if total != 0 || len(rows) != 0 {
t.Fatalf("got %d rows, total %d, want 0/0 (busy agent is not a stall)", len(rows), total)
}
}
// AC-004.17 aging: a run whose scheduled_retry_at has JUST passed is
// aged from that instant, not from requested_at.
func TestListStuckRuns_AgesFromScheduledRetryAtOncePassed(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
// requested_at is recent (would not itself be stuck), but
// scheduled_retry_at passed well over the grace period ago.
recent := now.Add(-30 * time.Second)
pastRetry := now.Add(-3 * time.Minute)
if _, err := db.Exec(
`UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`,
recent, pastRetry, run.ID,
); err != nil {
t.Fatalf("seed past retry: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1 (aged from scheduled_retry_at)", len(rows), total)
}
}
// AC-004.17 (F29/F30 accepted limitation): a queued run that has
// dispatched at least once (current_route_attempt_seq > 0) reads as
// silent rather than stuck, mirroring ClaimNextEligibleRun's own gap
// rather than fixing it here.
func TestListStuckRuns_ExcludesRunWithNonZeroRouteAttemptSeq(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
old := now.Add(-10 * time.Minute)
if _, err := db.Exec(
`UPDATE runs SET requested_at = ?, current_route_attempt_seq = 1 WHERE id = ?`,
old, run.ID,
); err != nil {
t.Fatalf("seed routed attempt: %v", err)
}
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 {
t.Fatalf("total = %d, want 0 (accepted limitation: a dispatched-once run reads as silent)", total)
}
}
// Ordering: a claimed-stuck run outranks an older queued-stuck one,
// and a NULL claimed_at row sorts first among claimed rows.
func TestListStuckRuns_ClaimedOutranksOlderQueuedAndNullClaimedAtSortsFirst(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
seedAgentProfile(t, db, "agent-a", "ws-a")
seedAgentProfile(t, db, "agent-b", "ws-a")
seedAgentProfile(t, db, "agent-c", "ws-a")
// Oldest queued-stuck run.
queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, queued); err != nil {
t.Fatalf("create queued: %v", err)
}
veryOld := now.Add(-60 * time.Minute)
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, veryOld, queued.ID); err != nil {
t.Fatalf("backdate queued: %v", err)
}
// Claimed-stuck run with a recorded claimed_at, younger than the
// queued one but still past the claimed grace.
claimedWithAge := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, claimedWithAge); err != nil {
t.Fatalf("create claimed: %v", err)
}
claimedAt := now.Add(-15 * time.Minute)
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, claimedAt, claimedWithAge.ID); err != nil {
t.Fatalf("claim: %v", err)
}
// Claimed-stuck run with NULL claimed_at.
claimedNull := &models.Run{AgentProfileID: "agent-c", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
if err := repo.CreateRun(ctx, claimedNull); err != nil {
t.Fatalf("create claimed-null: %v", err)
}
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = NULL WHERE id = ?`, claimedNull.ID); err != nil {
t.Fatalf("claim with null claimed_at: %v", err)
}
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 3 || len(rows) != 3 {
t.Fatalf("got %d rows, total %d, want 3/3", len(rows), total)
}
if rows[0].RunID != claimedNull.ID {
t.Errorf("rows[0] = %s, want the NULL-claimed_at row first", rows[0].RunID)
}
if rows[1].RunID != claimedWithAge.ID {
t.Errorf("rows[1] = %s, want the aged claimed row second", rows[1].RunID)
}
if rows[2].RunID != queued.ID {
t.Errorf("rows[2] = %s, want the queued row last despite being oldest", rows[2].RunID)
}
}
@@ -0,0 +1,228 @@
package sqlite_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
)
// seedCronTrigger creates a routine and one cron trigger for it,
// returning the trigger id. next_run_at/last_fired_at are set with a
// raw UPDATE afterward since CreateRoutineTrigger only accepts a
// caller-supplied NextRunAt at creation and never a NULL/backdated
// LastFiredAt.
func seedCronTrigger(t *testing.T, repo interface {
CreateRoutine(ctx context.Context, r *models.Routine) error
CreateRoutineTrigger(ctx context.Context, tr *models.RoutineTrigger) error
}, workspaceID, name string) *models.Routine {
t.Helper()
routine := &models.Routine{
WorkspaceID: workspaceID,
Name: name,
TaskTemplate: "",
Status: "active",
ConcurrencyPolicy: "always_create",
}
if err := repo.CreateRoutine(context.Background(), routine); err != nil {
t.Fatalf("create routine: %v", err)
}
return routine
}
func TestListOverdueOrStrandedTriggers_ReturnsOverdueArmedTrigger(t *testing.T) {
repo, db := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
routine := seedCronTrigger(t, repo, "ws-a", "Overdue")
overdueAt := now.Add(-4 * time.Minute)
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger: %v", err)
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
}
if rows[0].Condition != "overdue" {
t.Errorf("condition = %q, want overdue", rows[0].Condition)
}
if rows[0].RoutineName != "Overdue" {
t.Errorf("routine_name = %q, want Overdue", rows[0].RoutineName)
}
_ = db
}
func TestListOverdueOrStrandedTriggers_ReturnsStrandedClaimedTrigger(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
routine := seedCronTrigger(t, repo, "ws-a", "Stranded")
future := now.Add(time.Minute)
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &future,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger: %v", err)
}
// Claim it: next_run_at -> NULL, last_fired_at -> 6 minutes ago,
// past the 5-minute stranded grace.
claimedAt := now.Add(-6 * time.Minute)
if _, err := repo.ExecRaw(ctx,
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
claimedAt, trigger.ID); err != nil {
t.Fatalf("seed stranded state: %v", err)
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 1 || len(rows) != 1 {
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
}
if rows[0].Condition != "stranded" {
t.Errorf("condition = %q, want stranded", rows[0].Condition)
}
}
// AC-004.15: a trigger claimed less than strandedGrace ago is a
// claiming trigger — excluded from the evidence list entirely, not
// given a third label.
func TestListOverdueOrStrandedTriggers_ExcludesClaimingTrigger(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
routine := seedCronTrigger(t, repo, "ws-a", "Claiming")
future := now.Add(time.Minute)
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &future,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger: %v", err)
}
claimedAt := now.Add(-1 * time.Minute)
if _, err := repo.ExecRaw(ctx,
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
claimedAt, trigger.ID); err != nil {
t.Fatalf("seed claiming state: %v", err)
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
)
if err != nil {
t.Fatalf("list: %v", err)
}
if total != 0 || len(rows) != 0 {
t.Fatalf("got %d rows, total %d, want 0/0 (claiming trigger must be excluded)", len(rows), total)
}
eligible, err := repo.CountEligibleTriggers(ctx, "ws-a")
if err != nil {
t.Fatalf("count eligible: %v", err)
}
if eligible != 1 {
t.Errorf("eligible count = %d, want 1 (still an eligible trigger, just not overdue/stranded)", eligible)
}
}
// Ordering: stranded (NULL next_run_at) sorts before overdue.
func TestListOverdueOrStrandedTriggers_StrandedSortsBeforeOverdue(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
overdueRoutine := seedCronTrigger(t, repo, "ws-a", "Overdue")
overdueAt := now.Add(-10 * time.Minute)
overdueTrigger := &models.RoutineTrigger{
RoutineID: overdueRoutine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
}
if err := repo.CreateRoutineTrigger(ctx, overdueTrigger); err != nil {
t.Fatalf("create overdue trigger: %v", err)
}
strandedRoutine := seedCronTrigger(t, repo, "ws-a", "Stranded")
future := now.Add(time.Minute)
strandedTrigger := &models.RoutineTrigger{
RoutineID: strandedRoutine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &future,
}
if err := repo.CreateRoutineTrigger(ctx, strandedTrigger); err != nil {
t.Fatalf("create stranded trigger: %v", err)
}
if _, err := repo.ExecRaw(ctx,
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
now.Add(-10*time.Minute), strandedTrigger.ID); err != nil {
t.Fatalf("seed stranded state: %v", err)
}
rows, _, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 2 {
t.Fatalf("got %d rows, want 2", len(rows))
}
if rows[0].Condition != "stranded" || rows[1].Condition != "overdue" {
t.Errorf("order = [%s, %s], want [stranded, overdue]", rows[0].Condition, rows[1].Condition)
}
}
func TestListOverdueOrStrandedTriggers_CapsAndReportsTotal(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
ctx := context.Background()
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
for i := 0; i < 3; i++ {
routine := seedCronTrigger(t, repo, "ws-a", "R")
overdueAt := now.Add(-10 * time.Minute)
trigger := &models.RoutineTrigger{
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
}
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
t.Fatalf("create trigger %d: %v", i, err)
}
}
rows, total, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 2)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 2 {
t.Fatalf("got %d rows, want capped at 2", len(rows))
}
if total != 3 {
t.Fatalf("total = %d, want 3 (untruncated)", total)
}
}
func TestCountEligibleTriggers_ZeroForEmptyWorkspace(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
count, err := repo.CountEligibleTriggers(context.Background(), "ws-empty")
if err != nil {
t.Fatalf("count: %v", err)
}
if count != 0 {
t.Errorf("count = %d, want 0", count)
}
}
@@ -0,0 +1,89 @@
package sqlite
import (
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/persistence"
)
// loopLivenessActivationKey is the kandev_meta key published once a
// boot has verified all three causation_id columns exist
// (REQ-OFFICE-LOOP-LIVENESS-002/005). See docs/specs/office/system-design/
// loop-liveness.md, "Activation".
const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at"
// causationIDColumn is the column all three activation probes check.
const causationIDColumn = "causation_id"
// activateLoopLiveness writes loopLivenessActivationKey only after a
// positive probe confirms all three causation_id columns exist. Mirrors
// activateRunOutcome: the migration runner swallows failures at WARN,
// so this probe is what stands between a failed migration and a reader
// wrongly believing correlation and terminal-shape classification are
// live. WriteMetaKeyIfAbsent makes the write replay-safe — the instant
// is never overwritten once set, because every run requested before it
// must classify as pre_activation forever, not just until the next
// boot. Any failure here is logged and swallowed — activation is
// best-effort published fact, never a boot-blocking requirement.
func (r *Repository) activateLoopLiveness() {
if err := persistence.EnsureMetaTable(r.db); err != nil {
r.logActivationWarn("ensure kandev_meta table failed", err)
return
}
for _, probe := range []struct{ table, column string }{
{"office_routine_runs", causationIDColumn},
{"agent_wakeup_requests", causationIDColumn},
{"runs", causationIDColumn},
} {
exists, err := columnExists(r.db, probe.table, probe.column)
if err != nil {
r.logActivationWarn("loop liveness schema probe failed", err)
return
}
if !exists {
return
}
}
if _, err := persistence.WriteMetaKeyIfAbsent(
r.db, loopLivenessActivationKey, time.Now().UTC().Format(time.RFC3339),
); err != nil {
r.logActivationWarn("write loop liveness activation key failed", err)
}
}
// LoopLivenessActivation returns the published activation instant and
// whether it has been published at all. A run requested before the
// instant, or read when activation was never published, must classify
// as pre_activation (AC-005.6) rather than risk a false silent_success.
// Read failures are swallowed here (logged only) because every existing
// caller uses this for a safe-default classification, not a decision
// that must fail loud; the /loop-health read that must distinguish
// "unpublished" from "unreadable" uses LoopLivenessActivationChecked.
func (r *Repository) LoopLivenessActivation() (at time.Time, published bool) {
at, published, err := r.LoopLivenessActivationChecked()
if err != nil && r.log != nil {
r.log.Warn("read loop liveness activation key failed", zap.Error(err))
}
return at, published
}
// LoopLivenessActivationChecked is LoopLivenessActivation with the read
// or parse error surfaced instead of swallowed, for a caller that must
// fail loud (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.10) rather than let a
// broken kandev_meta read silently render as "not yet activated".
func (r *Repository) LoopLivenessActivationChecked() (at time.Time, published bool, err error) {
raw, err := persistence.ReadMetaKey(r.db, loopLivenessActivationKey)
if err != nil {
return time.Time{}, false, err
}
if raw == "" {
return time.Time{}, false, nil
}
parsed, err := time.Parse(time.RFC3339, raw)
if err != nil {
return time.Time{}, false, err
}
return parsed, true, nil
}
@@ -0,0 +1,185 @@
package sqlite_test
import (
"database/sql"
"strings"
"testing"
"time"
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/persistence"
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/testutil"
)
const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at"
// TestLoopLivenessMigration_AddsCausationIDColumns covers
// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: all three
// causation_id columns exist and default to ” (not NULL) on a
// freshly-migrated database.
func TestLoopLivenessMigration_AddsCausationIDColumns(t *testing.T) {
_, db := newTestRepoWithDB(t)
for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} {
var notnull int
var dflt sql.NullString
rows, err := db.Queryx("PRAGMA table_info(" + table + ")")
if err != nil {
t.Fatalf("table_info(%s): %v", table, err)
}
found := false
for rows.Next() {
var cid, nn, pk int
var name, ctype string
var d sql.NullString
if err := rows.Scan(&cid, &name, &ctype, &nn, &d, &pk); err != nil {
t.Fatalf("scan table_info(%s): %v", table, err)
}
if name == "causation_id" {
found = true
notnull = nn
dflt = d
}
}
_ = rows.Close()
if !found {
t.Fatalf("%s.causation_id column missing", table)
}
if notnull != 1 {
t.Fatalf("%s.causation_id NOT NULL = %d, want 1", table, notnull)
}
if !dflt.Valid || dflt.String != "''" {
t.Fatalf("%s.causation_id default = %v, want ''", table, dflt)
}
}
}
// TestLoopLivenessMigration_CausationIDIndexesArePartial covers
// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: each
// causation_id index exists and is partial (WHERE causation_id !=
// ”), not a plain index over every row — a full index would defeat
// the point of excluding the common empty-string case from the
// lookup path. sqlite_master.sql is the only way to see the stored
// WHERE predicate; PRAGMA index_list/index_info do not expose it.
func TestLoopLivenessMigration_CausationIDIndexesArePartial(t *testing.T) {
_, db := newTestRepoWithDB(t)
cases := []struct {
table string
indexName string
}{
{"office_routine_runs", "idx_office_routine_runs_causation_id"},
{"agent_wakeup_requests", "idx_agent_wakeup_requests_causation_id"},
{"runs", "idx_runs_causation_id"},
}
for _, tc := range cases {
var stored sql.NullString
err := db.QueryRow(
`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?`, tc.indexName,
).Scan(&stored)
if err != nil {
t.Fatalf("%s: look up index %s: %v", tc.table, tc.indexName, err)
}
if !stored.Valid || stored.String == "" {
t.Fatalf("%s: index %s missing or has no stored definition", tc.table, tc.indexName)
}
if !strings.Contains(stored.String, "WHERE") || !strings.Contains(stored.String, "causation_id != ''") {
t.Fatalf("%s: index %s definition = %q, want a WHERE causation_id != '' partial predicate",
tc.table, tc.indexName, stored.String)
}
}
}
// TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe mirrors
// TestRunOutcomeActivation_WrittenOnceAfterSchemaProbe: the key is
// written only after a positive probe of all three columns, and a
// replayed boot never re-writes it.
func TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe(t *testing.T) {
_, db := newTestRepoWithDB(t)
val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
if err != nil {
t.Fatalf("read activation key: %v", err)
}
if val == "" {
t.Fatal("expected telemetry.office_loop_liveness.activated_at to be written after first boot")
}
if _, err := time.Parse(time.RFC3339, val); err != nil {
t.Fatalf("activation value %q is not RFC3339: %v", val, err)
}
if _, err := db.Exec(db.Rebind(
`UPDATE kandev_meta SET value = ? WHERE key = ?`,
), "sentinel-value", loopLivenessActivationKey); err != nil {
t.Fatalf("seed sentinel: %v", err)
}
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
t.Fatalf("replay init: %v", err)
}
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
t.Fatalf("replay init twice: %v", err)
}
got, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
if err != nil {
t.Fatalf("read activation key after replay: %v", err)
}
if got != "sentinel-value" {
t.Fatalf("activation key = %q after replay, want unchanged sentinel-value", got)
}
}
// TestLoopLivenessActivation_ReadHelperRoundTrips covers
// Repository.LoopLivenessActivation: published=true with the parsed
// instant after boot, and published=false when the key is absent.
func TestLoopLivenessActivation_ReadHelperRoundTrips(t *testing.T) {
repo, _ := newTestRepoWithDB(t)
at, published := repo.LoopLivenessActivation()
if !published {
t.Fatal("expected activation to be published after boot")
}
if at.IsZero() {
t.Fatal("expected a non-zero activation instant")
}
}
// TestPostgresLoopLivenessMigration_AddsColumnsAndActivates is the
// PostgreSQL twin required by ADR 0027. Skips unless
// KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresLoopLivenessMigration_AddsColumnsAndActivates(t *testing.T) {
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init task repo: %v", err)
}
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init postgres schema: %v", err)
}
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
t.Fatalf("replay postgres schema: %v", err)
}
for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} {
var dataType string
err := db.QueryRow(`
SELECT data_type FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = $1 AND column_name = 'causation_id'
`, table).Scan(&dataType)
if err != nil {
t.Fatalf("inspect %s.causation_id: %v", table, err)
}
if dataType != "text" {
t.Fatalf("%s.causation_id data type = %q, want text", table, dataType)
}
}
val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
if err != nil {
t.Fatalf("read activation key: %v", err)
}
if val == "" {
t.Fatal("expected activation key to be written on postgres")
}
}
@@ -0,0 +1,129 @@
package sqlite_test
import (
"context"
"sync"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/kandev/kandev/internal/office/models"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/testutil"
)
// TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun is
// the two-connection regression for Review round 3, R3-1: MarkRunFailed's
// write used to carry no status guard at all, so a run cancelled on
// another connection between HandleAgentFailure's read and this write
// would have its 'cancelled' status and real finished_at silently
// overwritten to 'failed'. Real Postgres row locking forces the
// interleaving this needs: one connection holds an uncommitted cancel on
// the row, MarkRunFailed's write blocks on that row lock, and only
// proceeds once the cancel commits — at which point the guard
// (status = 'claimed') must see the fresh 'cancelled' status and write
// nothing. Skips unless KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) {
dsn := testutil.PostgresDSNFromEnv(t)
db := testutil.OpenIsolatedPostgres(t, dsn)
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init task repo: %v", err)
}
repo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init office repo: %v", err)
}
ctx := context.Background()
var schema string
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
t.Fatalf("read isolated schema name: %v", err)
}
cancellerDB, err := sqlx.Open("pgx", dsn)
if err != nil {
t.Fatalf("open canceller connection: %v", err)
}
cancellerDB.SetMaxOpenConns(1)
cancellerDB.SetMaxIdleConns(1)
t.Cleanup(func() { _ = cancellerDB.Close() })
if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil {
t.Fatalf("set canceller search_path: %v", err)
}
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
now := time.Now().UTC()
if _, err := db.ExecContext(ctx, db.Rebind(
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
), now, run.ID); err != nil {
t.Fatalf("seed claimed: %v", err)
}
tx, err := cancellerDB.BeginTxx(ctx, nil)
if err != nil {
t.Fatalf("begin canceller tx: %v", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, cancellerDB.Rebind(
`UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`,
), "too late", now, run.ID); err != nil {
t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err)
}
var (
wg sync.WaitGroup
wrote bool
markErr error
markDone = make(chan struct{})
commitErr error
)
wg.Add(1)
go func() {
defer wg.Done()
defer close(markDone)
wrote, markErr = repo.MarkRunFailed(ctx, run.ID, "provider exploded")
}()
// Give MarkRunFailed's write time to reach Postgres and block on the
// row lock the canceller's uncommitted transaction holds, before that
// transaction commits. Without this window the two goroutines could
// run in either order and the test would not exercise the lock-wait
// path this bug depends on.
select {
case <-markDone:
t.Fatal("MarkRunFailed returned before the canceller committed; it never blocked on the row lock")
case <-time.After(200 * time.Millisecond):
}
commitErr = tx.Commit()
wg.Wait()
if commitErr != nil {
t.Fatalf("canceller commit: %v", commitErr)
}
if markErr != nil {
t.Fatalf("mark run failed: %v", markErr)
}
if wrote {
t.Fatal("wrote = true, want false: a run that committed cancelled must not be reported failed")
}
if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" {
t.Fatalf("run status = %q, want cancelled (the concurrent fail must not have overwritten it)", got)
}
}
func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string {
t.Helper()
var status string
if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil {
t.Fatalf("get run status: %v", err)
}
return status
}
@@ -437,8 +437,11 @@ func TestCancelDisplacedParticipantRun_CancelsOnlyTheFanOutRunForThatAgentStep(t
if err != nil {
t.Fatalf("CancelDisplacedParticipantRun: %v", err)
}
if cancelled != 1 {
t.Fatalf("cancelled = %d, want 1", cancelled)
if len(cancelled) != 1 {
t.Fatalf("cancelled = %d, want 1", len(cancelled))
}
if cancelled[0].ID != "run-target" {
t.Fatalf("cancelled[0].ID = %q, want run-target", cancelled[0].ID)
}
assertRunStatus(t, repo, "run-target", "cancelled")
@@ -460,8 +463,8 @@ func TestCancelDisplacedParticipantRun_NoMatchIsNotAFailure(t *testing.T) {
if err != nil {
t.Fatalf("CancelDisplacedParticipantRun: %v", err)
}
if cancelled != 0 {
t.Errorf("cancelled = %d, want 0", cancelled)
if len(cancelled) != 0 {
t.Errorf("cancelled = %d, want 0", len(cancelled))
}
}
@@ -11,6 +11,7 @@ import (
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/db/dialect"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
"github.com/kandev/kandev/internal/workflow/models"
workflowrepo "github.com/kandev/kandev/internal/workflow/repository"
)
@@ -672,9 +673,11 @@ const displacedRunCancelReason = "participant_seat_claimed"
// SQLite-flavoured and the server dialect needs dialect.JSONExtract, which
// is why this selector lives here rather than in the shared runs writer
// (mirrors CancelRunsForTasks in tree_holds.go).
func (r *Repository) CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error) {
func (r *Repository) CancelDisplacedParticipantRun(
ctx context.Context, taskID, stepID, agentProfileID string,
) ([]runssqlite.CancelledRun, error) {
if taskID == "" || stepID == "" || agentProfileID == "" {
return 0, nil
return nil, nil
}
driver := r.db.DriverName()
selector := fmt.Sprintf(
@@ -0,0 +1,195 @@
package sqlite_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
)
func createTestRoutineForLastRun(t *testing.T, repo interface {
CreateRoutine(ctx context.Context, routine *models.Routine) error
}) *models.Routine {
t.Helper()
routine := &models.Routine{
WorkspaceID: "ws-1",
Name: "Last Run Test",
TaskTemplate: "{}",
Status: "active",
ConcurrencyPolicy: "skip_if_active",
Variables: "{}",
}
if err := repo.CreateRoutine(context.Background(), routine); err != nil {
t.Fatalf("create routine: %v", err)
}
return routine
}
// AC-OFFICE-LOOP-LIVENESS-001.1/.3/.4/.6: TouchRoutineLastRun advances
// last_run_at from NULL, is monotonic against an equal or older instant
// (zero-row no-op, not an error), and moves forward for a newer instant.
func TestTouchRoutineLastRun_Monotonic(t *testing.T) {
repo := newTestRepo(t)
ctx := context.Background()
routine := createTestRoutineForLastRun(t, repo)
first := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil {
t.Fatalf("touch (initial): %v", err)
}
got, err := repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.LastRunAt == nil || !got.LastRunAt.Equal(first) {
t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, first)
}
// Older instant: no-op.
older := first.Add(-time.Hour)
if err := repo.TouchRoutineLastRun(ctx, routine.ID, older); err != nil {
t.Fatalf("touch (older): %v", err)
}
got, err = repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !got.LastRunAt.Equal(first) {
t.Fatalf("last_run_at moved backwards: got %v, want %v", got.LastRunAt, first)
}
// Equal instant: no-op.
if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil {
t.Fatalf("touch (equal): %v", err)
}
got, err = repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !got.LastRunAt.Equal(first) {
t.Fatalf("last_run_at changed on equal instant: got %v, want %v", got.LastRunAt, first)
}
// Newer instant: advances.
newer := first.Add(time.Hour)
if err := repo.TouchRoutineLastRun(ctx, routine.ID, newer); err != nil {
t.Fatalf("touch (newer): %v", err)
}
got, err = repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !got.LastRunAt.Equal(newer) {
t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, newer)
}
}
// AC-OFFICE-LOOP-LIVENESS-001.6: a touch matching zero rows (unknown
// routine) is treated as success, not an error.
func TestTouchRoutineLastRun_UnknownRoutineIsNoop(t *testing.T) {
repo := newTestRepo(t)
ctx := context.Background()
if err := repo.TouchRoutineLastRun(ctx, "does-not-exist", time.Now().UTC()); err != nil {
t.Fatalf("touch unknown routine: %v", err)
}
}
// AC-OFFICE-LOOP-LIVENESS-001.5: the write touches only last_run_at (and
// updated_at) — every other column is left exactly as it was.
func TestTouchRoutineLastRun_TouchesOnlyLastRunAtAndUpdatedAt(t *testing.T) {
repo := newTestRepo(t)
ctx := context.Background()
routine := createTestRoutineForLastRun(t, repo)
beforeUpdatedAt := routine.UpdatedAt
at := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
if err := repo.TouchRoutineLastRun(ctx, routine.ID, at); err != nil {
t.Fatalf("touch: %v", err)
}
got, err := repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Name != routine.Name || got.Status != routine.Status ||
got.ConcurrencyPolicy != routine.ConcurrencyPolicy || got.TaskTemplate != routine.TaskTemplate {
t.Fatalf("touch mutated a non-last_run_at column: %+v", got)
}
if !got.UpdatedAt.After(beforeUpdatedAt) {
t.Fatalf("updated_at did not advance: got %v, before %v", got.UpdatedAt, beforeUpdatedAt)
}
}
// AC-OFFICE-LOOP-LIVENESS-001.8: UpdateRoutine no longer carries
// last_run_at — a caller holding a stale (or nil) snapshot cannot move
// the fresh instant this dispatch just wrote backwards or clear it.
func TestUpdateRoutine_DoesNotWriteLastRunAt(t *testing.T) {
repo := newTestRepo(t)
ctx := context.Background()
routine := createTestRoutineForLastRun(t, repo)
fresh := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
if err := repo.TouchRoutineLastRun(ctx, routine.ID, fresh); err != nil {
t.Fatalf("touch: %v", err)
}
// A caller holding a stale snapshot (older LastRunAt) must not move
// the stored instant backwards via an unrelated field update.
stale := fresh.Add(-24 * time.Hour)
routine.LastRunAt = &stale
routine.Name = "Renamed While Stale"
if err := repo.UpdateRoutine(ctx, routine); err != nil {
t.Fatalf("update (stale snapshot): %v", err)
}
got, err := repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
t.Fatalf("stale UpdateRoutine moved last_run_at: got %v, want %v", got.LastRunAt, fresh)
}
if got.Name != "Renamed While Stale" {
t.Fatalf("update did not apply other fields: name = %q", got.Name)
}
// A caller that never populated LastRunAt (nil) must not clear it either.
routine.LastRunAt = nil
routine.Description = "Updated from nil snapshot"
if err := repo.UpdateRoutine(ctx, routine); err != nil {
t.Fatalf("update (nil snapshot): %v", err)
}
got, err = repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
t.Fatalf("nil-snapshot UpdateRoutine cleared last_run_at: got %v, want %v", got.LastRunAt, fresh)
}
if got.Description != "Updated from nil snapshot" {
t.Fatalf("update did not apply other fields: description = %q", got.Description)
}
// A caller holding a snapshot newer than the stored instant must not
// advance it either: a stale-or-nil case alone cannot distinguish "the
// column is not written" from "the column is written monotonically",
// since both pass under a stale or nil snapshot. Only a newer snapshot
// tells them apart.
newer := fresh.Add(24 * time.Hour)
routine.LastRunAt = &newer
routine.Name = "Renamed From Newer Snapshot"
if err := repo.UpdateRoutine(ctx, routine); err != nil {
t.Fatalf("update (newer snapshot): %v", err)
}
got, err = repo.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
t.Fatalf("newer-snapshot UpdateRoutine advanced last_run_at: got %v, want %v", got.LastRunAt, fresh)
}
if got.Name != "Renamed From Newer Snapshot" {
t.Fatalf("update did not apply other fields: name = %q", got.Name)
}
}
@@ -196,7 +196,13 @@ func (r *Repository) ListRoutines(ctx context.Context, workspaceID string) ([]*m
return routines, nil
}
// UpdateRoutine updates an existing routine.
// UpdateRoutine updates an existing routine. It deliberately does not
// carry last_run_at (AC-OFFICE-LOOP-LIVENESS-001.8): that column only
// ever moves forward, written by TouchRoutineLastRun from a dispatch, and
// a caller here may be holding a routine loaded before a fire (or one
// that never populated the field) — writing it unconditionally would let
// that stale or nil snapshot silently move the value backwards or clear
// it. routine.LastRunAt stays on the struct for reads.
func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine) error {
routine.UpdatedAt = time.Now().UTC()
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
@@ -204,12 +210,27 @@ func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine)
name = ?, description = ?, task_template = ?,
assignee_agent_profile_id = ?, status = ?, concurrency_policy = ?,
catch_up_policy = ?, catch_up_max = ?,
variables = ?, last_run_at = ?, updated_at = ?
variables = ?, updated_at = ?
WHERE id = ?
`), routine.Name, routine.Description, routine.TaskTemplate,
routine.AssigneeAgentProfileID, routine.Status, routine.ConcurrencyPolicy,
routine.CatchUpPolicy, routine.CatchUpMax,
routine.Variables, routine.LastRunAt, routine.UpdatedAt, routine.ID)
routine.Variables, routine.UpdatedAt, routine.ID)
return err
}
// TouchRoutineLastRun advances office_routines.last_run_at to at,
// monotonically: the write is a zero-row no-op, not an error, when the
// stored value is already at or after at (AC-OFFICE-LOOP-LIVENESS-001.3,
// .4, .6). Deliberately narrower than UpdateRoutine's eleven-column
// read-modify-write, which would turn a routine dispatch into a race
// against a concurrent UI or config-sync edit (AC-OFFICE-LOOP-LIVENESS-001.5).
func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE office_routines
SET last_run_at = ?, updated_at = ?
WHERE id = ? AND (last_run_at IS NULL OR last_run_at < ?)
`), at, time.Now().UTC(), routineID, at)
return err
}
@@ -279,11 +300,11 @@ func (r *Repository) CreateRoutineRun(ctx context.Context, run *models.RoutineRu
INSERT INTO office_routine_runs (
id, routine_id, trigger_id, source, status, trigger_payload,
linked_task_id, coalesced_into_run_id, dispatch_fingerprint,
started_at, completed_at, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
started_at, completed_at, created_at, causation_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`), run.ID, run.RoutineID, run.TriggerID, run.Source, run.Status,
run.TriggerPayload, run.LinkedTaskID, run.CoalescedIntoRunID,
run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt)
run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt, run.CausationID)
return err
}
@@ -404,18 +425,6 @@ func (r *Repository) GetTaskTerminalStatus(ctx context.Context, taskID string) (
}
}
// TouchRoutineLastRun updates only last_run_at (+ updated_at) on a
// routine. Deliberately narrower than UpdateRoutine, whose whole-row
// snapshot write can revert concurrent edits to other columns (see
// UpdateRoutineConfigFields above) — a dispatch in flight should never
// clobber a config change made while it ran, or vice versa.
func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE office_routines SET last_run_at = ?, updated_at = ? WHERE id = ?
`), at, time.Now().UTC(), routineID)
return err
}
// UpdateRunStatus updates a run's status and optionally its linked task.
func (r *Repository) UpdateRunStatus(
ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string,
@@ -59,7 +59,7 @@ func TestCancelRun_QueuedRunIsCancelled(t *testing.T) {
repo := newTestRepo(t)
id := seedCancelRun(t, repo, "t1", "queued", nil, nil)
if err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil {
if _, err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil {
t.Fatalf("cancel: %v", err)
}
@@ -79,7 +79,7 @@ func TestCancelRun_ClaimedRunIsCancelled(t *testing.T) {
repo := newTestRepo(t)
id := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
if err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil {
if _, err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil {
t.Fatalf("cancel: %v", err)
}
@@ -115,7 +115,7 @@ func TestCancelRun_TerminalRunIsUntouched(t *testing.T) {
repo := newTestRepo(t)
id := seedCancelRun(t, repo, "t1", tc.status, tc.cancelReason, &finished)
if err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil {
if _, err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil {
t.Fatalf("cancel: %v", err)
}
@@ -155,16 +155,16 @@ func TestCancelRunsWhere_RowsAffected(t *testing.T) {
if err != nil {
t.Fatalf("cancel terminal: %v", err)
}
if rows != 0 {
t.Errorf("rows affected = %d, want 0 for an already-terminal run", rows)
if len(rows) != 0 {
t.Errorf("rows affected = %d, want 0 for an already-terminal run", len(rows))
}
rows, err = repo.CancelRunsWhere(ctx, "tree_cancelled", `id = ?`, queued)
if err != nil {
t.Fatalf("cancel queued: %v", err)
}
if rows != 1 {
t.Errorf("rows affected = %d, want 1", rows)
if len(rows) != 1 {
t.Errorf("rows affected = %d, want 1", len(rows))
}
}
@@ -177,7 +177,7 @@ func TestBulkCancelRuns_OnlyCancelsEligibleRuns(t *testing.T) {
claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
done := seedCancelRun(t, repo, "t1", "finished", nil, &finished)
if err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil {
if _, err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil {
t.Fatalf("bulk cancel: %v", err)
}
@@ -204,12 +204,12 @@ func TestCancelRunsForTasks_MixedSetCountMatches(t *testing.T) {
claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
done := seedCancelRun(t, repo, "t1", "finished", nil, &finished)
count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused")
cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused")
if err != nil {
t.Fatalf("cancel for tasks: %v", err)
}
if count != 2 {
t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", count)
if len(cancelled) != 2 {
t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", len(cancelled))
}
for _, id := range []string{queued, claimed} {
if run := fetchRun(t, repo, id); run.Status != "cancelled" {
@@ -229,12 +229,12 @@ func TestCancelRunsForTasks_MatchesPayloadTaskID(t *testing.T) {
mine := seedCancelRun(t, repo, "t1", "queued", nil, nil)
other := seedCancelRun(t, repo, "t2", "queued", nil, nil)
count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled")
cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled")
if err != nil {
t.Fatalf("cancel for tasks: %v", err)
}
if count != 1 {
t.Fatalf("count = %d, want 1", count)
if len(cancelled) != 1 {
t.Fatalf("count = %d, want 1", len(cancelled))
}
if run := fetchRun(t, repo, mine); run.Status != "cancelled" {
t.Errorf("t1 run status = %q, want cancelled", run.Status)
@@ -253,15 +253,15 @@ func TestCancelPaths_EmptySliceIsANoOp(t *testing.T) {
ctx := context.Background()
id := seedCancelRun(t, repo, "t1", "queued", nil, nil)
if err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil {
if _, err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil {
t.Errorf("bulk cancel with no ids: %v", err)
}
count, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled")
cancelled, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled")
if err != nil {
t.Errorf("cancel for tasks with no task ids: %v", err)
}
if count != 0 {
t.Errorf("count = %d, want 0", count)
if len(cancelled) != 0 {
t.Errorf("count = %d, want 0", len(cancelled))
}
if run := fetchRun(t, repo, id); run.Status != "queued" {
t.Errorf("unrelated run status = %q, want queued", run.Status)
@@ -235,14 +235,15 @@ func TestCleanExpired(t *testing.T) {
AgentProfileID: "a1",
Reason: "task_assigned",
Payload: "{}",
Status: "queued",
Status: "claimed",
CoalescedCount: 1,
}
if err := repo.CreateRun(ctx, req); err != nil {
t.Fatalf("create: %v", err)
}
// Finish it.
if err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil {
// Finish it. FinishRun is guarded to status = 'claimed' (Review round
// 3, R3-1), matching the seeded status above.
if _, err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -10,6 +10,7 @@ import (
"github.com/google/uuid"
"github.com/kandev/kandev/internal/office/models"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
)
func (r *Repository) createTreeHoldTables() error {
@@ -208,23 +209,22 @@ func (r *Repository) CountActiveRunsForTasks(ctx context.Context, taskIDs []stri
}
// CancelRunsForTasks cancels the queued or claimed runs belonging to the
// given tasks and returns how many rows were actually cancelled.
// given tasks and returns the rows actually cancelled, so a caller can
// classify and count each one's loop-liveness terminal shape.
//
// The terminal write itself lives in the runs repository's CancelRunsWhere,
// which applies the queued/claimed guard; only the by-task-id selector stays
// here. json_extract is SQLite-flavoured — Postgres is a supported driver
// (internal/persistence/provider.go) and would need dialect.JSONExtract — so
// keeping it local avoids baking dialect-specific SQL into the shared writer.
func (r *Repository) CancelRunsForTasks(ctx context.Context, taskIDs []string, reason string) (int, error) {
func (r *Repository) CancelRunsForTasks(
ctx context.Context, taskIDs []string, reason string,
) ([]runssqlite.CancelledRun, error) {
if len(taskIDs) == 0 {
return 0, nil
return nil, nil
}
selector, args := taskIDInQuery(`json_extract(payload, '$.task_id') IN (%s)`, taskIDs)
rows, err := r.CancelRunsWhere(ctx, reason, selector, args...)
if err != nil {
return 0, err
}
return int(rows), nil
return r.CancelRunsWhere(ctx, reason, selector, args...)
}
func (r *Repository) BulkUpdateTaskState(ctx context.Context, taskIDs []string, state string) error {
@@ -49,6 +49,11 @@ type WakeupRequest struct {
RequestedAt time.Time `db:"requested_at"`
ClaimedAt sql.NullTime `db:"claimed_at"`
FinishedAt sql.NullTime `db:"finished_at"`
// CausationID is copied from the routine fire that produced this
// wake, or minted here when the wake has no routine origin
// (REQ-OFFICE-LOOP-LIVENESS-002). "" for rows written before this
// feature or for a wake with no identifiable origin.
CausationID string `db:"causation_id"`
}
// CreateWakeupRequest inserts a new wakeup-request row. When
@@ -85,12 +90,12 @@ func (r *Repository) CreateWakeupRequest(ctx context.Context, req *WakeupRequest
INSERT INTO agent_wakeup_requests (
id, agent_profile_id, source, reason, payload, status,
coalesced_count, idempotency_key, run_id,
requested_at, claimed_at, finished_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
requested_at, claimed_at, finished_at, causation_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`),
req.ID, req.AgentProfileID, req.Source, req.Reason, req.Payload, req.Status,
req.CoalescedCount, req.IdempotencyKey, req.RunID,
req.RequestedAt, req.ClaimedAt, req.FinishedAt,
req.RequestedAt, req.ClaimedAt, req.FinishedAt, req.CausationID,
)
if err != nil && isUniqueConstraintErr(err) {
return fmt.Errorf("%w: %v", ErrWakeupIdempotencyConflict, err)
@@ -105,7 +110,7 @@ func (r *Repository) GetWakeupRequest(ctx context.Context, id string) (*WakeupRe
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT id, agent_profile_id, source, reason, payload, status,
coalesced_count, idempotency_key, run_id,
requested_at, claimed_at, finished_at
requested_at, claimed_at, finished_at, causation_id
FROM agent_wakeup_requests
WHERE id = ?
`), id).StructScan(&row)
@@ -125,7 +130,7 @@ func (r *Repository) ListQueuedWakeupRequestsForAgent(
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
SELECT id, agent_profile_id, source, reason, payload, status,
coalesced_count, idempotency_key, run_id,
requested_at, claimed_at, finished_at
requested_at, claimed_at, finished_at, causation_id
FROM agent_wakeup_requests
WHERE agent_profile_id = ? AND status = ?
ORDER BY requested_at ASC
@@ -0,0 +1,134 @@
package routines_test
import (
"context"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routines"
)
// newTestRoutineServiceWithTaskTables mirrors newTestRoutineService but
// also creates minimal tasks/task_sessions tables on the same database,
// so a heavy-path test can walk the real
// office_routine_runs.linked_task_id -> tasks.id -> task_sessions.task_id
// chain instead of only asserting the id fields in isolation.
func newTestRoutineServiceWithTaskTables(t *testing.T) (*routines.RoutineService, *sqlx.DB) {
t.Helper()
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
repo, err := sqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS tasks (
id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL DEFAULT ''
)`); err != nil {
t.Fatalf("create tasks table: %v", err)
}
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS task_sessions (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL
)`); err != nil {
t.Fatalf("create task_sessions table: %v", err)
}
log := logger.Default()
return routines.NewRoutineService(repo, log, &noopActivity{}), db
}
// dbBackedTaskCreator is CreateOfficeTaskInWorkflow's fake for this
// file only: unlike fakeTaskCreator (a synthetic id with nothing
// behind it), it inserts a real tasks row plus a task_sessions row
// naming it, so the produced id is actually reachable by a join —
// standing in for auto_start_agent launching the routine's task.
type dbBackedTaskCreator struct {
db *sqlx.DB
taskID string
sessID string
nextSeq int
captured struct{ workflowID string }
}
func (f *dbBackedTaskCreator) CreateOfficeTaskInWorkflow(
_ context.Context, workspaceID, _, _, workflowID, title, _ string,
) (string, error) {
f.nextSeq++
f.captured.workflowID = workflowID
f.taskID = "task-heavy-causation"
f.sessID = "session-heavy-causation"
if _, err := f.db.Exec(`INSERT INTO tasks (id, workspace_id, title) VALUES (?, ?, ?)`,
f.taskID, workspaceID, title); err != nil {
return "", err
}
if _, err := f.db.Exec(`INSERT INTO task_sessions (id, task_id) VALUES (?, ?)`,
f.sessID, f.taskID); err != nil {
return "", err
}
return f.taskID, nil
}
// AC-OFFICE-LOOP-LIVENESS-002.9: a heavy routine fire's run row carries
// both the minted causation id and the linked task id, and that task id
// is a real, reachable row — walkable forward to the session the task
// launched, the correlation chain a loop-liveness diagnosis depends on.
func TestDispatchRoutineRun_HeavyPathCausationIDAndLinkedTaskReachSession(t *testing.T) {
svc, db := newTestRoutineServiceWithTaskTables(t)
ctx := context.Background()
routine := &models.Routine{
WorkspaceID: "ws-1",
Name: "Heavy Causation",
TaskTemplate: `{"title":"Heavy {{name}}","description":"D"}`,
AssigneeAgentProfileID: "agent-heavy",
Status: "active",
ConcurrencyPolicy: "always_create",
Variables: `{"name":{"default":"routine"}}`,
}
if err := svc.CreateRoutine(ctx, routine); err != nil {
t.Fatalf("create routine: %v", err)
}
wf := &fakeWorkflowEnsurer{}
tc := &dbBackedTaskCreator{db: db}
svc.SetWorkflowEnsurer(wf)
svc.SetTaskCreator(tc)
run, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("fire manual: %v", err)
}
if run.CausationID == "" {
t.Fatal("expected non-empty causation id on the heavy-path run")
}
if run.LinkedTaskID != tc.taskID {
t.Fatalf("LinkedTaskID = %q, want %q", run.LinkedTaskID, tc.taskID)
}
var sessionID string
err = db.QueryRow(`
SELECT task_sessions.id
FROM office_routine_runs
JOIN tasks ON tasks.id = office_routine_runs.linked_task_id
JOIN task_sessions ON task_sessions.task_id = tasks.id
WHERE office_routine_runs.id = ?
`, run.ID).Scan(&sessionID)
if err != nil {
t.Fatalf("two-hop join from routine run to session: %v", err)
}
if sessionID != tc.sessID {
t.Fatalf("joined session id = %q, want %q", sessionID, tc.sessID)
}
}
@@ -0,0 +1,72 @@
package routines_test
import (
"context"
"testing"
"github.com/kandev/kandev/internal/office/models"
)
// AC-OFFICE-LOOP-LIVENESS-002.1: dispatchRoutineRun mints one causation
// id per fire — non-empty and distinct across separate fires.
func TestDispatchRoutineRun_MintsOneCausationIDPerFire(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := createTestRoutine(t, svc, "Causation Mint", "always_create")
first, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("first fire: %v", err)
}
if first.CausationID == "" {
t.Fatal("expected non-empty causation id on first fire")
}
second, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("second fire: %v", err)
}
if second.CausationID == "" {
t.Fatal("expected non-empty causation id on second fire")
}
if second.CausationID == first.CausationID {
t.Fatalf("expected distinct causation ids across fires, both were %q", first.CausationID)
}
}
// AC-OFFICE-LOOP-LIVENESS-002.2: the lightweight (taskless) path copies
// the fire's minted causation id onto the WakeupRequest it builds.
func TestDispatchRoutineRun_LightweightPathCopiesCausationIDOntoWakeupRequest(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := &models.Routine{
WorkspaceID: "ws-1",
Name: "Lightweight Causation",
TaskTemplate: "", // lightweight
AssigneeAgentProfileID: "agent-1",
Status: "active",
ConcurrencyPolicy: "always_create",
}
if err := svc.CreateRoutine(ctx, routine); err != nil {
t.Fatalf("create routine: %v", err)
}
enq := &fakeWakeupEnqueuer{}
svc.SetWakeupEnqueuer(enq)
run, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "alpha"})
if err != nil {
t.Fatalf("fire manual: %v", err)
}
if run.CausationID == "" {
t.Fatal("expected non-empty causation id on the routine run")
}
if len(enq.created) != 1 {
t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created))
}
if enq.created[0].CausationID != run.CausationID {
t.Fatalf("wakeup request causation id = %q, want %q (copied from the fire)",
enq.created[0].CausationID, run.CausationID)
}
}
@@ -0,0 +1,233 @@
package routines_test
import (
"context"
"expvar"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
)
// loopCounterInt reads one labelled entry off a registered
// office_loop_* expvar map. Missing entries read as zero so a test can
// assert a fresh delta without pre-seeding the map.
func loopCounterInt(t *testing.T, mapName, key string) int64 {
t.Helper()
v := expvar.Get(mapName)
if v == nil {
t.Fatalf("expvar map %s not registered", mapName)
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatalf("%s is not a Map", mapName)
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("%s[%q] is not an Int", mapName, key)
}
return iv.Value()
}
// REQ-OFFICE-LOOP-LIVENESS-003 / AC-003.1: TickScheduledTriggers counts
// one cron loop tick evaluation regardless of whether any trigger is due.
func TestTickScheduledTriggers_IncrementsCronTickCounter(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
before := loopCounterInt(t, "office_loop_cron_tick_total", "")
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC()); err != nil {
t.Fatalf("tick: %v", err)
}
after := loopCounterInt(t, "office_loop_cron_tick_total", "")
if after != before+1 {
t.Fatalf("cron_tick delta = %d, want 1", after-before)
}
}
// AC-003.1/AC-003.9: a due, successfully-claimed trigger increments
// office_loop_trigger_claimed_total labelled by the owning routine's
// workspace, at the site the claim persists.
func TestTickScheduledTriggers_IncrementsTriggerClaimedCounter(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{
RoutineID: routine.ID,
Kind: "cron",
CronExpression: "* * * * *",
Timezone: "UTC",
Enabled: true,
}); err != nil {
t.Fatalf("create trigger: %v", err)
}
enq := &fakeWakeupEnqueuer{}
svc.SetWakeupEnqueuer(enq)
key := "workspace=" + routine.WorkspaceID
before := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil {
t.Fatalf("tick: %v", err)
}
after := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
if after != before+1 {
t.Fatalf("trigger_claimed delta = %d, want 1", after-before)
}
}
// AC-003.9: a claim that persists must count even when the routine
// lookup afterward fails, so the claim can't be attributed to a
// workspace. Without the fallback, a claim that already wrote to the
// database silently vanished from office_loop_trigger_claimed_total
// (Review round 1, R1-4) — the counter must reflect the persisted
// write, not the lookup that happens to follow it.
func TestTickScheduledTriggers_ClaimedCounterSurvivesRoutineLookupFailure(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{
RoutineID: routine.ID,
Kind: "cron",
CronExpression: "* * * * *",
Timezone: "UTC",
Enabled: true,
}); err != nil {
t.Fatalf("create trigger: %v", err)
}
enq := &fakeWakeupEnqueuer{}
svc.SetWakeupEnqueuer(enq)
// Orphan the trigger: delete the routine it points at without
// touching the trigger row, so ClaimTrigger still succeeds but the
// subsequent GetRoutineFromConfig fails.
if err := svc.DeleteRoutine(ctx, routine.ID); err != nil {
t.Fatalf("delete routine: %v", err)
}
key := "workspace=_unattributed"
before := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil {
t.Fatalf("tick: %v", err)
}
after := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
if after != before+1 {
t.Fatalf("trigger_claimed[_unattributed] delta = %d, want 1", after-before)
}
}
// AC-003.2: a fire that runs to completion (lightweight, always_create)
// counts office_loop_routine_run_total with disposition=task_created.
func TestDispatchRoutineRun_CountsTaskCreatedDisposition(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
enq := &fakeWakeupEnqueuer{}
svc.SetWakeupEnqueuer(enq)
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created"
before := loopCounterInt(t, "office_loop_routine_run_total", key)
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("fire manual: %v", err)
}
after := loopCounterInt(t, "office_loop_routine_run_total", key)
if after != before+1 {
t.Fatalf("routine_run[task_created] delta = %d, want 1", after-before)
}
}
// AC-003.2: a fire that the concurrency policy skips counts
// office_loop_routine_run_total with disposition=skipped, distinct from
// the launched fire that decided the skip.
func TestDispatchRoutineRun_CountsSkippedDisposition(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
svc.SetTaskCreator(&fakeTaskCreator{})
routine := createTestRoutine(t, svc, "Skip Counter", "skip_if_active")
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("first fire: %v", err)
}
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=skipped"
before := loopCounterInt(t, "office_loop_routine_run_total", key)
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("second fire: %v", err)
}
after := loopCounterInt(t, "office_loop_routine_run_total", key)
if after != before+1 {
t.Fatalf("routine_run[skipped] delta = %d, want 1", after-before)
}
}
// AC-003.2: a fire the concurrency policy coalesces into an existing
// active run counts office_loop_routine_run_total with
// disposition=coalesced.
func TestDispatchRoutineRun_CountsCoalescedDisposition(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
svc.SetTaskCreator(&fakeTaskCreator{})
routine := createTestRoutine(t, svc, "Coalesce Counter", "coalesce_if_active")
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("first fire: %v", err)
}
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=coalesced"
before := loopCounterInt(t, "office_loop_routine_run_total", key)
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("second fire: %v", err)
}
after := loopCounterInt(t, "office_loop_routine_run_total", key)
if after != before+1 {
t.Fatalf("routine_run[coalesced] delta = %d, want 1", after-before)
}
}
// AC-003.1: the lightweight path's successfully-created wakeup request
// counts office_loop_wakeup_created_total labelled by the row's own
// source column, reusing routines.WakeupRequest.Source rather than
// inventing a parallel vocabulary.
func TestMaterialiseLightweightRoutineRun_IncrementsWakeupCreatedCounter(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
enq := &fakeWakeupEnqueuer{}
svc.SetWakeupEnqueuer(enq)
key := "workspace=" + routine.WorkspaceID + ";source=routine"
before := loopCounterInt(t, "office_loop_wakeup_created_total", key)
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("fire manual: %v", err)
}
if len(enq.created) != 1 {
t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created))
}
after := loopCounterInt(t, "office_loop_wakeup_created_total", key)
if after != before+1 {
t.Fatalf("wakeup_created delta = %d, want 1", after-before)
}
}
@@ -0,0 +1,128 @@
package routines_test
import (
"context"
"errors"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routines"
)
// AC-OFFICE-LOOP-LIVENESS-001.1: last_run_at advances on the coalesced
// and skipped concurrency-policy paths too, not only on a fresh
// dispatch — the call site runs before applyConcurrencyPolicy can
// short-circuit the return.
func TestDispatch_AdvancesLastRunAtOnSkippedAndCoalescedPaths(t *testing.T) {
t.Run("skipped", func(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
svc.SetTaskCreator(&fakeTaskCreator{})
routine := createTestRoutine(t, svc, "Skip Last Run", "skip_if_active")
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("first fire: %v", err)
}
afterFirst, err := svc.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if afterFirst.LastRunAt == nil {
t.Fatal("expected last_run_at set after first fire")
}
time.Sleep(2 * time.Millisecond)
run2, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("second fire: %v", err)
}
if run2.Status != "skipped" {
t.Fatalf("second fire status = %q, want skipped", run2.Status)
}
afterSecond, err := svc.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) {
t.Fatalf("last_run_at did not advance on skipped fire: before %v, after %v",
afterFirst.LastRunAt, afterSecond.LastRunAt)
}
})
t.Run("coalesced", func(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
svc.SetTaskCreator(&fakeTaskCreator{})
routine := createTestRoutine(t, svc, "Coalesce Last Run", "coalesce_if_active")
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
t.Fatalf("first fire: %v", err)
}
afterFirst, err := svc.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
time.Sleep(2 * time.Millisecond)
run2, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("second fire: %v", err)
}
if run2.Status != "coalesced" {
t.Fatalf("second fire status = %q, want coalesced", run2.Status)
}
afterSecond, err := svc.GetRoutine(ctx, routine.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) {
t.Fatalf("last_run_at did not advance on coalesced fire: before %v, after %v",
afterFirst.LastRunAt, afterSecond.LastRunAt)
}
})
}
// failingTouchRepo wraps the real sqlite repository but forces
// TouchRoutineLastRun to fail, so the dispatch's error-tolerance
// (AC-OFFICE-LOOP-LIVENESS-001.7) can be exercised without a real
// storage failure.
type failingTouchRepo struct {
*sqlite.Repository
}
func (f *failingTouchRepo) TouchRoutineLastRun(context.Context, string, time.Time) error {
return errors.New("simulated last_run_at write failure")
}
// AC-OFFICE-LOOP-LIVENESS-001.7: a TouchRoutineLastRun error does not
// fail the dispatch — the routine still fires and its run is created.
func TestDispatch_SurvivesLastRunAtWriteFailure(t *testing.T) {
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
repo, err := sqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
svc := routines.NewRoutineService(&failingTouchRepo{Repository: repo}, logger.Default(), &noopActivity{})
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
svc.SetTaskCreator(&fakeTaskCreator{})
routine := createTestRoutine(t, svc, "Failing Touch", "always_create")
run, err := svc.FireManual(context.Background(), routine.ID, nil)
if err != nil {
t.Fatalf("fire manual with failing touch: %v", err)
}
if run.Status != "task_created" {
t.Fatalf("status = %q, want task_created", run.Status)
}
}
@@ -0,0 +1,75 @@
package routines_test
import (
"context"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routines"
)
// newTestRoutineServiceRejectingRunStatusWrite is newTestRoutineService
// plus a trigger that aborts every UPDATE of office_routine_runs.status
// to 'task_created' or 'done', so a test can force materialiseRoutineRun's
// write to fail (on either the heavy or the lightweight path) without
// disturbing CreateRoutineRun (the write whose success is what
// office_loop_routine_run_total must count).
func newTestRoutineServiceRejectingRunStatusWrite(t *testing.T) *routines.RoutineService {
t.Helper()
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
repo, err := sqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
if _, err := db.Exec(`
CREATE TRIGGER reject_run_status_write
BEFORE UPDATE OF status ON office_routine_runs
WHEN NEW.status IN ('task_created', 'done')
BEGIN
SELECT RAISE(ABORT, 'run status write rejected for test');
END;
`); err != nil {
t.Fatalf("create trigger: %v", err)
}
log := logger.Default()
return routines.NewRoutineService(repo, log, &noopActivity{})
}
// AC-003.1 / AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total
// must count a routine run CreateRoutineRun genuinely persisted even when
// materialiseRoutineRun errors out afterward. Before the fix,
// IncLoopRoutineRun was only called inside the applyConcurrencyPolicy and
// materialiseRoutineRun success branches — an error from either meant
// dispatchRoutineRun returned without incrementing the counter at all,
// silently undercounting a fire that genuinely created a persisted row
// (the exact defect class this capability exists to catch: the reference
// incident was 322 consecutive fires that read as healthy because
// nothing counted them correctly).
func TestDispatchRoutineRun_CountsRunEvenWhenMaterializeFails(t *testing.T) {
svc := newTestRoutineServiceRejectingRunStatusWrite(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed"
before := loopCounterInt(t, "office_loop_routine_run_total", key)
if _, err := svc.FireManual(ctx, routine.ID, nil); err == nil {
t.Fatal("expected FireManual to return an error from the rejected status write")
}
after := loopCounterInt(t, "office_loop_routine_run_total", key)
if after != before+1 {
t.Fatalf("routine_run[failed] delta = %d, want 1 (the row was genuinely created, must not go uncounted)", after-before)
}
}
@@ -13,6 +13,7 @@ import (
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
"github.com/kandev/kandev/internal/office/shared"
taskservice "github.com/kandev/kandev/internal/task/service"
)
@@ -38,6 +39,7 @@ type Repository interface {
ListRoutines(ctx context.Context, workspaceID string) ([]*Routine, error)
UpdateRoutine(ctx context.Context, routine *Routine) error
DeleteRoutine(ctx context.Context, id string) error
TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error
CreateRoutineTrigger(ctx context.Context, t *RoutineTrigger) error
ListTriggersByRoutineID(ctx context.Context, routineID string) ([]*RoutineTrigger, error)
@@ -62,7 +64,6 @@ type Repository interface {
// read-then-write race between them.
UpdateRunStatusIfTaskCreated(ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string) (bool, error)
UpdateRunCoalesced(ctx context.Context, runID, coalescedIntoRunID string) error
TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error
// GetTaskTerminalStatus reads a task's real lifecycle state directly
// (not via the TaskMoved event — see applyConcurrencyPolicy) and
// reports "" when it is active, "done", "failed", "cancelled", or
@@ -95,6 +96,10 @@ type WakeupRequest struct {
Payload string
IdempotencyKey string
RequestedAt time.Time
// CausationID is copied from the firing RoutineRun
// (AC-OFFICE-LOOP-LIVENESS-002.2) so downstream runs and wakeup
// requests correlate back to the fire that produced them.
CausationID string
}
// RoutineWorkflowEnsurer materialises (lazily) the routine system
@@ -384,6 +389,7 @@ func (s *RoutineService) ListAllRoutineRuns(ctx context.Context, wsID string, li
// TickScheduledTriggers queries due cron triggers, claims each, and dispatches.
func (s *RoutineService) TickScheduledTriggers(ctx context.Context, now time.Time) error {
service.IncLoopCronTick(now.Format(time.RFC3339))
triggers, err := s.repo.GetDueTriggers(ctx, now)
if err != nil {
return fmt.Errorf("get due triggers: %w", err)
@@ -407,8 +413,12 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin
}
routine, err := s.GetRoutineFromConfig(ctx, trigger.RoutineID)
if err != nil {
// The claim already persisted, so the counter must still see it
// even though the workspace it belongs to can't be resolved.
service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace)
return fmt.Errorf("get routine: %w", err)
}
service.IncLoopTriggerClaimed(routine.WorkspaceID)
// Catch-up cap: count how many cron ticks elapsed between the missed
// trigger.NextRunAt (inclusive) and now, capped at routine.CatchUpMax.
// Mirror of the agent_heartbeat catch-up math, adapted for cron
@@ -587,28 +597,65 @@ func (s *RoutineService) dispatchRoutineRun(
TriggerPayload: string(payloadJSON),
DispatchFingerprint: fingerprint,
StartedAt: &now,
// CausationID is minted once per fire (AC-OFFICE-LOOP-LIVENESS-002.1)
// and copied onto every wakeup request and run this fire produces.
CausationID: uuid.New().String(),
}
if err := s.repo.CreateRoutineRun(ctx, run); err != nil {
return nil, fmt.Errorf("create run: %w", err)
}
// AC-003.1/AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total
// counts this fire's CreateRoutineRun above — a write that just
// succeeded — not whatever applyConcurrencyPolicy/materialiseRoutineRun
// do afterward. A single deferred increment fires exactly once no
// matter which return path below is taken, labelled by whatever
// disposition was reached; it defaults to "failed" (the existing
// RoutineRunStatus value, not an invented one) so a fire that created
// a real row but then errored is still counted rather than silently
// dropped.
disposition := string(models.RoutineRunStatusFailed)
defer func() {
service.IncLoopRoutineRun(routine.WorkspaceID, source, disposition)
}()
// AC-OFFICE-LOOP-LIVENESS-001.1: advances last_run_at for every
// dispatch, whatever disposition the run later reaches — so it must
// run before applyConcurrencyPolicy, which can short-circuit into a
// coalesced or skipped return. Errors are logged and counted, never
// returned (AC-001.7): a routine that fired must not read as having
// failed to fire because a bookkeeping write lost a lock.
if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, *run.StartedAt); err != nil {
service.IncLoopLastRunAtWriteFailed(routine.WorkspaceID)
s.logger.Warn("touch routine last_run_at",
zap.String("routine_id", routine.ID), zap.Error(err))
}
status, err := s.applyConcurrencyPolicy(ctx, routine, run, fingerprint)
if err != nil {
return run, err
}
if status != "" {
disposition = string(status)
return run, nil
}
if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil {
return run, err
}
if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, now); err != nil {
s.logger.Warn("touch routine last_run_at",
zap.String("routine", routine.Name), zap.Error(err))
// A nil error only means materialiseRoutineRun's terminal write
// succeeded, not that the write it made was a success: the
// lightweight path finalizes with a nil error even when the wakeup
// enqueue itself failed, persisting run.Status=failed. Disposition
// must reflect that instead of assuming "task_created" (the
// dispatched-successfully label, shared by the heavy path and a
// lightweight done) whenever this call merely didn't error.
if run.Status == models.RoutineRunStatusFailed {
disposition = string(models.RoutineRunStatusFailed)
} else {
disposition = string(models.RoutineRunStatusTaskCreated)
}
routine.LastRunAt = &now
s.logger.Info("routine run dispatched",
zap.String("routine", routine.Name),
zap.String("run_id", run.ID),
@@ -713,6 +760,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun(
Payload: payloadStr,
IdempotencyKey: idemKey,
RequestedAt: time.Now().UTC(),
CausationID: run.CausationID,
}
if err := s.wakeup.CreateWakeupRequest(ctx, req); err != nil {
if errors.Is(err, ErrWakeupAlreadyRequested) {
@@ -724,6 +772,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun(
zap.String("routine", routine.Name), zap.Error(err))
return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusFailed)
}
service.IncLoopWakeupCreated(routine.WorkspaceID, req.Source)
if err := s.wakeup.Dispatch(ctx, req.ID); err != nil {
s.logger.Warn("dispatch routine wakeup request",
zap.String("routine", routine.Name),
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
@@ -716,9 +717,13 @@ func TestDispatch_LastRunAt(t *testing.T) {
t.Fatal("expected last_run_at to be set after a materialised fire")
}
firstStamp := *afterFirst.LastRunAt
time.Sleep(2 * time.Millisecond)
// Second fire is skipped (run1's task is still open) — last_run_at
// must not advance for a fire that didn't materialise.
// Second fire is skipped (run1's task is still open), but
// AC-OFFICE-LOOP-LIVENESS-001.1 advances last_run_at for every
// dispatch regardless of disposition — the write runs before
// applyConcurrencyPolicy can short-circuit into skipped/coalesced,
// so a skip still counts as evidence the loop is alive.
run2, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("second fire: %v", err)
@@ -730,7 +735,7 @@ func TestDispatch_LastRunAt(t *testing.T) {
if err != nil {
t.Fatalf("get routine: %v", err)
}
if !afterSecond.LastRunAt.Equal(firstStamp) {
t.Errorf("last_run_at changed from %v to %v after a skipped fire", firstStamp, afterSecond.LastRunAt)
if !afterSecond.LastRunAt.After(firstStamp) {
t.Errorf("last_run_at did not advance on a skipped fire: before %v, after %v", firstStamp, afterSecond.LastRunAt)
}
}
@@ -0,0 +1,64 @@
package routines_test
import (
"context"
"errors"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/routines"
)
// failingCreateWakeupEnqueuer fails every CreateWakeupRequest call with a
// generic (non-ErrWakeupAlreadyRequested) error, but still lets Dispatch/
// FailWakeupRequest succeed trivially — they are never reached once
// CreateWakeupRequest itself fails.
type failingCreateWakeupEnqueuer struct{ fakeWakeupEnqueuer }
func (f *failingCreateWakeupEnqueuer) CreateWakeupRequest(context.Context, *routines.WakeupRequest) error {
return errors.New("wakeup store unavailable")
}
// TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated is the
// PR fixup round 2 regression test (CodeRabbit). materialiseLightweightRoutineRun
// finalizes a run as RoutineRunStatusFailed when CreateWakeupRequest errors,
// but returns a nil error once that finalizing UPDATE itself succeeds. The
// caller (dispatchRoutineRun) used to read that nil error as "materialise
// succeeded" and hard-code disposition = task_created regardless of what
// status was actually persisted — so a routine run that genuinely failed to
// enqueue its wakeup still recorded office_loop_routine_run_total as a
// success, corrupting the loop-health evidence this capability exists to
// produce.
func TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated(t *testing.T) {
svc := newTestRoutineService(t)
ctx := context.Background()
routine := newLightweightTestRoutine(t, svc)
svc.SetWakeupEnqueuer(&failingCreateWakeupEnqueuer{})
failedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed"
taskCreatedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created"
failedBefore := loopCounterInt(t, "office_loop_routine_run_total", failedKey)
taskCreatedBefore := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey)
run, err := svc.FireManual(ctx, routine.ID, nil)
if err != nil {
t.Fatalf("FireManual: %v", err)
}
if run.Status != models.RoutineRunStatusFailed {
t.Fatalf("run.Status = %q, want %q — the wakeup enqueue failed and finalizeLightweightRun persisted 'failed'",
run.Status, models.RoutineRunStatusFailed)
}
failedAfter := loopCounterInt(t, "office_loop_routine_run_total", failedKey)
if failedAfter != failedBefore+1 {
t.Fatalf("routine_run[failed] delta = %d, want 1 — a failed wakeup enqueue must not be "+
"reported as disposition=task_created", failedAfter-failedBefore)
}
taskCreatedAfter := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey)
if taskCreatedAfter != taskCreatedBefore {
t.Fatalf("routine_run[task_created] delta = %d, want 0 for a run whose wakeup enqueue failed",
taskCreatedAfter-taskCreatedBefore)
}
}
@@ -14,8 +14,23 @@ import (
"github.com/kandev/kandev/internal/office/models"
sqliterepo "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routing"
"github.com/kandev/kandev/internal/office/service"
)
// TaskStarterWithSession optionally returns the id of the agent session
// a routed launch created, mirroring service.TaskStarterWithSession for
// the legacy launch path (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that
// does not implement this falls back to StartTaskWithRoute, and the
// run's session id stays empty.
type TaskStarterWithSession interface {
StartTaskWithRouteReturningSession(
ctx context.Context,
taskID, agentProfileID string,
launch LaunchContext,
route RouteOverride,
) (sessionID string, err error)
}
// RouteAttemptOutcomeLaunched is the in-flight outcome string the
// dispatcher writes when an attempt is appended. It is overwritten with
// failed_provider_unavailable / failed_other on classifier verdict.
@@ -257,7 +272,7 @@ func (ss *SchedulerService) tryCandidates(
if err != nil {
return false, nil, err
}
launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch)
sessionID, launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch)
if launchErr == nil {
if prev != nil {
// We walked past at least one prior candidate — that's
@@ -270,7 +285,7 @@ func (ss *SchedulerService) tryCandidates(
}
ss.recordRouteAttempt(agent.WorkspaceID,
string(candidate.ProviderID), metricOutcomeSuccess, "")
return ss.handleLaunchSuccess(ctx, run, agent, candidate)
return ss.handleLaunchSuccess(ctx, run, agent, candidate, sessionID)
}
// Emit the failure-side route_attempt before classifying. The
// outcome label is refined by handleLaunchFailure.
@@ -402,20 +417,20 @@ func (ss *SchedulerService) recordAttemptStart(
func (ss *SchedulerService) launchCandidate(
ctx context.Context, taskID, agentID string,
candidate routing.Candidate, launch LaunchContext,
) error {
) (string, error) {
if taskID == "" {
return fmt.Errorf("dispatch: empty task id in run payload")
return "", fmt.Errorf("dispatch: empty task id in run payload")
}
if _, ok := routingerr.InjectedCode(string(candidate.ProviderID)); ok {
// Synthesize a launch failure via Classify so injection is
// honoured for deterministic E2E specs. Classify short-circuits
// to the injected code at the head of its decision chain.
return routingerr.Classify(routingerr.Input{
return "", routingerr.Classify(routingerr.Input{
Phase: routingerr.PhaseProcessStart,
ProviderID: string(candidate.ProviderID),
})
}
return ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, RouteOverride{
route := RouteOverride{
ExecutionProfileID: candidate.ExecutionProfileID,
ProviderID: string(candidate.ProviderID),
Model: candidate.Model,
@@ -423,24 +438,72 @@ func (ss *SchedulerService) launchCandidate(
Mode: candidate.Mode,
Flags: candidate.Flags,
Env: candidate.Env,
})
}
if starter, ok := ss.taskStarter.(TaskStarterWithSession); ok {
return starter.StartTaskWithRouteReturningSession(ctx, taskID, agentID, launch, route)
}
return "", ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, route)
}
// handleLaunchSuccess records the resolved provider/model on the run
// row and flips every health scope this candidate touched back to
// healthy. Returns (launched=true, nil, nil) on success.
// handleLaunchSuccess persists the launched session id, counts the
// launch, records the resolved provider/model on the run row, and
// flips every health scope this candidate touched back to healthy.
// Always returns (launched=true, nil, nil): by the time this is called
// launchCandidate has already started a real agent session, so nothing
// downstream may report the launch as failed or leave it uncounted
// (AC-002.7, AC-003.9). The session id and the launch counter are
// persisted first and unconditionally — they are the correlation this
// capability exists to preserve — before the best-effort
// resolved-route snapshot, whose own failure (Review round 3, R3-2)
// must not undo them or cause a caller to retry-launch an agent that
// is already running.
func (ss *SchedulerService) handleLaunchSuccess(
ctx context.Context, run *models.Run,
agent *models.AgentInstance, candidate routing.Candidate,
agent *models.AgentInstance, candidate routing.Candidate, sessionID string,
) (bool, *routing.BlockReason, error) {
service.IncLoopLaunch(agent.WorkspaceID)
ss.persistLaunchedSession(ctx, run.ID, agent.WorkspaceID, sessionID)
if err := ss.repo.SetRunResolvedRoute(ctx,
run.ID, candidate.ExecutionProfileID, string(candidate.ProviderID), candidate.Model); err != nil {
return false, nil, err
ss.logger.Warn("persist resolved route failed",
zap.String("run_id", run.ID), zap.String("provider_id", string(candidate.ProviderID)), zap.Error(err))
}
ss.markHealthScopes(ctx, agent.WorkspaceID, candidate)
return true, nil, nil
}
// persistLaunchedSession stores the session id a successful launch
// produced (AC-OFFICE-LOOP-LIVENESS-002.7). An empty id is counted as a
// without-session launch (AC-002.8) rather than attempted as a write.
// A write failure does not fail the launch — the agent is already
// running — and is counted separately from the without-session case
// (AC-002.11) so the two causes of an identical "processed, no
// session" row stay distinguishable in the counters. A non-empty id
// whose write affects zero rows (e.g. the run row vanished between
// claim and this write) is the same AC-002.11 case as a hard write
// error — a real session id that failed to persist — not the AC-002.8
// case, so it counts under the same persist-failed counter.
func (ss *SchedulerService) persistLaunchedSession(
ctx context.Context, runID, workspaceID, sessionID string,
) {
if sessionID == "" {
service.IncLoopLaunchWithoutSession(workspaceID)
return
}
wrote, err := ss.repo.SetRunSessionID(ctx, runID, sessionID)
if err != nil {
ss.logger.Warn("persist launched session id failed",
zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err))
service.IncLoopSessionPersistFailed(workspaceID)
return
}
if !wrote {
ss.logger.Warn("persist launched session id matched no row",
zap.String("run_id", runID), zap.String("session_id", sessionID))
service.IncLoopSessionPersistFailed(workspaceID)
}
}
// markHealthScopes marks the provider, tier, and model scopes healthy
// for the launched candidate. Best-effort: errors are logged but do
// not fail the launch.
@@ -0,0 +1,85 @@
package scheduler_test
import (
"context"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routing"
"github.com/kandev/kandev/internal/office/scheduler"
)
// newTestRepoSchedRejectingResolvedRouteWrite is newTestRepoSched plus a
// trigger that aborts every write to resolved_execution_profile_id, so a
// test can force SetRunResolvedRoute to fail without disturbing the
// session_id write persistLaunchedSession performs — the two writes
// this test needs to tell apart.
func newTestRepoSchedRejectingResolvedRouteWrite(t *testing.T) *officesqlite.Repository {
t.Helper()
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
db.SetMaxOpenConns(1)
t.Cleanup(func() { _ = db.Close() })
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
t.Fatalf("settings store: %v", err)
}
repo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
if _, err := db.Exec(`
CREATE TRIGGER reject_resolved_route_write
BEFORE UPDATE OF resolved_execution_profile_id ON runs
BEGIN
SELECT RAISE(ABORT, 'resolved route write rejected for test');
END;
`); err != nil {
t.Fatalf("create trigger: %v", err)
}
return repo
}
// AC-002.7 / AC-002.11 / AC-003.9 (Review round 3, R3-2): a launch that
// already started a real agent session must not lose that session's
// correlation, or go uncounted as a launch, just because an unrelated
// bookkeeping write (the resolved provider/model snapshot) fails
// afterward. Before the fix, handleLaunchSuccess returned early on a
// SetRunResolvedRoute error, before ever reaching persistLaunchedSession
// or IncLoopLaunch — silently abandoning a live, now-uncorrelated
// session and under-counting a launch that genuinely happened.
func TestDispatch_RoutedLaunchSurvivesResolvedRouteWriteFailure(t *testing.T) {
repo := newTestRepoSchedRejectingResolvedRouteWrite(t)
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-resolved-route-fail"}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-resolved-route-fail"}`)
beforeLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
if err != nil || !launched || parked {
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the resolved-route write failure", launched, parked, err)
}
if !starter.returnedCall {
t.Fatal("expected StartTaskWithRouteReturningSession to be called")
}
updated, err := repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if updated.SessionID != "sess-resolved-route-fail" {
t.Fatalf("session_id = %q, want sess-resolved-route-fail (a live launch must not lose its session correlation)", updated.SessionID)
}
afterLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
if afterLaunch != beforeLaunch+1 {
t.Fatalf("office_loop_launch_total delta = %d, want 1 (the launch genuinely happened)", afterLaunch-beforeLaunch)
}
}
@@ -0,0 +1,152 @@
package scheduler_test
import (
"context"
"fmt"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routing"
"github.com/kandev/kandev/internal/office/scheduler"
)
// newTestRepoSchedRejectingSessionWrite is newTestRepoSched plus a
// trigger that aborts only an UPDATE setting runs.session_id to
// wantSessionID — the exact value the launch under test will produce —
// so a test can force SetRunSessionID to fail (AC-002.11) without
// disturbing any other write persistLaunchedSession's callers depend on
// (route-attempt bookkeeping, resolved-route persistence).
func newTestRepoSchedRejectingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository {
t.Helper()
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
db.SetMaxOpenConns(1)
t.Cleanup(func() { _ = db.Close() })
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
t.Fatalf("settings store: %v", err)
}
repo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
if _, err := db.Exec(fmt.Sprintf(`
CREATE TRIGGER reject_session_id_write
BEFORE UPDATE OF session_id ON runs
WHEN NEW.session_id = '%s'
BEGIN
SELECT RAISE(ABORT, 'session_id write rejected for test');
END;
`, wantSessionID)); err != nil {
t.Fatalf("create trigger: %v", err)
}
return repo
}
// AC-002.11: on a routed launch, a session persist failure is counted
// separately from a without-session launch, and does not fail the
// launch itself or overwrite the run's session_id — the agent is
// already running by the time this write happens.
func TestDispatch_RoutedLaunchSessionPersistFailureCountsSeparately(t *testing.T) {
repo := newTestRepoSchedRejectingSessionWrite(t, "sess-routed-persist-fail")
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-persist-fail"}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-session-persist-fail"}`)
beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
if err != nil || !launched || parked {
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the persist failure", launched, parked, err)
}
updated, err := repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if updated.SessionID != "" {
t.Fatalf("session_id = %q, want unchanged/empty after a rejected write", updated.SessionID)
}
afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
if afterFailed != beforeFailed+1 {
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed)
}
afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
if afterWithoutSession != beforeWithoutSession {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession)
}
}
// newTestRepoSchedSilentlyDroppingSessionWrite is newTestRepoSched plus a
// trigger that uses RAISE(IGNORE) rather than RAISE(ABORT): the UPDATE
// setting runs.session_id to wantSessionID matches zero rows and returns
// no error, exactly the shape a run row that vanished between claim and
// launch (an unrelated delete/archive racing the session-persist write)
// would produce. This is a different failure mode than
// newTestRepoSchedRejectingSessionWrite's hard DB error: SetRunSessionID
// reports (wrote=false, err=nil), not (wrote=false, err=non-nil).
func newTestRepoSchedSilentlyDroppingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository {
t.Helper()
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
db.SetMaxOpenConns(1)
t.Cleanup(func() { _ = db.Close() })
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
t.Fatalf("settings store: %v", err)
}
repo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
if _, err := db.Exec(fmt.Sprintf(`
CREATE TRIGGER ignore_session_id_write
BEFORE UPDATE OF session_id ON runs
WHEN NEW.session_id = '%s'
BEGIN
SELECT RAISE(IGNORE);
END;
`, wantSessionID)); err != nil {
t.Fatalf("create trigger: %v", err)
}
return repo
}
// AC-002.11: a launch that yields a real, non-empty session id whose
// write affects zero rows (no error) must be counted as a session
// persist failure, not miscounted as "launch yielded no session id"
// (AC-002.8's counter) — the two causes of an identical "processed, no
// session" row must stay distinguishable.
func TestDispatch_RoutedLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) {
repo := newTestRepoSchedSilentlyDroppingSessionWrite(t, "sess-routed-vanished")
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-vanished"}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-session-vanished"}`)
beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
if err != nil || !launched || parked {
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the dropped write", launched, parked, err)
}
afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
if afterFailed != beforeFailed+1 {
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed)
}
afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
if afterWithoutSession != beforeWithoutSession {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession)
}
}
@@ -0,0 +1,153 @@
package scheduler_test
import (
"context"
"expvar"
"testing"
"github.com/kandev/kandev/internal/office/routing"
"github.com/kandev/kandev/internal/office/scheduler"
)
// fakeTaskStarterWithSession wraps fakeTaskStarter and additionally
// implements scheduler.TaskStarterWithSession, so dispatch tests can
// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7)
// without touching the production orchestrator adapter.
type fakeTaskStarterWithSession struct {
*fakeTaskStarter
sessionID string
sessionErr error
calledRoute scheduler.RouteOverride
returnedCall bool
}
func (f *fakeTaskStarterWithSession) StartTaskWithRouteReturningSession(
ctx context.Context, taskID, agentID string,
launch scheduler.LaunchContext, route scheduler.RouteOverride,
) (string, error) {
f.returnedCall = true
f.calledRoute = route
if err := f.StartTaskWithRoute(ctx, taskID, agentID, launch, route); err != nil {
return "", err
}
if f.sessionErr != nil {
return "", f.sessionErr
}
return f.sessionID, nil
}
// expvarMapInt reads one label key out of a package-global expvar.Map,
// returning 0 when the key was never incremented. Used to assert
// counter deltas without a reset seam — expvar's registry is global and
// process-wide, so this reads exactly what production code would.
func expvarMapInt(t *testing.T, mapName, key string) int64 {
t.Helper()
v := expvar.Get(mapName)
if v == nil {
t.Fatalf("expvar map %q not registered", mapName)
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatalf("expvar %q is not a Map", mapName)
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("expvar %q[%q] is not an Int", mapName, key)
}
return iv.Value()
}
// AC-OFFICE-LOOP-LIVENESS-002.7: on a routed launch, the session id the
// starter returns is persisted onto the run row.
func TestDispatch_RoutedLaunchPersistsSessionID(t *testing.T) {
repo := newTestRepoSched(t)
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-1"}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-session-1"}`)
before := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
if err != nil || !launched || parked {
t.Fatalf("launched=%v parked=%v err=%v, want launched", launched, parked, err)
}
if !starter.returnedCall {
t.Fatal("expected StartTaskWithRouteReturningSession to be called")
}
updated, err := repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if updated.SessionID != "sess-routed-1" {
t.Fatalf("session_id = %q, want sess-routed-1", updated.SessionID)
}
after := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
if after != before+1 {
t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before)
}
}
// AC-002.8: an empty returned session id is left empty and counted
// under launch_without_session, never replaced with a placeholder.
func TestDispatch_RoutedLaunchEmptySessionCountsWithoutSession(t *testing.T) {
repo := newTestRepoSched(t)
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: ""}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-session-2"}`)
before := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
launched, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
if err != nil || !launched {
t.Fatalf("launched=%v err=%v, want launched", launched, err)
}
updated, err := repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if updated.SessionID != "" {
t.Fatalf("session_id = %q, want empty (no placeholder)", updated.SessionID)
}
after := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
if after != before+1 {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before)
}
}
// AC-002.10: relaunching the same run with a new session id overwrites
// the previous one — last non-empty wins within one claim.
func TestDispatch_RoutedRelaunchKeepsLatestSessionID(t *testing.T) {
repo := newTestRepoSched(t)
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-first"}
ss := buildScheduler(t, repo, starter)
run := seedRun(t, repo, `{"task_id":"t-session-3"}`)
if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil {
t.Fatalf("first dispatch: %v", err)
}
starter.sessionID = "sess-second"
// Re-fetch: DispatchWithRouting mutates run.LogicalProviderOrder in
// place, so reuse the same struct to simulate a relaunch of the
// same claimed row.
if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil {
t.Fatalf("second dispatch: %v", err)
}
updated, err := repo.GetRunByID(context.Background(), run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if updated.SessionID != "sess-second" {
t.Fatalf("session_id = %q, want sess-second (last non-empty wins)", updated.SessionID)
}
}
@@ -33,13 +33,15 @@ func (ss *SchedulerService) ClaimNextRun(ctx context.Context) (*models.Run, erro
// happened, so it is not bucketed as processed (docs/specs/
// task-delivery-ledger/spec.md, "Office run outcome").
func (ss *SchedulerService) FinishRun(ctx context.Context, id string) error {
return ss.repo.FinishRun(ctx, id, RunStatusFinished, nil)
_, err := ss.repo.FinishRun(ctx, id, RunStatusFinished, nil)
return err
}
// FailRun marks a claimed run as failed. outcome is written as NULL, same
// as the service-layer FailRun: a failed run is bucketed on status alone.
func (ss *SchedulerService) FailRun(ctx context.Context, id string) error {
return ss.repo.FinishRun(ctx, id, RunStatusFailed, nil)
_, err := ss.repo.FinishRun(ctx, id, RunStatusFailed, nil)
return err
}
// ProcessRunGuard checks if the agent is still eligible to be woken.
@@ -172,7 +172,18 @@ func (si *SchedulerIntegration) finishPolicyBlock(
} else {
incBudgetBlockedByLimit(provenance)
}
_ = si.svc.FinishRun(ctx, run.ID, outcome)
wrote, err := si.svc.FinishRun(ctx, run.ID, outcome)
if err != nil {
si.logger.Error("failed to finish policy-blocked run",
zap.String("run_id", run.ID), zap.Error(err))
return false
}
if !wrote {
// Another writer already moved the run out of claimed between the
// admission decision and this write; it did not actually end via a
// budget block, so there is nothing to log (Review round 3, R3-1).
return false
}
fields := map[string]string{"degraded": strconv.FormatBool(p.Degraded)}
if p.IsDefault {
@@ -197,11 +208,21 @@ func (si *SchedulerIntegration) cancelBudgetRun(
si.releaseCheckoutIfNeeded(ctx, run)
si.svc.clearAgentWorking(ctx, agent.ID, run.ID)
if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil {
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason)
if err != nil {
si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err))
} else {
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
return false
}
if !cancelled {
// Another writer already moved the run out of claimed between the
// admission decision and this write; it did not actually end via
// this cancellation, so there is nothing to classify, publish, or
// log.
return false
}
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
fields := map[string]string{activityFieldCeiling: ceilingNotDetermined}
for k, v := range extraFields {
@@ -383,11 +404,17 @@ func (si *SchedulerIntegration) cancelUnresolvableAgentRun(ctx context.Context,
incBudgetCancelledNoWorkspace(provenance)
si.cleanupWorkspaceLookupRun(ctx, run)
if err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace"); err != nil {
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace")
if err != nil {
si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err))
} else {
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
return
}
if !cancelled {
return
}
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
si.svc.LogActivityWithRun(ctx, "", "scheduler", "office-scheduler",
"run_budget_workspace_unresolvable", "run", run.ID,
mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "")
@@ -409,11 +436,19 @@ func (si *SchedulerIntegration) deferWorkspaceLookupFailure(ctx context.Context,
if run.RetryCount >= MaxRetryCount {
incBudgetFailedWorkspaceLookup(provenance)
if err := si.svc.FailRun(ctx, run.ID); err != nil {
wrote, err := si.svc.FailRun(ctx, run.ID)
if err != nil {
si.logger.Error("failed to fail run without escalation",
zap.String("run_id", run.ID), zap.Error(err))
return
}
if !wrote {
// Already terminal via another writer (e.g. a concurrent
// cancel) between the lookup failure and this write; it did
// not actually end via this failure, so there is nothing to
// log (Review round 3, R3-1).
return
}
si.svc.LogActivityWithRun(ctx, "", "system", "scheduler",
"run_budget_workspace_lookup_failed", "run", run.ID,
mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "")
@@ -0,0 +1,201 @@
package service_test
import (
"context"
"testing"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// cancelBudgetRun and cancelUnresolvableAgentRun mirror cancelStaleRun's
// release/cancel/classify/publish/log sequence, but their CancelRun call
// must still be gated the same way: a run that another writer already
// moved out of claimed/queued before this cancel runs must not be
// classified, broadcast, or logged as cancelled.
func TestCancelBudgetRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) {
svc, eb := newTestServiceWithBus(t)
ctx := context.Background()
agent := makeAgent("worker-budget-cancel-race", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
// Another writer wins the terminal-transition race before gate 1's
// cancel below runs: the row is already finished, so CancelRun's
// guarded UPDATE (status IN ('queued','claimed')) will match nothing.
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish (simulating the winning writer): %v", err)
}
got := make(chan *bus.Event, 1)
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
got <- e
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
unresolvedAgent := *agent
unresolvedAgent.WorkspaceID = ""
service.AdmitRunForTest(svc, ctx, run, &unresolvedAgent)
select {
case e := <-got:
t.Fatalf("expected no OfficeRunProcessed publish for a lost budget-cancel race; got %+v", e)
default:
}
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries))
}
survivor, err := svc.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if survivor.Status != service.RunStatusFinished {
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
}
}
// deferWorkspaceLookupFailure's MaxRetryCount-exhausted branch (CodeRabbit,
// PR fixup round 2) has the same lost-race obligation as cancelBudgetRun
// above: it discarded FailRun's wrote result and logged
// run_budget_workspace_lookup_failed unconditionally whenever err was nil,
// even when another writer had already made the run terminal.
func TestDeferWorkspaceLookupFailure_LostRaceDoesNotLog(t *testing.T) {
svc, eb := newTestServiceWithBus(t)
ctx := context.Background()
agent := makeAgent("worker-workspace-lookup-race", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
run.RetryCount = service.MaxRetryCount
// Another writer wins the terminal-transition race before this call's
// FailRun below runs.
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish (simulating the winning writer): %v", err)
}
got := make(chan *bus.Event, 1)
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
got <- e
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
service.DeferWorkspaceLookupFailureForTest(svc, ctx, run)
select {
case e := <-got:
t.Fatalf("expected no OfficeRunProcessed publish for a lost workspace-lookup-failure race; got %+v", e)
default:
}
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_lookup_failed", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_budget_workspace_lookup_failed activity entries = %d, want 0 for a lost race", len(entries))
}
survivor, err := svc.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if survivor.Status != service.RunStatusFinished {
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
}
}
// cancelUnresolvableAgentRun (reached when the run's agent was deleted)
// has the same lost-race obligation as cancelBudgetRun above.
func TestCancelUnresolvableAgentRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) {
svc, eb := newTestServiceWithBus(t)
ctx := context.Background()
agent := makeAgent("worker-unresolvable-cancel-race", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
if err := svc.RepoForTest().DeleteAgentInstance(ctx, agent.ID); err != nil {
t.Fatalf("delete agent: %v", err)
}
// Another writer wins the terminal-transition race before the
// missing-agent cancel below runs.
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish (simulating the winning writer): %v", err)
}
got := make(chan *bus.Event, 1)
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
got <- e
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
service.ProcessRunForTest(svc, ctx, run)
select {
case e := <-got:
t.Fatalf("expected no OfficeRunProcessed publish for a lost missing-agent cancel race; got %+v", e)
default:
}
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries))
}
survivor, err := svc.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if survivor.Status != service.RunStatusFinished {
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
}
}
@@ -180,7 +180,14 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T)
// Drive the run to MaxRetryCount and re-process it directly (bypassing
// the real backoff delay, exactly as the existing retry tests do).
// FailRun's guarded write only takes effect while the row is still
// status='claimed' (Review round 3, R3-1), so the deferred row —
// left 'queued' by the first tick above — must be reclaimed before
// ProcessRunForTest, mirroring scheduler_run_outcome_test.go.
svc.ExecSQL(t, `UPDATE runs SET retry_count = ?, status = 'claimed' WHERE id = ?`,
service.MaxRetryCount, run.ID)
run.RetryCount = service.MaxRetryCount
run.Status = "claimed"
service.ProcessRunForTest(svc, ctx, run)
failed, err := svc.GetRun(ctx, run.ID)
@@ -421,14 +421,6 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er
}
return err
}
// Lifecycle: terminal "complete" event for the run detail Events log.
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
"task_id": data.TaskID,
"session_id": data.SessionID,
})
s.markRoutingSuccess(ctx, run)
s.recordRunOutputSummary(ctx, run, *data)
s.warnIfReviewDecisionMissing(ctx, run)
// resolveLifecycleRun prefers the immutable run ID from the event and
// falls back to task plus agent only for legacy events. Releasing here
// (rather than unconditionally inside transitionRunTerminal) is safe —
@@ -438,10 +430,35 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er
// must stay held so ReapStaleCheckouts (not a same-agent race) is what
// eventually reclaims it, instead of releasing a lock for a run that
// never actually reached a terminal state.
if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil {
wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed)
if err != nil {
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
return err
}
if !wrote {
// The run reached a terminal state through another writer (e.g. a
// concurrent cancel) between resolveLifecycleRun's read and this
// write — it never actually held the checkout from this call's
// point of view, so releasing it here would steal a live lock the
// same way an unconditional release would (Review round 3, R3-1).
// The agent may still be stuck "working" from the launch though.
// The completion side effects below (timeline event, routing
// health, output summary, review-decision warning) must not run
// either: this run did not actually complete from this call's
// point of view, so persisting them would attribute another
// writer's outcome (e.g. a cancel) with this one's evidence
// (CodeRabbit, PR fixup round 2).
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
return nil
}
// Lifecycle: terminal "complete" event for the run detail Events log.
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
"task_id": data.TaskID,
"session_id": data.SessionID,
})
s.markRoutingSuccess(ctx, run)
s.recordRunOutputSummary(ctx, run, *data)
s.warnIfReviewDecisionMissing(ctx, run)
s.releaseTaskCheckoutForRun(ctx, run)
s.stampRunFinished(ctx, run)
return nil
@@ -507,6 +524,12 @@ func (s *Service) markRoutingSuccess(ctx context.Context, run *models.Run) {
rd.MarkRunSuccessHealth(ctx, run, agent)
}
// runEventFieldAgentID is the run-event payload key for an agent id.
// Named to avoid a duplicate-literal lint failure — "agent_id" also
// appears as a JSON struct tag elsewhere in this file, and those two
// uses are otherwise unrelated to each other.
const runEventFieldAgentID = "agent_id"
// handleTasklessAgentCompleted attributes a taskless run completion,
// finishes the run, and refreshes the per-agent, per-scope continuation
// summary so the next fire has bridge context. The
@@ -540,12 +563,6 @@ func (s *Service) handleTasklessAgentCompleted(
}
return err
}
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
"agent_id": data.AgentID,
"session_id": data.SessionID,
})
s.refreshContinuationSummary(ctx, run, run.AgentProfileID)
s.recordRunOutputSummary(ctx, run, *data)
// run came from GetClaimedTasklessRunForAgent: it is the run that
// actually launched. Taskless runs typically carry no task_id, so this
// is a no-op in the common case, but call it for the same reason as
@@ -553,9 +570,26 @@ func (s *Service) handleTasklessAgentCompleted(
//
// Finish before releasing, same as handleAgentCompleted: a failed
// FinishRun must not still give up the checkout.
if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil {
wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed)
if err != nil {
return err
}
if !wrote {
// Already terminal via another writer — see handleAgentCompleted's
// matching branch (Review round 3, R3-1). The completion side
// effects below must not run either: this run did not actually
// complete from this call's point of view, so recording them
// (timeline event, continuation summary, output summary) would
// attribute another writer's outcome with this one's evidence
// (CodeRabbit, PR fixup round 2).
return nil
}
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
runEventFieldAgentID: data.AgentID,
"session_id": data.SessionID,
})
s.refreshContinuationSummary(ctx, run, run.AgentProfileID)
s.recordRunOutputSummary(ctx, run, *data)
s.releaseTaskCheckoutForRun(ctx, run)
s.stampRunFinished(ctx, run)
return nil
@@ -708,9 +742,16 @@ func (s *Service) handleAgentFailed(ctx context.Context, event *bus.Event) error
// rate-limit-retry callers; we deliberately do NOT call into it
// here. See docs/specs/office/requirements/runtime.md.
errMsg := enrichModelFailureMessage(run, data.ErrorMessage)
if err := s.HandleAgentFailure(ctx, run, errMsg); err != nil {
wrote, err := s.HandleAgentFailure(ctx, run, errMsg)
if err != nil {
return err
}
if !wrote {
// Already terminal via another writer — waking the CEO agent
// over a run that never actually failed would be a false alarm
// (Review round 3, R3-1).
return nil
}
s.dispatchAgentErrorTrigger(ctx, run, data.TaskID, data.SessionID, errMsg)
return nil
}
@@ -0,0 +1,147 @@
package service_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects is the
// PR fixup round 2 regression test (CodeRabbit). handleAgentCompleted used
// to record the "complete" run event and the output summary BEFORE checking
// FinishRun's result at all — so a run whose terminal write never actually
// landed (a genuine persistence failure here, or a lost race to a
// concurrent writer in production) still got its evidence recorded as if
// this call had won. Both outcomes reach the same guard now: the
// completion side effects only run once FinishRun confirms wrote=true.
func TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) {
svc, eb := newTestServiceWithBus(t)
ctx := context.Background()
ensureTaskSessionMessagesTable(t, svc)
createTestAgent(t, svc, "ws-1", "worker-race")
taskID := createOfficeTask(t, svc, "ws-1", "worker-race")
if err := svc.QueueRun(
ctx, "worker-race", service.RunReasonTaskAssigned,
`{"task_id":"`+taskID+`"}`, "run-output-race",
); err != nil {
t.Fatalf("queue run: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim run: %v (run=%v)", err, run)
}
claimedAt := requireClaimedAt(t, run)
svc.ExecSQL(t, `
INSERT INTO task_session_messages (id, task_session_id, type, author_type, content, created_at)
VALUES ('m-race-1', 'sess-race', 'message', 'agent', 'should never be recorded', ?)
`, claimedAt.Add(5*time.Second))
// Force FinishRun's UPDATE to fail with a genuine error, matching
// TestSchedulerTick_AgentCompletedKeepsCheckoutWhenFinishRunFails'
// fault injection: a targeted fault (drop the column FinishRun sets)
// rather than a global read-only pragma.
svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at")
completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{
"task_id": taskID,
"session_id": "sess-race",
"agent_profile_id": "worker-race",
})
if pErr := eb.Publish(ctx, events.AgentCompleted, completed); pErr != nil {
t.Fatalf("publish completed: %v", pErr)
}
got := findRunByID(t, svc, "ws-1", run.ID)
if got.OutputSummary != "" {
t.Errorf("output_summary = %q, want empty — FinishRun never landed, so recordRunOutputSummary "+
"must not have run", got.OutputSummary)
}
if got.Status != service.RunStatusClaimed {
t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status)
}
runEvents, err := svc.ListRunEventsForTest(ctx, run.ID)
if err != nil {
t.Fatalf("list run events: %v", err)
}
for _, e := range runEvents {
if e.EventType == "complete" {
t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e)
}
}
}
// TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects
// is the taskless-path counterpart: refreshContinuationSummary and
// recordRunOutputSummary must not persist evidence for a completion whose
// terminal write never landed either.
func TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) {
svc := newTestService(t)
svc.SetSyncHandlers(true)
ctx := context.Background()
eb := bus.NewMemoryEventBus(logger.Default())
if err := svc.RegisterEventSubscribers(eb); err != nil {
t.Fatalf("register subscribers: %v", err)
}
agent := &models.AgentInstance{
ID: "coordinator-taskless-race-effects",
WorkspaceID: "ws-1",
Name: "coordinator-taskless-race-effects",
Role: models.AgentRoleCEO,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `
INSERT INTO runs (
id, agent_profile_id, reason, payload, status, coalesced_count,
context_snapshot, requested_at, claimed_at
) VALUES (
'run-taskless-race-1', ?, 'routine_trigger', '{}',
'claimed', 1, '{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
)
`, agent.ID)
svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at")
completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{
"agent_id": agent.ID,
"agent_profile_id": agent.ID,
"session_id": "sess-taskless-race",
})
if err := eb.Publish(ctx, events.AgentCompleted, completed); err != nil {
t.Fatalf("publish completed: %v", err)
}
got := findRunByID(t, svc, "ws-1", "run-taskless-race-1")
if got.Status != service.RunStatusClaimed {
t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status)
}
runEvents, err := svc.ListRunEventsForTest(ctx, "run-taskless-race-1")
if err != nil {
t.Fatalf("list run events: %v", err)
}
for _, e := range runEvents {
if e.EventType == "complete" {
t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e)
}
}
summary, err := svc.GetContinuationSummaryForTest(ctx, agent.ID, "")
if err == nil && summary != nil {
t.Fatalf("expected no continuation summary written when FinishRun's write failed; got %+v", summary)
}
}
@@ -200,7 +200,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) {
taskID := uuidish("task-fail-runner", i)
insertSyntheticTask(t, svc, taskID, "ws-1", "runner-pm4")
w := queueAndReadRun(t, svc, "runner-pm4", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure (runner) %d: %v", i, err)
}
}
@@ -208,7 +208,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) {
taskID := uuidish("task-fail-critic", i)
insertSyntheticTask(t, svc, taskID, "ws-1", "critic4")
w := queueAndReadRun(t, svc, "critic4", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure (critic) %d: %v", i, err)
}
}
@@ -33,18 +33,36 @@ const (
//
// No retry is scheduled — the user resolves via Resume session in the
// chat or Mark fixed in the inbox.
//
// Returns wrote=false when MarkRunFailed's guarded write was a no-op —
// the run reached a terminal state through another writer (e.g. a
// concurrent cancel) between the caller's read and this call — so
// callers know not to treat a cancelled/already-terminal run as a
// genuine agent failure (Review round 3, R3-1).
func (s *Service) HandleAgentFailure(
ctx context.Context,
run *models.Run,
errorMessage string,
) error {
if err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage); err != nil {
return fmt.Errorf("mark run failed: %w", err)
) (bool, error) {
wrote, err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage)
if err != nil {
return false, fmt.Errorf("mark run failed: %w", err)
}
if !wrote {
// Nothing to classify, release, or escalate: the row's real
// terminal state was written by someone else. Still make sure
// the agent isn't left stuck "working" from the launch.
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
return false, nil
}
// MarkRunFailed bypasses transitionRunTerminal (this is the office v1
// failure path, not FailRun), so the checkout release that lives there
// has to be duplicated here — otherwise every agent-error terminal
// transition leaks the task checkout the same way FinishRun used to.
// failure path, not FailRun), so the checkout release and terminal-shape
// count that live there have to be duplicated here — otherwise every
// agent-error terminal transition leaks the task checkout the same way
// FinishRun used to, and every genuine post-launch crash goes uncounted
// in office_loop_terminal_total. MarkRunFailed writes status='failed'
// with outcome left untouched (NULL for a launched run), matching FailRun.
s.recordTerminalShape(ctx, run, RunStatusFailed, nil)
s.releaseTaskCheckoutForRun(ctx, run)
// Leave "working" before the auto-pause decision below, not after: the
// reset is a working → idle CAS, so running it first lets a subsequent
@@ -78,7 +96,7 @@ func (s *Service) HandleAgentFailure(
}
}
return nil
return true, nil
}
// RecordAgentSuccess resets the consecutive-failure counter for the
@@ -5,15 +5,20 @@ import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// queueAndReadRun enqueues a run with a unique idempotency key
// (so multiple calls in one test don't coalesce) and reads it back.
// Mirrors the production path (QueueRun) without going through the
// claim step (which lives on the repository).
// (so multiple calls in one test don't coalesce), reads it back, and
// marks it claimed — the precondition every production caller of
// HandleAgentFailure actually has (resolveLifecycleRun resolves via
// GetClaimedRunByID before ever reaching it). Review round 3, R3-1
// guards MarkRunFailed's write to status = 'claimed', so a test that
// skipped this step would see every HandleAgentFailure call in this
// file silently no-op.
func queueAndReadRun(
t *testing.T, svc *service.Service, agentID, taskID string,
) *models.Run {
@@ -30,6 +35,9 @@ func queueAndReadRun(
}
for _, w := range rows {
if w.AgentProfileID == agentID && taskIDFromPayload(t, w.Payload) == taskID && w.Status != service.RunStatusFailed {
svc.ExecSQL(t, `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
time.Now().UTC(), w.ID)
w.Status = service.RunStatusClaimed
return w
}
}
@@ -94,7 +102,7 @@ func TestHandleAgentFailure_AutoPausesAtThreshold(t *testing.T) {
taskID := uuidish("task-pause", i)
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-pause")
w := queueAndReadRun(t, svc, "agent-pause", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure %d: %v", i, err)
}
}
@@ -128,7 +136,7 @@ func TestRecordAgentSuccess_ResetsCounter(t *testing.T) {
taskID := uuidish("task-succ", i)
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-success")
w := queueAndReadRun(t, svc, "agent-success", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure %d: %v", i, err)
}
}
@@ -172,7 +180,7 @@ func TestHandleAgentFailure_RespectsPerAgentThreshold(t *testing.T) {
taskID := "task-tight-1"
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-tight")
w := queueAndReadRun(t, svc, "agent-tight", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure: %v", err)
}
@@ -198,7 +206,7 @@ func autoPauseAgent(
taskID := agentID + "-task-" + uuidish("p", i)
insertSyntheticTask(t, svc, taskID, wsID, agentID)
w := queueAndReadRun(t, svc, agentID, taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure %d: %v", i, err)
}
}
@@ -328,7 +336,7 @@ func TestMarkAgentRunFailedFixed_LeavesInboxWhenRequeueFails(t *testing.T) {
taskID := "task-retry-after-queue-error"
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-task-retry")
run := queueAndReadRun(t, svc, "agent-task-retry", taskID)
if err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
t.Fatalf("handle failure: %v", err)
}
if err := svc.UpdateAgentStatusFields(ctx, "agent-task-retry",
@@ -358,7 +366,7 @@ func TestMarkAgentPausedFixed_UsesPauseSnapshot(t *testing.T) {
oldTaskID := "old-failure-task"
insertSyntheticTask(t, svc, oldTaskID, "ws-1", "agent-snapshot")
oldRun := queueAndReadRun(t, svc, "agent-snapshot", oldTaskID)
if err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil {
t.Fatalf("old failure: %v", err)
}
svc.RecordAgentSuccess(ctx, "agent-snapshot")
@@ -392,7 +400,7 @@ func TestMarkAgentPausedFixed_DiscardsRecoveryForReassignedTask(t *testing.T) {
reassignedTaskID := "reassigned-task"
insertSyntheticTask(t, svc, reassignedTaskID, "ws-1", "agent-reassigned-from")
failedRun := queueAndReadRun(t, svc, "agent-reassigned-from", reassignedTaskID)
if err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil {
t.Fatalf("handle failure: %v", err)
}
// Two more failures (on other tasks) to cross the default threshold
@@ -520,6 +528,35 @@ func TestMarkAgentPausedFixed_NonPausedAgentGuardAndRequeueError(t *testing.T) {
}
}
// HandleAgentFailure is Office's v1 post-launch failure path
// (event_subscribers.go's handleAgentFailed) and bypasses
// transitionRunTerminal's own counting, so it must record its own
// terminal shape or office_loop_terminal_total silently misses every
// genuine agent crash (Review round 1, R1-1).
func TestHandleAgentFailure_RecordsTerminalShape(t *testing.T) {
svc, _ := newTestServiceWithBus(t)
ctx := context.Background()
createTestAgent(t, svc, "ws-1", "agent-crash")
taskID := "task-crash"
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-crash")
run := queueAndReadRun(t, svc, "agent-crash", taskID)
svc.ExecSQL(t, `UPDATE runs SET session_id = ? WHERE id = ?`, "session-crash", run.ID)
run.SessionID = "session-crash"
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeLaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
t.Fatalf("handle failure: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("launched_failed delta = %d, want 1", after-before)
}
}
// Pins that reassigning a task auto-dismisses the per-task inbox
// entry for the OLD agent without resetting that agent's counter.
func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing.T) {
@@ -532,7 +569,7 @@ func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing
taskID := "task-reassign-1"
insertSyntheticTask(t, svc, taskID, "ws-1", "old-agent")
w := queueAndReadRun(t, svc, "old-agent", taskID)
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
t.Fatalf("handle failure: %v", err)
}
@@ -0,0 +1,71 @@
package service_test
import (
"context"
"testing"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged is the PR fixup
// round 2 regression test (CodeRabbit). The idle-skip FinishRun call site
// used to discard FinishRun's error (`wrote, _ := ...`), treating a genuine
// persistence failure exactly like a benign lost terminal-write race
// (wrote=false, err=nil) — silently, with no log line either way. A run
// stuck 'claimed' because the terminal write itself failed needs to be
// distinguishable in logs from one that lost a race to another writer, so
// operators (and ReapStaleCheckouts/RecoverStale, the actual recovery path)
// aren't flying blind on which case occurred.
func TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged(t *testing.T) {
core, logs := observer.New(zapcore.ErrorLevel)
log, err := logger.NewFromZap(zap.New(core))
if err != nil {
t.Fatalf("create observer logger: %v", err)
}
svc := newTestService(t, service.ServiceOptions{Logger: log})
ctx := context.Background()
// Reject the UPDATE FinishRun issues (status -> 'finished') so the
// idle-skip path's FinishRun call returns a genuine persistence error
// instead of a lost-race no-op.
svc.ExecSQL(t, `
CREATE TRIGGER reject_idle_skip_finish
BEFORE UPDATE OF status ON runs
WHEN NEW.status = 'finished'
BEGIN
SELECT RAISE(ABORT, 'finish run write rejected for test');
END;
`)
agent := &models.AgentInstance{
WorkspaceID: "ws-1",
Name: "idle-outcome-worker-err",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
service.RunSchedulerTick(svc, ctx)
if logs.FilterMessage("failed to finish idle-skipped run").Len() == 0 {
t.Fatal("expected the FinishRun persistence error to be logged, not silently swallowed")
}
run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonHeartbeat)
if run.Status != "claimed" {
t.Fatalf("status = %q, want claimed — the write was rejected, so the run must stay "+
"claimed for the stale-claim reaper to recover, not silently read as finished", run.Status)
}
}
@@ -0,0 +1,137 @@
package service
import (
"expvar"
"strings"
)
// Loop-liveness counters (REQ-OFFICE-LOOP-LIVENESS-003). Same
// "k1=v1;k2=v2" expvar.NewMap label convention as
// internal/office/scheduler/metrics_vars.go and
// internal/orchestrator/office_stall_metrics.go, so a Prometheus
// translation layer can split on the same delimiters everywhere.
//
// AC-003.9's two increment rules: the six counters naming a persisted
// change (trigger_claimed, routine_run, wakeup_created, run_claimed,
// launch, terminal) increment at the persisting site after the write
// succeeds. The five naming an event that persists nothing (cron_tick,
// launch_without_session, session_persist_failed,
// last_run_at_write_failed, liveness_degraded) increment at the
// observing site regardless of any write outcome.
var (
loopCronTickTotal = expvar.NewMap("office_loop_cron_tick_total")
loopTriggerClaimedTotal = expvar.NewMap("office_loop_trigger_claimed_total")
loopRoutineRunTotal = expvar.NewMap("office_loop_routine_run_total")
loopWakeupCreatedTotal = expvar.NewMap("office_loop_wakeup_created_total")
loopRunClaimedTotal = expvar.NewMap("office_loop_run_claimed_total")
loopLaunchTotal = expvar.NewMap("office_loop_launch_total")
loopLaunchWithoutSessionTotal = expvar.NewMap("office_loop_launch_without_session_total")
loopSessionPersistFailedTotal = expvar.NewMap("office_loop_session_persist_failed_total")
loopTerminalTotal = expvar.NewMap("office_loop_terminal_total")
loopLastRunAtWriteFailedTotal = expvar.NewMap("office_loop_last_run_at_write_failed_total")
loopLivenessDegradedTotal = expvar.NewMap("office_loop_liveness_degraded_total")
// Instants, not counters — see LoopMetricLabel's callers below.
// Both are monotonic (never decrease), unlike the drifting gauges
// metrics_vars.go warns against: without them a flat counter is
// ambiguous between a restarted process and a stopped loop
// (AC-003.6).
loopCronTickAt = expvar.NewString("office_loop_cron_tick_at")
loopProcessStartedAt = expvar.NewString("office_loop_process_started_at")
)
// LoopUnattributedWorkspace is the explicit label used when an event's
// workspace cannot be resolved (AC-003.8) — never dropped, never
// guessed, and never merged into a real workspace's totals.
const LoopUnattributedWorkspace = "_unattributed"
// LoopMetricLabel builds a "k1=v1;k2=v2;..." expvar map key. pairs must
// be an even-length (key, value) sequence.
func LoopMetricLabel(pairs ...string) string {
if len(pairs)%2 != 0 {
return ""
}
parts := make([]string, 0, len(pairs)/2)
for i := 0; i < len(pairs); i += 2 {
parts = append(parts, pairs[i]+"="+pairs[i+1])
}
return strings.Join(parts, ";")
}
// IncLoopCronTick counts one cron loop tick evaluation and publishes
// the tick instant (AC-003.1, AC-003.6).
func IncLoopCronTick(atRFC3339 string) {
loopCronTickTotal.Add("", 1)
loopCronTickAt.Set(atRFC3339)
}
// RecordLoopProcessStarted publishes the process-start instant once,
// at boot (AC-003.6).
func RecordLoopProcessStarted(atRFC3339 string) {
loopProcessStartedAt.Set(atRFC3339)
}
// IncLoopTriggerClaimed counts one eligible trigger claimed for firing.
func IncLoopTriggerClaimed(workspaceID string) {
loopTriggerClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopRoutineRun counts one routine run created, by (source,
// disposition) so a coalesced, a skipped, and a launched fire are
// separately countable (AC-003.2).
func IncLoopRoutineRun(workspaceID, source, disposition string) {
loopRoutineRunTotal.Add(
LoopMetricLabel("workspace", workspaceID, "source", source, "disposition", disposition), 1)
}
// IncLoopWakeupCreated counts one wakeup request created.
func IncLoopWakeupCreated(workspaceID, source string) {
loopWakeupCreatedTotal.Add(LoopMetricLabel("workspace", workspaceID, "source", source), 1)
}
// IncLoopRunClaimed counts one run claimed off the queue.
func IncLoopRunClaimed(workspaceID string) {
loopRunClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopLaunch counts one agent launch attempt that reached the
// adapter successfully.
func IncLoopLaunch(workspaceID string) {
loopLaunchTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopLaunchWithoutSession counts a launch that reported success but
// yielded no session id (AC-002.8).
func IncLoopLaunchWithoutSession(workspaceID string) {
loopLaunchWithoutSessionTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopSessionPersistFailed counts a launch that yielded a session id
// which then failed to persist (AC-002.11) — distinct from
// IncLoopLaunchWithoutSession because the underlying rows are otherwise
// byte-identical.
func IncLoopSessionPersistFailed(workspaceID string) {
loopSessionPersistFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopTerminal counts one terminal run transition by its classified
// shape. Counts transitions, not entities: a run that reaches terminal
// twice (a requeue after a post-start fallback) contributes two events.
func IncLoopTerminal(workspaceID, shape string) {
loopTerminalTotal.Add(LoopMetricLabel("workspace", workspaceID, "shape", shape), 1)
}
// IncLoopLastRunAtWriteFailed counts a TouchRoutineLastRun write that
// returned an error (AC-001.7). The dispatch still completes — a
// routine that fired must not read as having failed to fire because a
// bookkeeping write lost a lock.
func IncLoopLastRunAtWriteFailed(workspaceID string) {
loopLastRunAtWriteFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
}
// IncLoopLivenessDegraded counts one /loop-health read that failed
// loudly, labelled by which input could not be read (AC-004.10,
// "Failure" in the design).
func IncLoopLivenessDegraded(workspaceID, reason string) {
loopLivenessDegradedTotal.Add(LoopMetricLabel("workspace", workspaceID, "reason", reason), 1)
}
+26 -4
View File
@@ -95,15 +95,23 @@ func isRetryStale(run *models.Run) (bool, string) {
return false, ""
}
// cancelRetry cancels a run that is too stale to retry.
// cancelRetry cancels a run that is too stale to retry. Terminal-shape
// recording and the OfficeRunProcessed broadcast only happen when the
// cancel actually applied: a run another writer already finished or
// failed must not be reported to subscribers as cancelled.
func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason string) error {
s.logger.Info("cancelling stale run retry",
zap.String("run_id", run.ID),
zap.String("reason", reason))
if err := s.repo.CancelRun(ctx, run.ID, reason); err != nil {
cancelled, err := s.repo.CancelRun(ctx, run.ID, reason)
if err != nil {
return err
}
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
if !cancelled {
return nil
}
s.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
s.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
return nil
}
@@ -113,10 +121,17 @@ func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason strin
func (s *Service) escalateFailure(
ctx context.Context, run *models.Run, runErr error,
) error {
if err := s.FailRun(ctx, run.ID); err != nil {
wrote, err := s.FailRun(ctx, run.ID)
if err != nil {
return fmt.Errorf("fail run: %w", err)
}
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
if !wrote {
// Already terminal via another writer (e.g. a concurrent cancel)
// between the retry decision and this write — nothing left to
// escalate (Review round 3, R3-1).
return nil
}
agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID)
if err != nil {
@@ -158,10 +173,17 @@ func (s *Service) failRunNoEscalation(
ctx context.Context, run *models.Run, agent *models.AgentInstance,
cause budgetDeferralCause, policyID string,
) error {
if err := s.FailRun(ctx, run.ID); err != nil {
wrote, err := s.FailRun(ctx, run.ID)
if err != nil {
return fmt.Errorf("fail run: %w", err)
}
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
if !wrote {
// Already terminal via another writer (e.g. a concurrent cancel)
// between the budget decision and this write — nothing left to
// report (Review round 3, R3-1).
return nil
}
fields := map[string]string{activityFieldCeiling: ceilingNotDetermined}
if policyID != "" {
@@ -25,5 +25,10 @@ func (s *Service) CancelPendingRetriesForTask(ctx context.Context, taskID string
zap.String("task_id", taskID),
zap.Int("count", len(ids)))
return s.repo.BulkCancelRuns(ctx, ids, "task_reassigned")
cancelled, err := s.repo.BulkCancelRuns(ctx, ids, "task_reassigned")
if err != nil {
return err
}
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
return nil
}
@@ -0,0 +1,149 @@
package service_test
import (
"context"
"errors"
"fmt"
"testing"
"time"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// A stale-retry cancel that loses the terminal-transition race (another
// writer already finished the run) must not tell subscribers the run was
// cancelled: the run's real terminal state is finished, and a status
// mismatch between the persisted row and the broadcast event misleads
// every WS-driven UI reading it.
func TestHandleRunFailure_LostCancelRaceDoesNotPublishFalseCancelled(t *testing.T) {
svc, eb := newTestServiceWithBus(t)
ctx := context.Background()
agent := makeAgent("worker-stale-retry-race", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry-race"); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour)
svc.ExecSQL(t,
`UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`,
staleRequestedAt.Add(-time.Hour).Format(time.RFC3339))
svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID)
run.RequestedAt = staleRequestedAt
// Another writer wins the terminal-transition race before the stale
// cancel below runs: the row is already finished, so CancelRun's
// guarded UPDATE (status IN ('queued','claimed')) will match it.
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish (simulating the winning writer): %v", err)
}
got := make(chan *bus.Event, 1)
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
got <- e
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
netErr := errors.New("connection timed out")
if err := svc.HandleRunFailure(ctx, run, netErr); err != nil {
t.Fatalf("handle failure: %v", err)
}
select {
case e := <-got:
t.Fatalf("expected no OfficeRunProcessed publish for a lost cancel race; got %+v", e)
default:
}
reqs, err := svc.ListRuns(ctx, "ws-1")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(reqs) != 1 || reqs[0].Status != service.RunStatusFinished {
t.Fatalf("run status = %+v, want the winning writer's finished state to survive", reqs)
}
}
// The stale-claim cancel path (evaluateRunStaleness's workflow_step_changed
// branch, reached from the scheduler tick) has the same obligation as the
// stale-retry path above: a lost cancel race must not publish a false
// "cancelled" event or log a false cancel activity entry.
func TestSchedulerIntegration_CancelStaleRunLostRaceDoesNotPublishOrLog(t *testing.T) {
mock := &mockTaskStarter{}
eb := bus.NewMemoryEventBus(logger.Default())
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock, EventBus: eb})
ctx := context.Background()
agent := makeAgent("worker-moved-step-race", models.AgentRoleWorker)
agent.ExecutorPreference = `{"type":"local_pc"}`
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks
(id, workspace_id, workflow_step_id, title, created_at, updated_at)
VALUES ('task-moved-step-race', 'ws-1', 'step-current', 'Moved task',
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"task-moved-step-race","workflow_step_id":"step-old"}`, ""); err != nil {
t.Fatalf("queue run: %v", err)
}
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil || len(runs) != 1 {
t.Fatalf("list runs: %v (%d)", err, len(runs))
}
runID := runs[0].ID
// Simulate another writer winning the race for this exact row: the
// tick's own CancelRun write below will match zero rows and return
// no error, the same shape a genuinely lost race produces.
svc.ExecSQL(t, fmt.Sprintf(`
CREATE TRIGGER ignore_stale_cancel_write
BEFORE UPDATE OF status ON runs
WHEN NEW.status = 'cancelled' AND OLD.id = '%s'
BEGIN
SELECT RAISE(IGNORE);
END;
`, runID))
got := make(chan *bus.Event, 1)
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
got <- e
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
defer func() { _ = sub.Unsubscribe() }()
service.RunSchedulerTick(svc, ctx)
select {
case e := <-got:
t.Fatalf("expected no OfficeRunProcessed publish for a lost stale-cancel race; got %+v", e)
default:
}
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_cancelled_stale", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_cancelled_stale activity entries = %d, want 0 for a lost cancel race", len(entries))
}
}
@@ -0,0 +1,56 @@
package service_test
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// TestCancelPendingRetriesForTask_RecordsTerminalShape is the Testing
// round 3 regression test. CancelPendingRetriesForTask calls
// repo.BulkCancelRuns directly, which bypassed office_loop_terminal_total
// entirely — the same bypass class Review round 1 (R1-1) fixed for
// HandleAgentFailure and the single-run CancelRun call sites, just missed
// here. A pending retry never launched, so cancelling it must classify
// as unlaunched_failed.
func TestCancelPendingRetriesForTask_RecordsTerminalShape(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := makeAgent("retry-cancel-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
taskID := "task-retry-cancel"
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
VALUES (?, ?, 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
taskID, agent.WorkspaceID)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"`+taskID+`"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
runs, err := svc.ListRuns(ctx, agent.WorkspaceID)
if err != nil || len(runs) != 1 {
t.Fatalf("list runs: runs=%v err=%v", runs, err)
}
run := runs[0]
if err := svc.RepoForTest().ScheduleRetry(ctx, run.ID, run.RequestedAt.Add(time.Hour), 1); err != nil {
t.Fatalf("schedule retry: %v", err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if err := svc.CancelPendingRetriesForTask(ctx, taskID); err != nil {
t.Fatalf("cancel pending retries: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
@@ -0,0 +1,74 @@
package service_test
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// cancelRetry (the 24h-stale-retry path reached through HandleRunFailure)
// bypasses transitionRunTerminal's own counting the same way
// HandleAgentFailure and cancelStaleRun do, so it must record its own
// terminal shape or office_loop_terminal_total silently misses every
// abandoned-retry cancellation (Review round 1, R1-1). This is a distinct
// trigger condition from cancelStaleRun's workflow_step_changed path,
// covered separately in TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape.
func TestHandleRunFailure_CancelsStaleRetryAndRecordsTerminalShape(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := makeAgent("worker-stale-retry", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry"); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
// Backdate past retryMaxAge (24h) so isRetryStale routes this failure
// to cancelRetry instead of a normal backoff reschedule. Also push the
// loop-liveness activation instant back further still, so the
// backdated run doesn't fall outside the activation window and
// classify as pre_activation instead of the shape under test.
staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour)
svc.ExecSQL(t,
`UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`,
staleRequestedAt.Add(-time.Hour).Format(time.RFC3339))
svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID)
run.RequestedAt = staleRequestedAt
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
netErr := errors.New("connection timed out")
if err := svc.HandleRunFailure(ctx, run, netErr); err != nil {
t.Fatalf("handle failure: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
reqs, err := svc.ListRuns(ctx, "ws-1")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(reqs) != 1 {
t.Fatalf("expected 1 run, got %d", len(reqs))
}
if reqs[0].Status != service.RunStatusCancelled {
t.Errorf("status = %q, want cancelled", reqs[0].Status)
}
if reqs[0].CancelReason == nil || *reqs[0].CancelReason != "run_too_old" {
t.Errorf("cancel_reason = %v, want run_too_old", reqs[0].CancelReason)
}
}
@@ -0,0 +1,81 @@
package service_test
import (
"context"
"expvar"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
func runClaimedExpvarInt(t *testing.T, key string) int64 {
t.Helper()
v := expvar.Get("office_loop_run_claimed_total")
if v == nil {
t.Fatal("expvar map office_loop_run_claimed_total not registered")
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatal("office_loop_run_claimed_total is not a Map")
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("office_loop_run_claimed_total[%q] is not an Int", key)
}
return iv.Value()
}
// REQ-OFFICE-LOOP-LIVENESS-003: ClaimNextRun counts a successfully
// claimed run labelled by the claiming agent's workspace.
func TestClaimNextRun_IncrementsRunClaimedCounter(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := makeAgent("run-claimed-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
t.Fatalf("queue: %v", err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID)
before := runClaimedExpvarInt(t, key)
req, err := svc.ClaimNextRun(ctx)
if err != nil || req == nil {
t.Fatalf("claim: req=%v err=%v", req, err)
}
after := runClaimedExpvarInt(t, key)
if after != before+1 {
t.Fatalf("run_claimed delta = %d, want 1", after-before)
}
}
// ClaimNextRun with no queued work never increments the counter.
func TestClaimNextRun_EmptyQueueDoesNotIncrementRunClaimedCounter(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
key := service.LoopMetricLabel("workspace", "ws-1")
before := runClaimedExpvarInt(t, key)
req, err := svc.ClaimNextRun(ctx)
if err != nil {
t.Fatalf("claim: %v", err)
}
if req != nil {
t.Fatalf("expected nil run, got %+v", req)
}
after := runClaimedExpvarInt(t, key)
if after != before {
t.Fatalf("run_claimed delta = %d, want 0", after-before)
}
}
@@ -314,7 +314,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) {
}
defer func() { _ = sub.Unsubscribe() }()
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -375,7 +375,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T)
}
defer func() { _ = sub.Unsubscribe() }()
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -427,7 +427,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) {
}
defer func() { _ = sub.Unsubscribe() }()
if err := svc.FailRun(ctx, run.ID); err != nil {
if _, err := svc.FailRun(ctx, run.ID); err != nil {
t.Fatalf("fail: %v", err)
}
@@ -229,7 +229,10 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
si.logger.Info("run skipped (agent not active)",
zap.String("run_id", runID),
zap.String("agent_status", string(agent.Status)))
_ = si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive)
if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive); err != nil {
si.logger.Error("failed to finish agent-inactive run",
zap.String("run_id", runID), zap.Error(err))
}
return
}
@@ -251,6 +254,19 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
si.logger.Info("run skipped (no actionable tasks)",
zap.String("run_id", runID),
zap.String("agent", agent.Name))
si.svc.clearAgentWorking(ctx, agent.ID, runID)
wrote, err := si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped)
if err != nil {
si.logger.Error("failed to finish idle-skipped run",
zap.String("run_id", runID), zap.Error(err))
return
}
if !wrote {
// Another writer already moved the run out of claimed; it did
// not actually end via an idle skip, so there is nothing to
// log.
return
}
si.svc.LogActivityWithRun(ctx, agent.WorkspaceID,
"scheduler", "office-scheduler",
"run_idle_skipped", "run", runID,
@@ -258,8 +274,6 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
"agent": agent.Name,
"agent_id": agent.ID,
}), runID, "")
si.svc.clearAgentWorking(ctx, agent.ID, runID)
_ = si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped)
return
}
@@ -547,7 +561,10 @@ func (si *SchedulerIntegration) isTaskTreeGated(ctx context.Context, runID, task
zap.String("task_id", taskID),
zap.String("hold_id", hold.ID),
zap.String("mode", hold.Mode))
_ = si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld)
if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld); err != nil {
si.logger.Error("failed to finish tree-held run",
zap.String("run_id", runID), zap.String("task_id", taskID), zap.Error(err))
}
return true
}
@@ -606,12 +623,19 @@ func (si *SchedulerIntegration) launchAgent(
return launched
}
var err error
if starter, ok := si.svc.taskStarter.(TaskStarterWithLaunchContext); ok {
var sessionID string
switch starter := si.svc.taskStarter.(type) {
case TaskStarterWithLaunchContextSession:
sessionID, err = starter.StartTaskWithLaunchContextReturningSession(ctx, taskID, launch.ProfileID, launch)
case TaskStarterWithLaunchContext:
err = starter.StartTaskWithLaunchContext(ctx, taskID, launch.ProfileID, launch)
} else if starter, ok := si.svc.taskStarter.(TaskStarterWithEnv); ok {
case TaskStarterWithSession:
sessionID, err = starter.StartTaskWithEnvReturningSession(ctx, taskID, launch.ProfileID, "", "", "",
launch.Prompt, "", false, nil, launch.Env)
case TaskStarterWithEnv:
err = starter.StartTaskWithEnv(ctx, taskID, launch.ProfileID, "", "", "",
launch.Prompt, "", false, nil, launch.Env)
} else {
default:
err = si.svc.taskStarter.StartTask(ctx, taskID, launch.ProfileID, "", "", "",
launch.Prompt, "", false, nil)
}
@@ -626,9 +650,37 @@ func (si *SchedulerIntegration) launchAgent(
_ = si.svc.HandleRunFailure(ctx, run, err)
return false
}
IncLoopLaunch(agent.WorkspaceID)
si.persistLaunchedSession(ctx, runID, agent.WorkspaceID, sessionID)
return true
}
// persistLaunchedSession stores the session id a successful direct
// launch produced (AC-OFFICE-LOOP-LIVENESS-002.7) — the scheduler-side
// counterpart to scheduler.SchedulerService.persistLaunchedSession for
// the routed launch path. See that method's doc comment for the
// without-session / persist-failed counter semantics (AC-002.8, .11).
func (si *SchedulerIntegration) persistLaunchedSession(
ctx context.Context, runID, workspaceID, sessionID string,
) {
if sessionID == "" {
IncLoopLaunchWithoutSession(workspaceID)
return
}
wrote, err := si.svc.repo.SetRunSessionID(ctx, runID, sessionID)
if err != nil {
si.logger.Warn("persist launched session id failed",
zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err))
IncLoopSessionPersistFailed(workspaceID)
return
}
if !wrote {
si.logger.Warn("persist launched session id matched no row",
zap.String("run_id", runID), zap.String("session_id", sessionID))
IncLoopSessionPersistFailed(workspaceID)
}
}
// failTasklessRun terminally fails a run that launchAgent determined has
// no task_id. This is a scheduler capability gap, not an agent failure, so
// it must NOT go through HandleAgentFailure's consecutive-failure/
@@ -665,16 +717,25 @@ func (si *SchedulerIntegration) launchAgent(
func (si *SchedulerIntegration) failTasklessRun(
ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string,
) {
si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{
"phase": "scheduler.launch",
"error_message": msg,
})
si.releaseCheckoutIfNeeded(ctx, run)
if err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg); err != nil {
wrote, err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg)
if err != nil {
si.logger.Error("failed to mark taskless run as failed",
zap.String("run_id", run.ID), zap.Error(err))
return // don't publish a terminal event when persistence failed
}
if !wrote {
// Already terminal via another writer (e.g. a concurrent cancel)
// between the caller's read and this write — nothing to classify,
// publish, or log as a scheduler-launch error on this run's
// timeline (Review round 3, R3-1).
return
}
si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{
"phase": "scheduler.launch",
"error_message": msg,
})
si.svc.recordTerminalShape(ctx, run, RunStatusFailed, nil)
run.ErrorMessage = msg
// Settle the auto-dismiss decision before publishing: the frontend
@@ -728,11 +789,17 @@ func (si *SchedulerIntegration) failUnlaunchableRun(
"error_message": msg,
})
si.releaseCheckoutIfNeeded(ctx, run)
if err := si.svc.HandleAgentFailure(ctx, run, msg); err != nil {
wrote, err := si.svc.HandleAgentFailure(ctx, run, msg)
if err != nil {
si.logger.Error("failed to handle agent failure for unlaunchable run",
zap.String("run_id", run.ID), zap.Error(err))
return // don't publish a terminal event when persistence failed
}
if !wrote {
// Already terminal via another writer — nothing to publish
// (Review round 3, R3-1).
return
}
run.ErrorMessage = msg
si.svc.publishRunProcessedForWorkspace(ctx, run.ID, RunStatusFailed, run, agent.WorkspaceID)
}
@@ -44,7 +44,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) {
}
// Finish the run.
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -94,6 +94,40 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) {
}
}
// cancelStaleRun bypasses transitionRunTerminal's own counting the same
// way HandleAgentFailure does, so it must record its own terminal shape
// or office_loop_terminal_total silently misses every stale-claim
// cancellation (Review round 1, R1-1).
func TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := makeAgent("worker-moved-step-shape", models.AgentRoleWorker)
agent.ExecutorPreference = `{"type":"local_pc"}`
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks
(id, workspace_id, workflow_step_id, title, created_at, updated_at)
VALUES ('task-moved-step-shape', 'ws-1', 'step-current', 'Moved task',
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"task-moved-step-shape","workflow_step_id":"step-old"}`, ""); err != nil {
t.Fatalf("queue run: %v", err)
}
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
service.RunSchedulerTick(svc, ctx)
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
@@ -236,7 +270,7 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) {
}
// Finish the first run.
if err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil {
if _, err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -314,7 +348,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) {
t.Errorf("prompt should contain %q, got: %s", tt.contains, prompt)
}
_ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed)
_, _ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed)
})
}
}
@@ -0,0 +1,126 @@
package service_test
import (
"context"
"fmt"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// ignoreFinishedWrite installs a trigger that makes the given run's
// terminal-transition write to status='finished' a no-op, simulating
// another writer having already claimed the terminal-transition race.
func ignoreFinishedWrite(t *testing.T, svc *service.Service, runID string) {
t.Helper()
svc.ExecSQL(t, fmt.Sprintf(`
CREATE TRIGGER ignore_finished_write
BEFORE UPDATE OF status ON runs
WHEN NEW.status = 'finished' AND OLD.id = '%s'
BEGIN
SELECT RAISE(IGNORE);
END;
`, runID))
}
// A heartbeat run that loses the idle-skip terminal-transition race (another
// writer already moved it out of 'claimed') must not log a false
// run_idle_skipped activity entry: the run's real terminal state came from
// whichever writer actually won, and a stray activity entry misreports why
// it ended.
func TestSchedulerIntegration_IdleSkipLostRaceDoesNotLogFalseActivity(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
WorkspaceID: "ws-1",
Name: "idle-worker-race",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
// Worker defaults to skip_idle_runs=true, no tasks assigned.
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil || len(runs) != 1 {
t.Fatalf("list runs: %v (%d)", err, len(runs))
}
ignoreFinishedWrite(t, svc, runs[0].ID)
service.RunSchedulerTick(svc, ctx)
if mock.callCount() != 0 {
t.Errorf("expected 0 StartTask calls, got %d", mock.callCount())
}
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_idle_skipped", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_idle_skipped activity entries = %d, want 0 for a lost terminal-transition race", len(entries))
}
}
// A run that loses the budget-blocked terminal-transition race has the same
// obligation: no false run_budget_blocked activity entry.
func TestSchedulerIntegration_BudgetBlockedLostRaceDoesNotLogFalseActivity(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := &models.AgentInstance{
WorkspaceID: "ws-1",
Name: "budget-worker-race",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
policy := &models.BudgetPolicy{
WorkspaceID: "ws-1",
ScopeType: "agent",
ScopeID: agent.ID,
LimitSubcents: 100,
Period: "monthly",
AlertThresholdPct: 80,
ActionOnExceed: "pause_agent",
}
if err := svc.CreateBudgetPolicy(ctx, policy); err != nil {
t.Fatalf("create policy: %v", err)
}
insertTestTask(t, svc, "task-budget-race", "ws-1")
insertTestCostEvent(t, svc, agent.ID, "task-budget-race", int64(600))
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"task-budget-race"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil || len(runs) != 1 {
t.Fatalf("list runs: %v (%d)", err, len(runs))
}
ignoreFinishedWrite(t, svc, runs[0].ID)
service.RunSchedulerTick(svc, ctx)
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_budget_blocked", 10)
if err != nil {
t.Fatalf("list activity: %v", err)
}
if len(entries) != 0 {
t.Fatalf("run_budget_blocked activity entries = %d, want 0 for a lost terminal-transition race", len(entries))
}
}
@@ -12,6 +12,7 @@ import (
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/runs/commentkeys"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
)
// ClaimNextRun atomically claims the next eligible run from the queue.
@@ -28,6 +29,11 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) {
zap.String("id", req.ID),
zap.String("agent", req.AgentProfileID),
zap.String("reason", req.Reason))
workspaceID := LoopUnattributedWorkspace
if agent, err := s.GetAgentFromConfig(ctx, req.AgentProfileID); err == nil && agent != nil {
workspaceID = agent.WorkspaceID
}
IncLoopRunClaimed(workspaceID)
return req, nil
}
@@ -37,7 +43,12 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) {
// an OfficeRunProcessed bus event. The run row is fetched first so the
// published payload carries enough context (agent, task, comment, reason)
// for downstream WS consumers to scope updates.
func (s *Service) FinishRun(ctx context.Context, id, outcome string) error {
//
// Returns wrote=false when the guarded write was a no-op — the run was
// no longer claimed (already terminal via a concurrent writer, e.g. a
// cancel) by the time this statement ran — so there is nothing to
// classify or publish (Review round 3, R3-1).
func (s *Service) FinishRun(ctx context.Context, id, outcome string) (bool, error) {
return s.transitionRunTerminal(ctx, id, RunStatusFinished, &outcome)
}
@@ -45,17 +56,26 @@ func (s *Service) FinishRun(ctx context.Context, id, outcome string) error {
// OfficeRunProcessed event. outcome is written as NULL: a failed run is
// bucketed by RunCountsByDayForAgent on status alone and never reaches the
// outcome-derived buckets, so no value from the five-value vocabulary is
// invented for it. See FinishRun for the lifecycle contract.
func (s *Service) FailRun(ctx context.Context, id string) error {
// invented for it. See FinishRun for the lifecycle contract and the
// wrote=false no-op case.
func (s *Service) FailRun(ctx context.Context, id string) (bool, error) {
return s.transitionRunTerminal(ctx, id, RunStatusFailed, nil)
}
// transitionRunTerminal updates the run row to the given terminal status
// and outcome (nil writes SQL NULL) and emits OfficeRunProcessed.
// Pre-fetching the run keeps the published payload self-contained even
// when the caller doesn't hold a reference to the model. Publish errors
// are logged at debug and swallowed; persistence errors are returned to
// the caller.
// FinishRun returns the row as it stands right after that write (via
// RETURNING), so the published payload and the terminal-shape
// classification both read the same statement that persisted the change —
// neither depends on a separate read succeeding independently of it.
// Publish errors are logged at debug and swallowed; persistence errors are
// returned to the caller.
//
// FinishRun's guarded write (status = 'claimed') returns a nil run when
// another writer already moved the row to a different terminal state
// between the caller's read and this statement — nothing to classify or
// publish in that case, so this returns wrote=false without touching
// recordTerminalShape or publishRunProcessed (Review round 3, R3-1).
//
// Deliberately does NOT release the task checkout: transitionRunTerminal
// is reached by every terminal run, including ones that never held the
@@ -67,24 +87,68 @@ func (s *Service) FailRun(ctx context.Context, id string) error {
// for the SAME agent + task races a first run that is genuinely still
// executing and holds the checkout — that second run's release matches
// the first run's own checkout_agent_id and steals its own live lock out
// from under it (Review round 3, BLOCKING FINDING 1). Callers that KNOW
// their run actually held the checkout call releaseTaskCheckoutForRun
// explicitly instead: handleAgentCompleted / handleTasklessAgentCompleted
// (event_subscribers.go) for the launched-run completion path, and
// HandleAgentFailure (failure.go) for the launched-run failure path.
func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) error {
run, getErr := s.repo.GetRunByID(ctx, id)
if getErr != nil && !errors.Is(getErr, sql.ErrNoRows) {
s.logger.Debug("get run for terminal transition failed",
zap.String("run_id", id),
zap.String("status", status),
zap.Error(getErr))
// from under it. Callers that KNOW their run actually held the checkout
// call releaseTaskCheckoutForRun explicitly instead, and only when this
// method reports wrote=true: handleAgentCompleted /
// handleTasklessAgentCompleted (event_subscribers.go) for the
// launched-run completion path, and HandleAgentFailure (failure.go) for
// the launched-run failure path.
func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) (bool, error) {
run, err := s.repo.FinishRun(ctx, id, status, outcome)
if err != nil {
return false, err
}
if err := s.repo.FinishRun(ctx, id, status, outcome); err != nil {
return err
if run == nil {
return false, nil
}
s.recordTerminalShape(ctx, run, status, outcome)
s.publishRunProcessed(ctx, id, status, run)
return nil
return true, nil
}
// recordTerminalShape classifies the just-persisted terminal transition
// (REQ-OFFICE-LOOP-LIVENESS-005) and increments office_loop_terminal_total
// by the resulting shape. run is nil when transitionRunTerminal's
// pre-fetch failed; there is nothing to classify in that case. The
// workspace label falls back to LoopUnattributedWorkspace when the
// owning agent can't be resolved, matching AC-003.8 — never dropped,
// never guessed into a real workspace's totals.
func (s *Service) recordTerminalShape(ctx context.Context, run *models.Run, status string, outcome *string) {
if run == nil {
return
}
activationInstant, activationPublished := s.repo.LoopLivenessActivation()
shape := ClassifyTerminalRun(status, outcome, run.SessionID, run.RequestedAt, activationInstant, activationPublished)
workspaceID := LoopUnattributedWorkspace
if agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID); err == nil && agent != nil {
workspaceID = agent.WorkspaceID
}
IncLoopTerminal(workspaceID, string(shape))
}
// recordTerminalShapesForCancelledRuns classifies and counts one
// cancelled-transition shape per row a bulk cancel (CancelRunsForTasks,
// BulkCancelRuns) actually persisted — a bulk write covers rows whose
// session_id can differ per row (a claimed run may have already launched),
// so each row is classified individually rather than assuming one shape
// for the whole batch.
func (s *Service) recordTerminalShapesForCancelledRuns(ctx context.Context, cancelled []runssqlite.CancelledRun) {
for _, row := range cancelled {
s.recordTerminalShape(ctx, &models.Run{
AgentProfileID: row.AgentProfileID,
SessionID: row.SessionID,
RequestedAt: row.RequestedAt,
}, RunStatusCancelled, nil)
}
}
// RecordCancelledRunTerminalShapes is recordTerminalShapesForCancelledRuns,
// exported for the dashboard package's TerminalShapeRecorder seam — a
// dashboard-driven cancellation (displaced participant) reaches the same
// counter every other cancellation path reaches, without a second,
// divergence-prone classification implementation in dashboard.
func (s *Service) RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun) {
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
}
// releaseTaskCheckoutForRun releases the run's task checkout. Call this
@@ -105,7 +105,7 @@ func TestFinishAndFailRun(t *testing.T) {
t.Fatal("expected claimed run")
}
if err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil {
if _, err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
@@ -141,7 +141,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) {
t.Fatal("expected claimed run")
}
if err := svc.FailRun(ctx, req.ID); err != nil {
if _, err := svc.FailRun(ctx, req.ID); err != nil {
t.Fatalf("fail: %v", err)
}
@@ -157,14 +157,19 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) {
}
}
// TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome is Review
// round 2's lower-priority TEST-002 (spec:2130-2132): two terminal callers
// reaching the same run row must never leave status and outcome
// disagreeing, because transitionRunTerminal's underlying FinishRun writes
// both columns in a single UPDATE statement. This drives both orderings —
// FinishRun-then-FailRun and FailRun-then-FinishRun — and asserts the row
// always ends up wholly consistent with whichever call went last.
func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) {
// TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome supersedes
// Review round 2's lower-priority TEST-002 (spec:2130-2132) under Review
// round 3's R3-1 guard: transitionRunTerminal's underlying FinishRun now
// only writes while the row is still status = 'claimed' (the same guard
// that stops a stale write from clobbering a concurrent cancel), so two
// terminal callers reaching the same row no longer race to "whichever
// wrote last" — the first write reaches a real terminal state and every
// later one is a no-op that leaves status and outcome exactly as the
// first write left them. This drives both orderings — FinishRun-then-
// FailRun and FailRun-then-FinishRun — and asserts the row stays
// consistent with whichever call went FIRST, and that the second call
// reports wrote=false.
func TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome(t *testing.T) {
ctx := context.Background()
queueAndClaim := func(t *testing.T, svc *service.Service) string {
@@ -186,36 +191,46 @@ func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) {
t.Run("FailRun after FinishRun", func(t *testing.T) {
svc := newTestService(t)
id := queueAndClaim(t, svc)
if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
wrote, err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed)
if err != nil || !wrote {
t.Fatalf("finish: wrote=%v err=%v, want wrote=true", wrote, err)
}
if err := svc.FailRun(ctx, id); err != nil {
t.Fatalf("fail: %v", err)
}
got, err := svc.GetRun(ctx, id)
wrote, err = svc.FailRun(ctx, id)
if err != nil {
t.Fatalf("get run: %v", err)
}
if got.Status != models.RunStatusFailed || got.Outcome != nil {
t.Fatalf("row = {status: %q, outcome: %v}, want the later FailRun write: {failed, nil}", got.Status, got.Outcome)
}
})
t.Run("FinishRun after FailRun", func(t *testing.T) {
svc := newTestService(t)
id := queueAndClaim(t, svc)
if err := svc.FailRun(ctx, id); err != nil {
t.Fatalf("fail: %v", err)
}
if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
if wrote {
t.Error("fail: wrote = true, want false (run was already finished, not claimed)")
}
got, err := svc.GetRun(ctx, id)
if err != nil {
t.Fatalf("get run: %v", err)
}
if got.Status != models.RunStatusFinished || got.Outcome == nil || *got.Outcome != service.RunOutcomeProcessed {
t.Fatalf("row = {status: %q, outcome: %v}, want the later FinishRun write: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed)
t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed)
}
})
t.Run("FinishRun after FailRun", func(t *testing.T) {
svc := newTestService(t)
id := queueAndClaim(t, svc)
wrote, err := svc.FailRun(ctx, id)
if err != nil || !wrote {
t.Fatalf("fail: wrote=%v err=%v, want wrote=true", wrote, err)
}
wrote, err = svc.FinishRun(ctx, id, service.RunOutcomeProcessed)
if err != nil {
t.Fatalf("finish: %v", err)
}
if wrote {
t.Error("finish: wrote = true, want false (run was already failed, not claimed)")
}
got, err := svc.GetRun(ctx, id)
if err != nil {
t.Fatalf("get run: %v", err)
}
if got.Status != models.RunStatusFailed || got.Outcome != nil {
t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {failed, nil}", got.Status, got.Outcome)
}
})
}
@@ -44,7 +44,11 @@ func (si *SchedulerIntegration) evaluateRunStaleness(
return false, "", nil
}
// cancelStaleRun marks the run cancelled, logs the event, and releases any checkout.
// cancelStaleRun marks the run cancelled, logs the event, and releases
// any checkout. Terminal-shape recording, the OfficeRunProcessed
// broadcast, and the activity log entry only happen when the cancel
// actually applied: a run another writer already finished or failed
// must not be reported as cancelled.
func (si *SchedulerIntegration) cancelStaleRun(
ctx context.Context,
run *models.Run,
@@ -60,12 +64,18 @@ func (si *SchedulerIntegration) cancelStaleRun(
si.releaseCheckoutIfNeeded(ctx, run)
si.svc.clearAgentWorking(ctx, agent.ID, run.ID)
if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil {
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason)
if err != nil {
si.logger.Error("failed to cancel stale run",
zap.String("run_id", run.ID), zap.Error(err))
} else {
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
return
}
if !cancelled {
return
}
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
si.svc.LogActivityWithRun(ctx, agent.WorkspaceID,
"scheduler", "office-scheduler",
@@ -74,6 +74,103 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) {
}
}
// TestSchedulerTick_TasklessRunRecordsTerminalShape is the Testing round 3
// regression test. failTasklessRun calls repo.MarkRunFailed directly
// (not through HandleAgentFailure), so it bypassed office_loop_terminal_total
// entirely — the same bypass class Review round 1 (R1-1) fixed for the other
// three production terminal-writers, just missed here. A taskless run never
// launches, so it must classify as unlaunched_failed.
func TestSchedulerTick_TasklessRunRecordsTerminalShape(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "coordinator-terminal-shape",
WorkspaceID: "ws-1",
Name: "coordinator-terminal-shape",
Role: models.AgentRoleCEO,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
service.RunSchedulerTick(svc, ctx)
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
// TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent is the PR
// fixup round 2 regression test (CodeRabbit). failTasklessRun used to
// call AppendRunEvent before checking MarkRunFailed's wrote result, so a
// run a concurrent writer (e.g. a cancel) already made terminal still
// picked up a "scheduler.launch" error event on its timeline — a spurious
// entry on a run whose actual outcome was decided by the other writer.
// AppendRunEvent must only fire once wrote=true confirms this call won
// the terminal-write race.
func TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "coordinator-taskless-race",
WorkspaceID: "ws-1",
Name: "coordinator-taskless-race",
Role: models.AgentRoleCEO,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: %v %v", run, err)
}
// Another writer wins the terminal-transition race before
// failTasklessRun below runs: the row is already finished, so
// MarkRunFailed's guarded UPDATE (status = 'claimed') matches nothing.
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish (simulating the winning writer): %v", err)
}
service.FailTasklessRunForTest(svc, ctx, run, agent, "scheduler cannot launch a taskless run")
events, err := svc.ListRunEventsForTest(ctx, run.ID)
if err != nil {
t.Fatalf("list run events: %v", err)
}
for _, e := range events {
if e.EventType == "error" {
t.Fatalf("expected no error event appended for a lost taskless-fail race; got %+v", e)
}
}
survivor, err := svc.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if survivor.Status != service.RunStatusFinished {
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
}
}
// TestSchedulerTick_TaskBoundRunStillLaunches is the regression guard
// alongside the taskless-failure fix above: an ordinary task-bound run with
// a wired task starter must still launch normally and stay `claimed` (not
@@ -53,6 +53,30 @@ type TaskStarterWithLaunchContext interface {
StartTaskWithLaunchContext(ctx context.Context, taskID string, agentProfileID string, launch LaunchContext) error
}
// TaskStarterWithSession optionally returns the id of the agent session
// a direct (non-routed) launch created, so the caller can persist it on
// the run row (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that does not
// implement this leaves the run's session id empty, counted as a
// without-session launch.
type TaskStarterWithSession interface {
StartTaskWithEnvReturningSession(ctx context.Context, taskID string, agentProfileID string, executorID string,
executorProfileID string, priority string, prompt string, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment, env map[string]string) (sessionID string, err error)
}
// TaskStarterWithLaunchContextSession combines TaskStarterWithLaunchContext
// and TaskStarterWithSession: a starter satisfying this carries the full
// launch context (skills included) AND returns the launched session id in
// the same call, so neither capability has to be dropped for the other.
// AC-OFFICE-LOOP-LIVENESS-002.7 requires the session id unconditionally, on
// every direct launch, regardless of whether that launch also carries
// per-run skill additions — the production adapter must satisfy this
// rather than TaskStarterWithLaunchContext alone.
type TaskStarterWithLaunchContextSession interface {
StartTaskWithLaunchContextReturningSession(ctx context.Context, taskID string, agentProfileID string,
launch LaunchContext) (sessionID string, err error)
}
// LaunchContext mirrors scheduler.LaunchContext so the office.service
// package can carry the Office-built launch context (prompt, env,
// workflow step, attachments, plan-mode, profile) into the routing
@@ -0,0 +1,139 @@
package service_test
import (
"context"
"expvar"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// mockTaskStarterWithSession wraps mockTaskStarter and additionally
// implements service.TaskStarterWithSession, so direct-launch tests can
// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7)
// without touching the production orchestrator adapter.
type mockTaskStarterWithSession struct {
*mockTaskStarter
sessionID string
}
func (m *mockTaskStarterWithSession) StartTaskWithEnvReturningSession(
ctx context.Context, taskID, agentProfileID, executorID,
executorProfileID string, priority string, prompt, workflowStepID string,
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
) (string, error) {
if err := m.StartTaskWithEnv(ctx, taskID, agentProfileID, executorID,
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env); err != nil {
return "", err
}
return m.sessionID, nil
}
func directHopExpvarInt(t *testing.T, mapName, key string) int64 {
t.Helper()
v := expvar.Get(mapName)
if v == nil {
t.Fatalf("expvar map %q not registered", mapName)
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatalf("expvar %q is not a Map", mapName)
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("expvar %q[%q] is not an Int", mapName, key)
}
return iv.Value()
}
// AC-OFFICE-LOOP-LIVENESS-002.7: on a direct (non-routed) launch, the
// session id the starter returns is persisted onto the run row.
func TestSchedulerTick_DirectLaunchPersistsSessionID(t *testing.T) {
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-1"}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "direct-session-agent",
WorkspaceID: "ws-1",
Name: "direct-session-worker",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
VALUES ('task-direct-session-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-session-1"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
before := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1")
service.RunSchedulerTick(svc, ctx)
if mock.callCount() != 1 {
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
}
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 1 || runs[0].SessionID != "sess-direct-1" {
t.Fatalf("session_id = %q, want sess-direct-1", runs[0].SessionID)
}
after := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1")
if after != before+1 {
t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before)
}
}
// AC-002.8: a starter that only implements TaskStarterWithEnv (no
// session-returning seam) leaves the run's session id empty and counts
// under launch_without_session — the existing production fallback.
func TestSchedulerTick_DirectLaunchWithoutSessionSeamCountsWithoutSession(t *testing.T) {
mock := &mockTaskStarter{}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "direct-nosession-agent",
WorkspaceID: "ws-1",
Name: "direct-nosession-worker",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
VALUES ('task-direct-nosession-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-nosession-1"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
before := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
service.RunSchedulerTick(svc, ctx)
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 1 || runs[0].SessionID != "" {
t.Fatalf("session_id = %q, want empty", runs[0].SessionID)
}
after := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
if after != before+1 {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before)
}
}
@@ -0,0 +1,154 @@
package service_test
import (
"context"
"fmt"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
// rejectSessionIDWrites installs a trigger that aborts only an UPDATE
// setting runs.session_id to wantSessionID — the exact value the
// launch under test will produce — so it targets persistLaunchedSession's
// post-launch SetRunSessionID write without also catching the
// pre-launch runtime-context build, which writes session_id too (see
// runtime/context_builder.go) but with the run's pre-launch id.
func rejectSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) {
t.Helper()
svc.ExecSQL(t, fmt.Sprintf(`
CREATE TRIGGER reject_session_id_write
BEFORE UPDATE OF session_id ON runs
WHEN NEW.session_id = '%s'
BEGIN
SELECT RAISE(ABORT, 'session_id write rejected for test');
END;
`, wantSessionID))
}
// AC-002.11: on a direct (non-routed) launch, a session persist failure
// is counted separately from a without-session launch, and does not
// fail the launch itself or overwrite the run's session_id — the agent
// is already running by the time this write happens.
func TestSchedulerTick_DirectLaunchSessionPersistFailureCountsSeparately(t *testing.T) {
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-persist-fail"}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "direct-persist-fail-agent",
WorkspaceID: "ws-1",
Name: "direct-persist-fail-worker",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
VALUES ('task-direct-persist-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-persist-fail-1"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
rejectSessionIDWrites(t, svc, "sess-direct-persist-fail")
beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
service.RunSchedulerTick(svc, ctx)
if mock.callCount() != 1 {
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
}
runs, err := svc.ListRuns(ctx, "ws-1")
if err != nil {
t.Fatalf("list runs: %v", err)
}
if len(runs) != 1 {
t.Fatalf("run count = %d, want 1", len(runs))
}
if runs[0].SessionID != "" {
t.Fatalf("session_id = %q, want empty after a rejected write", runs[0].SessionID)
}
// The launch itself must still count as claimed/in-flight, not failed —
// the agent is already running despite the persist failure.
if runs[0].Status != service.RunStatusClaimed {
t.Fatalf("run status = %q, want claimed (launch succeeded despite persist failure)", runs[0].Status)
}
afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
if afterFailed != beforeFailed+1 {
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed)
}
afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
if afterWithoutSession != beforeWithoutSession {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession)
}
}
// ignoreSessionIDWrites installs a trigger that uses RAISE(IGNORE)
// rather than RAISE(ABORT): the UPDATE setting runs.session_id to
// wantSessionID matches zero rows and returns no error, the shape a run
// row that vanished between claim and launch would produce — a
// different failure mode than rejectSessionIDWrites' hard DB error.
func ignoreSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) {
t.Helper()
svc.ExecSQL(t, fmt.Sprintf(`
CREATE TRIGGER ignore_session_id_write
BEFORE UPDATE OF session_id ON runs
WHEN NEW.session_id = '%s'
BEGIN
SELECT RAISE(IGNORE);
END;
`, wantSessionID))
}
// AC-002.11: on a direct (non-routed) launch, a real, non-empty session
// id whose write affects zero rows (no error) must be counted as a
// session persist failure, not miscounted as "launch yielded no session
// id" (AC-002.8's counter).
func TestSchedulerTick_DirectLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) {
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-vanished"}
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
ctx := context.Background()
agent := &models.AgentInstance{
ID: "direct-vanished-agent",
WorkspaceID: "ws-1",
Name: "direct-vanished-worker",
Role: models.AgentRoleWorker,
Status: models.AgentStatusIdle,
ExecutorPreference: `{"type":"worktree"}`,
}
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
VALUES ('task-direct-vanished-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-vanished-1"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
ignoreSessionIDWrites(t, svc, "sess-direct-vanished")
beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
service.RunSchedulerTick(svc, ctx)
if mock.callCount() != 1 {
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
}
afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
if afterFailed != beforeFailed+1 {
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed)
}
afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
if afterWithoutSession != beforeWithoutSession {
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession)
}
}
@@ -0,0 +1,95 @@
package service_test
import (
"context"
"expvar"
"testing"
"github.com/kandev/kandev/internal/office/models"
"github.com/kandev/kandev/internal/office/service"
)
func terminalShapeExpvarInt(t *testing.T, key string) int64 {
t.Helper()
v := expvar.Get("office_loop_terminal_total")
if v == nil {
t.Fatal("expvar map office_loop_terminal_total not registered")
}
m, ok := v.(*expvar.Map)
if !ok {
t.Fatal("office_loop_terminal_total is not a Map")
}
sub := m.Get(key)
if sub == nil {
return 0
}
iv, ok := sub.(*expvar.Int)
if !ok {
t.Fatalf("office_loop_terminal_total[%q] is not an Int", key)
}
return iv.Value()
}
// REQ-OFFICE-LOOP-LIVENESS-005: FinishRun classifies the just-finished
// run and increments office_loop_terminal_total by shape. A run
// finished with outcome=processed and no session id classifies as
// silent_success (activation is published at repo construction in
// these in-memory test DBs, so the run's requested_at is after it).
func TestFinishRun_RecordsSilentSuccessTerminalShape(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := makeAgent("terminal-shape-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: run=%v err=%v", run, err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeSilentSuccess))
before := terminalShapeExpvarInt(t, key)
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
t.Fatalf("finish: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("silent_success delta = %d, want 1", after-before)
}
}
// A failed run with no session id classifies as unlaunched_failed.
func TestFailRun_RecordsUnlaunchedFailedTerminalShape(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
agent := makeAgent("terminal-shape-fail-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
t.Fatalf("queue: %v", err)
}
run, err := svc.ClaimNextRun(ctx)
if err != nil || run == nil {
t.Fatalf("claim: run=%v err=%v", run, err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if _, err := svc.FailRun(ctx, run.ID); err != nil {
t.Fatalf("fail: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
@@ -0,0 +1,108 @@
package service
import "time"
// TerminalShape is the derived classification of a terminal run
// (REQ-OFFICE-LOOP-LIVENESS-005). It never widens runs.status or
// runs.outcome — see docs/specs/task-delivery-ledger/spec.md, which
// owns runs.outcome.
type TerminalShape string
const (
// ShapePreActivation is a run predating the point this capability's
// columns were confirmed present, or read before that point was ever
// published — never a defect (AC-005.6).
ShapePreActivation TerminalShape = "pre_activation"
// ShapeLaunchedCompleted is a run that reported success and named a
// session.
ShapeLaunchedCompleted TerminalShape = "launched_completed"
// ShapeLaunchedFailed is a run that reached a terminal failure and
// named a session.
ShapeLaunchedFailed TerminalShape = "launched_failed"
// ShapeSilentSuccess is a run whose outcome asserts work happened but
// which never named a session — NFR-1 made observable (AC-005.4).
ShapeSilentSuccess TerminalShape = "silent_success"
// ShapeUnlaunchedSkipped is a legitimate no-launch skip, kept apart
// from a launch failure (AC-005.5).
ShapeUnlaunchedSkipped TerminalShape = "unlaunched_skipped"
// ShapeUnlaunchedFailed is a terminal failure that never named a
// session.
ShapeUnlaunchedFailed TerminalShape = "unlaunched_failed"
// ShapeUnclassified is every combination none of the above predicates
// claim, including a null outcome, an outcome this document does not
// enumerate (such as the historical no_agent_launched), and an
// unknown status — the row that makes the classification total
// (AC-005.2).
ShapeUnclassified TerminalShape = "unclassified"
)
var terminalFailedStatuses = map[string]bool{
RunStatusFailed: true,
"timed_out": true,
RunStatusCancelled: true,
}
var skipOutcomes = map[string]bool{
RunOutcomeIdleSkipped: true,
RunOutcomeBudgetBlocked: true,
RunOutcomeBudgetUnmeasurable: true,
RunOutcomeAgentInactive: true,
RunOutcomeTaskTreeHeld: true,
}
// ClassifyTerminalRun classifies one terminal run (status not in
// {queued, claimed} — AC-005.9 excludes those from every terminal
// total; they are the stuck-run detector's input instead) into exactly
// one TerminalShape. Reads only the four persisted values named in its
// signature, never a free-text message (AC-005.7), and evaluates
// top-to-bottom with first match winning — the ordering is load-bearing:
// pre_activation first so the activation guard cannot be bypassed by a
// later row matching, silent_success below launched_completed so the
// session test is what separates them, and above unclassified so a null
// outcome never absorbs it.
//
// activationInstant/activationPublished carry the boot-time probe result
// (persisted separately in kandev_meta); a run requested before
// activation, or read when activation was never published, is always
// pre_activation regardless of what it would otherwise classify as.
func ClassifyTerminalRun(
status string,
outcome *string,
sessionID string,
requestedAt time.Time,
activationInstant time.Time,
activationPublished bool,
) TerminalShape {
if !activationPublished || requestedAt.Before(activationInstant) {
return ShapePreActivation
}
outcomeVal := ""
if outcome != nil {
outcomeVal = *outcome
}
return classifyByOutcome(status, outcomeVal, sessionID != "")
}
// classifyByOutcome applies the first-match-wins predicate table once
// activation is confirmed, split out from ClassifyTerminalRun to keep
// each function's branching within the complexity limit.
func classifyByOutcome(status, outcomeVal string, launched bool) TerminalShape {
processed := outcomeVal == RunOutcomeProcessed
failedTerminal := terminalFailedStatuses[status]
switch {
case status == RunStatusFinished && processed && launched:
return ShapeLaunchedCompleted
case failedTerminal && launched:
return ShapeLaunchedFailed
case status == RunStatusFinished && processed && !launched:
return ShapeSilentSuccess
case status == RunStatusFinished && !launched && skipOutcomes[outcomeVal]:
return ShapeUnlaunchedSkipped
case failedTerminal && !launched:
return ShapeUnlaunchedFailed
default:
return ShapeUnclassified
}
}
@@ -0,0 +1,188 @@
package service_test
import (
"testing"
"time"
"github.com/kandev/kandev/internal/office/service"
)
func strp(s string) *string { return &s }
// crossProductCase pins one (status, outcome, session presence) cell of
// the AC-005.2 cross-product to the exact shape the spec's predicate
// table assigns it — a literal expectation, not a second computation of
// ClassifyTerminalRun's own branching, so a misclassification (not just
// an out-of-set value) fails the test.
type crossProductCase struct {
status string
outcome *string
sessionID string
want service.TerminalShape
}
// crossProductCases enumerates all 5 statuses x 8 outcomes x 2 session
// states (80 cells) this codebase's runs.status/outcome/session_id can
// hold, each mapped to its expected shape by hand from the spec table.
func crossProductCases() []crossProductCase {
outcomeCases := []*string{
nil,
strp(service.RunOutcomeProcessed),
strp(service.RunOutcomeIdleSkipped),
strp(service.RunOutcomeBudgetBlocked),
strp(service.RunOutcomeBudgetUnmeasurable),
strp(service.RunOutcomeAgentInactive),
strp(service.RunOutcomeTaskTreeHeld),
strp("no_agent_launched"), // legacy value; not in current code
}
cases := []crossProductCase{
// status=finished: processed+session decides launched_completed
// vs. silent_success; a skip outcome decides unlaunched_skipped
// only without a session; everything else is unclassified.
{"finished", nil, "", service.ShapeUnclassified},
{"finished", nil, "sess-1", service.ShapeUnclassified},
{"finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess},
{"finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted},
{"finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped},
{"finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
{"finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped},
{"finished", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified},
{"finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped},
{"finished", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified},
{"finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped},
{"finished", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified},
{"finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped},
{"finished", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified},
{"finished", strp("no_agent_launched"), "", service.ShapeUnclassified},
{"finished", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified},
// status=some_unknown_status: never finished, never a recognized
// failure status, so always unclassified regardless of outcome
// or session.
{"some_unknown_status", nil, "", service.ShapeUnclassified},
{"some_unknown_status", nil, "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeProcessed), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnclassified},
{"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified},
{"some_unknown_status", strp("no_agent_launched"), "", service.ShapeUnclassified},
{"some_unknown_status", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified},
}
// status IN (failed, cancelled, timed_out): the spec's predicate
// table names all three under one row and never reads outcome for
// them, so every outcome yields the same pair of shapes for each —
// unlaunched_failed without a session, launched_failed with one.
for _, status := range []string{"failed", "cancelled", "timed_out"} {
for _, outcome := range outcomeCases {
cases = append(cases,
crossProductCase{status, outcome, "", service.ShapeUnlaunchedFailed},
crossProductCase{status, outcome, "sess-1", service.ShapeLaunchedFailed},
)
}
}
return cases
}
// AC-OFFICE-LOOP-LIVENESS-005.1/.2: classification is total over the
// cross-product of every persisted status this codebase writes
// (finished, failed, cancelled, timed_out, and an unknown status —
// runs.status is not the closed RunStatus enum) against every observed
// outcome (including the legacy no_agent_launched value and NULL),
// crossed with session_id present/absent — pinned to the exact shape
// each of the 80 cells must produce, not just closed-set membership.
func TestClassifyTerminalRun_CrossProduct(t *testing.T) {
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
requestedAt := activation.Add(time.Hour)
cases := crossProductCases()
if len(cases) != 5*8*2 {
t.Fatalf("cross-product case count = %d, want %d (5 statuses x 8 outcomes x 2 session states)",
len(cases), 5*8*2)
}
for _, tc := range cases {
got := service.ClassifyTerminalRun(
tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true,
)
if got != tc.want {
t.Errorf("ClassifyTerminalRun(status=%q, outcome=%v, session=%q) = %q, want %q",
tc.status, tc.outcome, tc.sessionID, got, tc.want)
}
}
}
func TestClassifyTerminalRun_PreActivation(t *testing.T) {
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
// Before the activation instant: pre_activation regardless of
// otherwise-silent-success shape (AC-005.6).
requestedBefore := activation.Add(-time.Minute)
got := service.ClassifyTerminalRun(
"finished", strp(service.RunOutcomeProcessed), "", requestedBefore, activation, true,
)
if got != service.ShapePreActivation {
t.Fatalf("got %q, want pre_activation", got)
}
// Activation instant never published: always pre_activation.
got = service.ClassifyTerminalRun(
"finished", strp(service.RunOutcomeProcessed), "", requestedBefore.Add(10*time.Hour), activation, false,
)
if got != service.ShapePreActivation {
t.Fatalf("got %q, want pre_activation when unpublished", got)
}
}
func TestClassifyTerminalRun_NamedShapes(t *testing.T) {
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
requestedAt := activation.Add(time.Hour)
cases := []struct {
name string
status string
outcome *string
sessionID string
want service.TerminalShape
}{
{"launched_completed", "finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted},
{"launched_failed/failed", "failed", nil, "sess-1", service.ShapeLaunchedFailed},
{"launched_failed/timed_out", "timed_out", nil, "sess-1", service.ShapeLaunchedFailed},
{"launched_failed/cancelled", "cancelled", nil, "sess-1", service.ShapeLaunchedFailed},
{"silent_success", "finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess},
{"unlaunched_skipped/idle", "finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped},
{"unlaunched_skipped/budget", "finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped},
{"unlaunched_skipped/budget_unmeasurable", "finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped},
{"unlaunched_skipped/inactive", "finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped},
{"unlaunched_skipped/tree_held", "finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped},
{"unlaunched_failed", "failed", nil, "", service.ShapeUnlaunchedFailed},
{"unclassified/null_outcome_finished", "finished", nil, "", service.ShapeUnclassified},
{"unclassified/legacy_no_agent_launched", "finished", strp("no_agent_launched"), "", service.ShapeUnclassified},
// A skip outcome carrying a session id is not a real production
// shape (skip outcomes are pre-launch guards) but the total
// classification must still place it somewhere, and it must not
// be reported as "unlaunched" while a session is on record
// (AC-005.3) — it falls to unclassified.
{"unclassified/skip_with_session", "finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
{"unclassified/unknown_status", "some_unknown_status", nil, "", service.ShapeUnclassified},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := service.ClassifyTerminalRun(tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true)
if got != tc.want {
t.Fatalf("ClassifyTerminalRun(%q, %v, %q) = %q, want %q",
tc.status, tc.outcome, tc.sessionID, got, tc.want)
}
})
}
}
@@ -205,6 +205,25 @@ func AdmitRunForTest(svc *Service, ctx context.Context, run *models.Run, agent *
return si.admitRun(ctx, run, agent)
}
// DeferWorkspaceLookupFailureForTest exposes deferWorkspaceLookupFailure
// directly for external test packages, so its MaxRetryCount-exhausted
// lost-race path (another writer already made the run terminal before
// this call's FailRun runs) can be driven without reproducing a transient
// GetAgentFromConfig lookup error through the full scheduler pipeline.
func DeferWorkspaceLookupFailureForTest(svc *Service, ctx context.Context, run *models.Run) {
si := &SchedulerIntegration{svc: svc, logger: svc.logger}
si.deferWorkspaceLookupFailure(ctx, run)
}
// FailTasklessRunForTest exposes failTasklessRun directly for external test
// packages, so the lost-race path (another writer already made the run
// terminal before this call runs) can be driven without the scheduler's
// atomic claim+launch pipeline standing in the way.
func FailTasklessRunForTest(svc *Service, ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string) {
si := &SchedulerIntegration{svc: svc, logger: svc.logger}
si.failTasklessRun(ctx, run, agent, msg)
}
// LogPolicyObservabilityForTest exposes logPolicyObservability for external
// test packages, so the skip/degraded-admitted entries and their
// per-policy-per-day dedup (AC-OFFICE-BUDGET-002.13/-004.8) can be tested
@@ -55,9 +55,11 @@ func (s *Service) PauseTaskTree(ctx context.Context, rootTaskID string) (*models
return nil, fmt.Errorf("create pause members: %w", err)
}
s.cancelTaskExecutions(ctx, members, "tree_paused")
if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused"); err != nil {
cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused")
if err != nil {
s.logger.Warn("cancel runs for paused tree failed", zap.Error(err))
}
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold)
return hold, nil
}
@@ -99,9 +101,11 @@ func (s *Service) CancelTaskTree(ctx context.Context, rootTaskID, cancelledBy st
if err := s.repo.BulkReleaseTaskCheckout(ctx, memberTaskIDs(members)); err != nil {
return nil, fmt.Errorf("release task checkouts: %w", err)
}
if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled"); err != nil {
cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled")
if err != nil {
s.logger.Warn("cancel runs for cancelled tree failed", zap.Error(err))
}
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
_ = cancelledBy
s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold)
return hold, nil
@@ -61,6 +61,40 @@ func TestPauseTaskTree(t *testing.T) {
}
}
// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns is the Testing
// round 3 regression test. PauseTaskTree cancels every member task's runs
// via repo.CancelRunsForTasks, which bypassed office_loop_terminal_total
// entirely — the same bypass class Review round 1 fixed for the
// single-run CancelRun call sites. A queued run never launched, so
// pausing its tree must classify the cancellation as unlaunched_failed.
func TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) {
canceller := &mockTaskCanceller{}
svc := newTestService(t, service.ServiceOptions{TaskCanceller: canceller})
ctx := context.Background()
insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS")
agent := makeAgent("tree-pause-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"root"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if _, err := svc.PauseTaskTree(ctx, "root"); err != nil {
t.Fatalf("PauseTaskTree: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
func TestCancelAndRestoreTaskTree(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
@@ -106,3 +140,33 @@ func TestCancelAndRestoreTaskTree(t *testing.T) {
t.Fatalf("active hold after restore = %+v, want nil; cancel hold was %s", preview.ActiveHold, hold.ID)
}
}
// TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns mirrors
// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns for
// CancelTaskTree's own repo.CancelRunsForTasks call site.
func TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) {
svc := newTestService(t)
ctx := context.Background()
insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS")
agent := makeAgent("tree-cancel-worker", models.AgentRoleWorker)
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
t.Fatalf("create agent: %v", err)
}
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
`{"task_id":"root"}`, ""); err != nil {
t.Fatalf("queue: %v", err)
}
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
before := terminalShapeExpvarInt(t, key)
if _, err := svc.CancelTaskTree(ctx, "root", "user:test"); err != nil {
t.Fatalf("CancelTaskTree: %v", err)
}
after := terminalShapeExpvarInt(t, key)
if after != before+1 {
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
}
}
@@ -189,7 +189,7 @@ func seedAgentFailureHandler(
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil {
if _, err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil {
log.Error("test harness: mark run failed", zap.Error(err))
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
@@ -0,0 +1,199 @@
package wakeup_test
import (
"context"
"testing"
officemodels "github.com/kandev/kandev/internal/office/models"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/shared"
"github.com/kandev/kandev/internal/office/wakeup"
)
func (h *testHarness) seedWakeupWithCausation(id, source, payload, causationID string) {
h.t.Helper()
if err := h.repo.CreateWakeupRequest(context.Background(), &officesqlite.WakeupRequest{
ID: id, AgentProfileID: h.agentID, Source: source, Payload: payload, CausationID: causationID,
}); err != nil {
h.t.Fatalf("seed wakeup: %v", err)
}
}
func (h *testHarness) seedRunWithReasonAndCausation(id, status, reason, causationID string) {
h.t.Helper()
run := &officemodels.Run{
ID: id,
AgentProfileID: h.agentID,
Reason: reason,
Payload: "{}",
Status: officemodels.RunStatus(status),
CoalescedCount: 1,
ContextSnapshot: `{"prior":"snapshot"}`,
CausationID: causationID,
}
if err := h.repo.CreateRun(context.Background(), run); err != nil {
h.t.Fatalf("seed run: %v", err)
}
}
// AC-OFFICE-LOOP-LIVENESS-002.3: createFreshRun copies the causation id
// from the wakeup request onto the fresh run.
func TestDispatch_FreshRunCopiesCausationIDFromWakeup(t *testing.T) {
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
h.seedWakeupWithCausation("w-1", wakeup.SourceSelf, `{"reason":"test"}`, "cause-fresh-1")
if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil {
t.Fatalf("dispatch: %v", err)
}
got, err := h.repo.GetWakeupRequest(context.Background(), "w-1")
if err != nil {
t.Fatalf("get wakeup request: %v", err)
}
run, err := h.repo.GetRunByID(context.Background(), got.RunID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if run.CausationID != "cause-fresh-1" {
t.Fatalf("run.CausationID = %q, want cause-fresh-1", run.CausationID)
}
}
// AC-OFFICE-LOOP-LIVENESS-002.4: coalescing touches only the wakeup
// request, never the run. The in-flight run keeps the id of the wake
// that created it; the coalesced request keeps its own — the two stay
// joinable through agent_wakeup_requests.run_id.
func TestDispatch_CoalesceLeavesRunCausationIDUntouched(t *testing.T) {
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
h.seedRunWithReasonAndCausation("run-pre", "queued", shared.RunReasonHeartbeat, "cause-run-original")
h.seedWakeupWithCausation("w-1", wakeup.SourceComment, `{"task_id":"t-1"}`, "cause-wakeup-coalesced")
if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil {
t.Fatalf("dispatch: %v", err)
}
gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-1")
if err != nil {
t.Fatalf("get wakeup request: %v", err)
}
if gotReq.CausationID != "cause-wakeup-coalesced" {
t.Fatalf("coalesced request CausationID = %q, want unchanged cause-wakeup-coalesced", gotReq.CausationID)
}
if gotReq.RunID != "run-pre" {
t.Fatalf("coalesced request run_id = %q, want run-pre", gotReq.RunID)
}
gotRun, err := h.repo.GetRunByID(context.Background(), "run-pre")
if err != nil {
t.Fatalf("get run: %v", err)
}
if gotRun.CausationID != "cause-run-original" {
t.Fatalf("run.CausationID = %q, want unchanged cause-run-original (coalesce must not touch it)", gotRun.CausationID)
}
}
// AC-002.4 skip half: MarkWakeupRequestSkipped touches only the
// request; it has no run to touch, and the request's own causation id
// survives the transition.
func TestDispatch_SkipLeavesWakeupCausationIDUntouched(t *testing.T) {
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
// skip_if_active requires the routine lookup to resolve that policy;
// simplest deterministic path here is a non-routine source with an
// in-flight run under the default coalesce policy is not skip — so
// drive skip via the routine lookup fallback used elsewhere in this
// package (skip_if_active policy routine).
h.dispatcher.SetRoutineLookup(fakeRoutineLookupSkip{})
h.seedRunWithReasonAndCausation("run-inflight", "claimed", shared.RunReasonRoutineDispatchCron, "cause-inflight")
h.seedWakeupWithCausation("w-skip", wakeup.SourceRoutine, `{"routine_id":"r-skip"}`, "cause-skip-1")
if err := h.dispatcher.Dispatch(context.Background(), "w-skip"); err != nil {
t.Fatalf("dispatch: %v", err)
}
got, err := h.repo.GetWakeupRequest(context.Background(), "w-skip")
if err != nil {
t.Fatalf("get wakeup request: %v", err)
}
if got.Status != officesqlite.WakeupStatusSkipped {
t.Fatalf("status = %q, want skipped", got.Status)
}
if got.CausationID != "cause-skip-1" {
t.Fatalf("skipped request CausationID = %q, want unchanged cause-skip-1", got.CausationID)
}
}
// AC-002.4/AC-002.5: PromoteRunAndCoalesceWakeupIfQueued promotes the
// run's reason but must never touch causation_id — a promotion is not a
// new cause.
func TestDispatch_PromotionDoesNotChangeRunCausationID(t *testing.T) {
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
h.seedRunWithReasonAndCausation("run-cron", "queued", shared.RunReasonRoutineDispatchCron, "cause-cron-original")
h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-1")
// Give the event wakeup an event-classified reason so promotion fires.
if _, err := h.repo.ExecRaw(context.Background(),
`UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`,
shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil {
t.Fatalf("seed reason: %v", err)
}
if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil {
t.Fatalf("dispatch: %v", err)
}
run, err := h.repo.GetRunByID(context.Background(), "run-cron")
if err != nil {
t.Fatalf("get run: %v", err)
}
if run.Reason != shared.RunReasonRoutineDispatchEvent {
t.Fatalf("run.Reason = %q, want promoted to %q", run.Reason, shared.RunReasonRoutineDispatchEvent)
}
if run.CausationID != "cause-cron-original" {
t.Fatalf("run.CausationID = %q, want unchanged cause-cron-original (promotion is not a new cause)", run.CausationID)
}
}
// AC-002.3/AC-002.4: the lost-CAS fresh-run path (event coalescing into
// an already-claimed cron run) creates its own run, and that run
// carries the *requesting* wake's causation id, not the claimed run's.
func TestDispatch_LostCASFreshRunCarriesRequestingWakeCausationID(t *testing.T) {
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
h.seedRunWithReasonAndCausation("run-cron", "claimed", shared.RunReasonRoutineDispatchCron, "cause-claimed-cron")
h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-fresh")
if _, err := h.repo.ExecRaw(context.Background(),
`UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`,
shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil {
t.Fatalf("seed reason: %v", err)
}
if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil {
t.Fatalf("dispatch: %v", err)
}
gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-event")
if err != nil {
t.Fatalf("get wakeup request: %v", err)
}
if gotReq.RunID == "" || gotReq.RunID == "run-cron" {
t.Fatalf("expected a fresh run distinct from the claimed one, got %q", gotReq.RunID)
}
freshRun, err := h.repo.GetRunByID(context.Background(), gotReq.RunID)
if err != nil {
t.Fatalf("get fresh run: %v", err)
}
if freshRun.CausationID != "cause-event-fresh" {
t.Fatalf("fresh run.CausationID = %q, want cause-event-fresh (the requesting wake's id)", freshRun.CausationID)
}
claimedRun, err := h.repo.GetRunByID(context.Background(), "run-cron")
if err != nil {
t.Fatalf("get claimed run: %v", err)
}
if claimedRun.CausationID != "cause-claimed-cron" {
t.Fatalf("claimed run.CausationID = %q, want unchanged cause-claimed-cron", claimedRun.CausationID)
}
}
// fakeRoutineLookupSkip resolves every routine to skip_if_active so the
// skip transition can be driven deterministically without a real
// office_routines row.
type fakeRoutineLookupSkip struct{}
func (fakeRoutineLookupSkip) GetRoutine(_ context.Context, id string) (*officemodels.Routine, error) {
return &officemodels.Routine{ID: id, ConcurrencyPolicy: officemodels.RoutineConcurrencyPolicy(wakeup.PolicySkipIfActive)}, nil
}
@@ -290,6 +290,10 @@ func (d *Dispatcher) createFreshRun(
CoalescedCount: 1,
ContextSnapshot: payload,
RequestedAt: time.Now().UTC(),
// AC-OFFICE-LOOP-LIVENESS-002.3: copied from the requesting
// wake, including on the lost-CAS fresh-run path — that run
// still carries the requesting wake's id, not a new one.
CausationID: req.CausationID,
}
if err := d.repo.CreateRun(ctx, run); err != nil {
return fmt.Errorf("create run for wakeup %s: %w", req.ID, err)
@@ -1102,8 +1102,8 @@ type startTaskOptions struct {
func (s *Service) StartTaskWithRoute(
ctx context.Context, taskID, agentProfileID string,
launch executor.LaunchContext, route executor.RouteOverride,
) error {
_, err := s.startTask(ctx, taskID, agentProfileID,
) (*executor.TaskExecution, error) {
return s.startTask(ctx, taskID, agentProfileID,
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority,
launch.Prompt, launch.WorkflowStepID, launch.PlanMode, false,
launch.Attachments, startTaskOptions{
@@ -1111,7 +1111,6 @@ func (s *Service) StartTaskWithRoute(
AdditionalSkillSlugs: append([]string(nil), launch.AdditionalSkillSlugs...),
Route: &route,
})
return err
}
func (s *Service) prepareExplicitWorkflowStartRoute(
@@ -225,7 +225,7 @@ func TestReadPathsUseTheReadOnlyHandle(t *testing.T) {
}
// Writes must not fall back to the reader.
if err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil {
if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil {
t.Errorf("FinishRun succeeded with a closed writer; the write path is not using r.db")
}
}
@@ -16,54 +16,81 @@ import (
// and have its real finished_at overwritten.
const cancellableRunStatuses = `status IN ('queued', 'claimed')`
// CancelledRun identifies one row CancelRunsWhere actually cancelled,
// carrying the fields a caller needs to classify the transition's
// terminal shape (office_loop_terminal_total) without a second read.
type CancelledRun struct {
ID string
AgentProfileID string
SessionID string
RequestedAt time.Time
}
// CancelRunsWhere is the single writer of the terminal cancel state on the
// runs table. CancelRun, BulkCancelRuns and the office repository's
// CancelRunsForTasks are all thin selectors over it, so the guard above
// cannot be bypassed by picking a different entry point.
// CancelRunsForTasks and CancelDisplacedParticipantRun are all thin
// selectors over it, so the guard above cannot be bypassed by picking a
// different entry point.
//
// selector is an extra WHERE fragment ANDed onto the guard; it must be a
// SQL literal owned by the calling repository method, using ? placeholders
// (rebound here) for every value. Never interpolate caller-supplied data
// into it. selectorArgs bind those placeholders, in order.
//
// Returns the number of rows actually cancelled: 0 means every run the
// selector matched had already reached a terminal state.
// Returns the rows actually cancelled: an empty slice means every run the
// selector matched had already reached a terminal state. The guard and the
// write are the same UPDATE statement (RETURNING reads the classification
// columns off the rows it just changed), so there is no separate read a
// row's status could change behind: a run that reaches a terminal state on
// another connection is either fully visible to this statement or not, and
// either way it cannot be cancelled by it.
func (r *Repository) CancelRunsWhere(
ctx context.Context,
cancelReason string,
selector string,
selectorArgs ...interface{},
) (int64, error) {
args := make([]interface{}, 0, len(selectorArgs)+2)
args = append(args, cancelReason, time.Now().UTC())
args = append(args, selectorArgs...)
) ([]CancelledRun, error) {
query := fmt.Sprintf(`
UPDATE runs
SET status = 'cancelled', cancel_reason = ?, finished_at = ?
WHERE %s
AND (%s)
RETURNING id, agent_profile_id, session_id, requested_at
`, cancellableRunStatuses, selector)
res, err := r.db.ExecContext(ctx, r.db.Rebind(query), args...)
args := append([]interface{}{cancelReason, time.Now().UTC()}, selectorArgs...)
rows, err := r.db.QueryxContext(ctx, r.db.Rebind(query), args...)
if err != nil {
return 0, err
return nil, err
}
return res.RowsAffected()
defer rows.Close() //nolint:errcheck
var cancelled []CancelledRun
for rows.Next() {
var row CancelledRun
if err := rows.Scan(&row.ID, &row.AgentProfileID, &row.SessionID, &row.RequestedAt); err != nil {
return nil, err
}
cancelled = append(cancelled, row)
}
return cancelled, rows.Err()
}
// CancelRun marks a run as cancelled with an optional cancel reason.
// A run that already reached a terminal state is left untouched.
func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) error {
_, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id)
return err
// A run that already reached a terminal state is left untouched. Returns
// whether this call actually cancelled the row, so a caller counting the
// transition (office_loop_terminal_total) can tell a persisted change
// from a no-op on a run that was already terminal.
func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) (bool, error) {
rows, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id)
return len(rows) > 0, err
}
// BulkCancelRuns cancels multiple runs by ID with the given reason.
// Runs in the set that already reached a terminal state are left untouched.
func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) error {
// BulkCancelRuns cancels multiple runs by ID with the given reason,
// returning the rows actually cancelled (see CancelRunsWhere). Runs in
// the set that already reached a terminal state are left untouched.
func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) ([]CancelledRun, error) {
if len(ids) == 0 {
return nil
return nil, nil
}
placeholders := make([]string, len(ids))
args := make([]interface{}, len(ids))
@@ -72,6 +99,5 @@ func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelRea
args[i] = id
}
selector := fmt.Sprintf(`id IN (%s)`, strings.Join(placeholders, ","))
_, err := r.CancelRunsWhere(ctx, cancelReason, selector, args...)
return err
return r.CancelRunsWhere(ctx, cancelReason, selector, args...)
}
@@ -0,0 +1,139 @@
package sqlite_test
import (
"context"
"sync"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/kandev/kandev/internal/office/models"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/testutil"
)
// TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun is
// the two-connection regression for a TOCTOU CancelRunsWhere used to have:
// its cancellable-status guard was applied only to the candidate read, not
// to the write, so a run that genuinely finished on another connection
// between the two could be silently overwritten to cancelled. Real
// Postgres row locking forces the interleaving this needs: one connection
// holds an uncommitted finish on the row, CancelRunsWhere's write blocks on
// that row lock, and only proceeds once the finish commits — at which
// point it must see the fresh 'finished' status and cancel nothing. Skips
// unless KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun(t *testing.T) {
dsn := testutil.PostgresDSNFromEnv(t)
// db is the repo's own pool: OpenIsolatedPostgres pins it to exactly
// one physical connection and sets that connection's search_path to a
// fresh schema, so every statement through it (including the schema
// init below and repo.CancelRunsWhere's own transaction) lands there.
db := testutil.OpenIsolatedPostgres(t, dsn)
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init task repo: %v", err)
}
officeRepo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init office repo: %v", err)
}
repo := officeRepo.RunsRepository()
ctx := context.Background()
// finisherDB is a second, genuinely separate physical connection to
// the SAME schema. A pool-wide MaxOpenConns bump on a single *sqlx.DB
// would not do this safely: search_path is a per-connection session
// setting, so a new connection the pool opens under load would not
// inherit it and could silently operate against the wrong schema.
var schema string
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
t.Fatalf("read isolated schema name: %v", err)
}
finisherDB, err := sqlx.Open("pgx", dsn)
if err != nil {
t.Fatalf("open finisher connection: %v", err)
}
finisherDB.SetMaxOpenConns(1)
finisherDB.SetMaxIdleConns(1)
t.Cleanup(func() { _ = finisherDB.Close() })
if _, err := finisherDB.Exec("SET search_path TO " + schema); err != nil {
t.Fatalf("set finisher search_path: %v", err)
}
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
now := time.Now().UTC()
if _, err := db.ExecContext(ctx, db.Rebind(
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
), now, run.ID); err != nil {
t.Fatalf("seed claimed: %v", err)
}
tx, err := finisherDB.BeginTxx(ctx, nil)
if err != nil {
t.Fatalf("begin finisher tx: %v", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, finisherDB.Rebind(
`UPDATE runs SET status = 'finished', outcome = 'processed', finished_at = ? WHERE id = ?`,
), now, run.ID); err != nil {
t.Fatalf("finisher update (holds the row lock, uncommitted): %v", err)
}
var (
wg sync.WaitGroup
cancelled []runssqlite.CancelledRun
cancelErr error
cancelDone = make(chan struct{})
commitErr error
)
wg.Add(1)
go func() {
defer wg.Done()
defer close(cancelDone)
cancelled, cancelErr = repo.CancelRunsWhere(ctx, "too late", `id = ?`, run.ID)
}()
// Give CancelRunsWhere's write time to reach Postgres and block on the
// row lock the finisher's uncommitted transaction holds, before that
// transaction commits. Without this window the two goroutines could
// run in either order and the test would not exercise the lock-wait
// path this bug depends on.
select {
case <-cancelDone:
t.Fatal("CancelRunsWhere returned before the finisher committed; it never blocked on the row lock")
case <-time.After(200 * time.Millisecond):
}
commitErr = tx.Commit()
wg.Wait()
if commitErr != nil {
t.Fatalf("finisher commit: %v", commitErr)
}
if cancelErr != nil {
t.Fatalf("cancel where: %v", cancelErr)
}
if len(cancelled) != 0 {
t.Fatalf("cancelled = %d, want 0: a run that committed finished must not be reported cancelled", len(cancelled))
}
if got := mustGetRunStatus(t, db, run.ID); got != "finished" {
t.Fatalf("run status = %q, want finished (the concurrent cancel must not have overwritten it)", got)
}
}
func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string {
t.Helper()
var status string
if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil {
t.Fatalf("get run status: %v", err)
}
return status
}
@@ -6,6 +6,7 @@ import (
"time"
"github.com/kandev/kandev/internal/office/models"
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
)
// TestCancelRun_CancelsQueuedAndClaimedRuns pins every column the
@@ -21,9 +22,13 @@ func TestCancelRun_CancelsQueuedAndClaimedRuns(t *testing.T) {
setStatus(t, repo, claimed.ID, "claimed", timePtr(now), nil)
for _, id := range []string{queued.ID, claimed.ID} {
if err := repo.CancelRun(ctx, id, "user stopped the task"); err != nil {
cancelled, err := repo.CancelRun(ctx, id, "user stopped the task")
if err != nil {
t.Fatalf("cancel %q: %v", id, err)
}
if !cancelled {
t.Errorf("%s: cancelled = false, want true", id)
}
got := mustGetRun(t, repo, id)
checkString(t, id+" status", string(got.Status), "cancelled")
checkStringPtr(t, id+" cancel_reason", got.CancelReason, strPtr("user stopped the task"))
@@ -47,9 +52,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) {
run := queueRunAt(t, repo, "run-"+status, "a1", finishedAt.Add(-time.Hour))
setStatus(t, repo, run.ID, status, nil, timePtr(finishedAt))
if err := repo.CancelRun(ctx, run.ID, "too late"); err != nil {
cancelled, err := repo.CancelRun(ctx, run.ID, "too late")
if err != nil {
t.Fatalf("cancel %q: %v", run.ID, err)
}
if cancelled {
t.Errorf("%s: cancelled = true, want false (already terminal)", run.ID)
}
got := mustGetRun(t, repo, run.ID)
checkString(t, run.ID+" status", string(got.Status), status)
if got.CancelReason != nil {
@@ -64,9 +73,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) {
// TestCancelRun_UnknownRunIsANoOp pins that a missing id is not an error.
func TestCancelRun_UnknownRunIsANoOp(t *testing.T) {
repo := newTestRepo(t)
if err := repo.CancelRun(context.Background(), "nope", "reason"); err != nil {
cancelled, err := repo.CancelRun(context.Background(), "nope", "reason")
if err != nil {
t.Fatalf("cancel unknown run: %v", err)
}
if cancelled {
t.Error("cancelled = true, want false (unknown run)")
}
}
// TestCancelRunsWhere_ReportsRowsActuallyCancelled pins the returned
@@ -87,8 +100,8 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) {
if err != nil {
t.Fatalf("cancel where: %v", err)
}
if cancelled != 2 {
t.Errorf("cancelled = %d, want 2", cancelled)
if len(cancelled) != 2 {
t.Errorf("cancelled = %d, want 2", len(cancelled))
}
checkString(t, "q1", string(mustGetRun(t, repo, "q1").Status), "cancelled")
checkString(t, "c1", string(mustGetRun(t, repo, "c1").Status), "cancelled")
@@ -101,8 +114,71 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) {
if err != nil {
t.Fatalf("cancel where (second pass): %v", err)
}
if cancelled != 0 {
t.Errorf("second pass cancelled = %d, want 0", cancelled)
if len(cancelled) != 0 {
t.Errorf("second pass cancelled = %d, want 0", len(cancelled))
}
}
// TestCancelRunsWhere_ReturnedRowsCarryClassificationFields pins the
// fields a caller needs to classify each cancelled run's loop-liveness
// terminal shape (office_loop_terminal_total) without a second read: the
// launched run's session_id must survive onto the returned row, distinct
// from the never-launched run's empty one.
func TestCancelRunsWhere_ReturnedRowsCarryClassificationFields(t *testing.T) {
repo := newTestRepo(t)
ctx := context.Background()
requestedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
launched := mustCreateRun(t, repo, &models.Run{
ID: "launched", AgentProfileID: "a1", Reason: "task_assigned",
Payload: `{}`, Status: "claimed",
})
if err := repo.SetRunRequestedAtForTest(ctx, launched.ID, requestedAt); err != nil {
t.Fatalf("backdate requested_at: %v", err)
}
if _, err := repo.SetRunSessionID(ctx, launched.ID, "session-in-flight"); err != nil {
t.Fatalf("set session id: %v", err)
}
neverLaunched := mustCreateRun(t, repo, &models.Run{
ID: "never-launched", AgentProfileID: "a2", Reason: "task_assigned",
Payload: `{}`, Status: "queued",
})
if err := repo.SetRunRequestedAtForTest(ctx, neverLaunched.ID, requestedAt); err != nil {
t.Fatalf("backdate requested_at: %v", err)
}
cancelled, err := repo.CancelRunsWhere(ctx, "reason",
`id IN (?, ?)`, launched.ID, neverLaunched.ID)
if err != nil {
t.Fatalf("cancel where: %v", err)
}
if len(cancelled) != 2 {
t.Fatalf("cancelled = %d, want 2", len(cancelled))
}
byID := map[string]runssqlite.CancelledRun{}
for _, row := range cancelled {
byID[row.ID] = row
}
got, ok := byID[launched.ID]
if !ok {
t.Fatalf("launched run missing from cancelled rows: %v", cancelled)
}
if got.AgentProfileID != "a1" {
t.Errorf("launched agent_profile_id = %q, want a1", got.AgentProfileID)
}
if got.SessionID != "session-in-flight" {
t.Errorf("launched session_id = %q, want session-in-flight", got.SessionID)
}
if !got.RequestedAt.Equal(requestedAt) {
t.Errorf("launched requested_at = %v, want %v", got.RequestedAt, requestedAt)
}
got, ok = byID[neverLaunched.ID]
if !ok {
t.Fatalf("never-launched run missing from cancelled rows: %v", cancelled)
}
if got.SessionID != "" {
t.Errorf("never-launched session_id = %q, want empty", got.SessionID)
}
}
@@ -126,8 +202,8 @@ func TestCancelRunsWhere_MultiArgSelectorBindsInOrder(t *testing.T) {
if err != nil {
t.Fatalf("cancel where: %v", err)
}
if cancelled != 1 {
t.Errorf("cancelled = %d, want 1", cancelled)
if len(cancelled) != 1 {
t.Errorf("cancelled = %d, want 1", len(cancelled))
}
checkString(t, "match", string(mustGetRun(t, repo, "match").Status), "cancelled")
checkString(t, "wrong-reason", string(mustGetRun(t, repo, "wrong-reason").Status), "queued")
@@ -148,12 +224,15 @@ func TestBulkCancelRuns_CancelsOnlyTheListedEligibleRuns(t *testing.T) {
setStatus(t, repo, finished.ID, "finished", nil, timePtr(now))
queueRunAt(t, repo, "out-of-set", "a1", now)
err := repo.BulkCancelRuns(ctx,
cancelled, err := repo.BulkCancelRuns(ctx,
[]string{"in-set-queued", "in-set-claimed", "in-set-finished", "missing-id"},
"workspace deleted")
if err != nil {
t.Fatalf("bulk cancel: %v", err)
}
if len(cancelled) != 2 {
t.Errorf("cancelled = %d, want 2 (queued + claimed, not the finished run)", len(cancelled))
}
checkString(t, "in-set-queued", string(mustGetRun(t, repo, "in-set-queued").Status), "cancelled")
checkString(t, "in-set-claimed", string(mustGetRun(t, repo, "in-set-claimed").Status), "cancelled")
@@ -174,9 +253,13 @@ func TestBulkCancelRuns_EmptyIDListIsANoOp(t *testing.T) {
queueRunAt(t, repo, "survivor", "a1", time.Now().UTC())
for _, ids := range [][]string{nil, {}} {
if err := repo.BulkCancelRuns(ctx, ids, "nothing"); err != nil {
cancelled, err := repo.BulkCancelRuns(ctx, ids, "nothing")
if err != nil {
t.Fatalf("bulk cancel %v: %v", ids, err)
}
if len(cancelled) != 0 {
t.Errorf("bulk cancel %v: cancelled = %d, want 0", ids, len(cancelled))
}
}
got := mustGetRun(t, repo, "survivor")
checkString(t, "status", string(got.Status), "queued")
@@ -116,7 +116,8 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) {
return repo.UpdateRunOutputSummary(ctx, run.ID, "o", "f")
},
"FinishRun": func() error {
return repo.FinishRun(ctx, run.ID, "finished", nil)
_, err := repo.FinishRun(ctx, run.ID, "finished", nil)
return err
},
"ClaimRun": func() error {
_, err := repo.ClaimRun(ctx, "a1")
@@ -146,10 +147,12 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) {
return err
},
"CancelRun": func() error {
return repo.CancelRun(ctx, run.ID, "reason")
_, err := repo.CancelRun(ctx, run.ID, "reason")
return err
},
"BulkCancelRuns": func() error {
return repo.BulkCancelRuns(ctx, []string{run.ID}, "reason")
_, err := repo.BulkCancelRuns(ctx, []string{run.ID}, "reason")
return err
},
"AppendRunEvent": func() error {
_, err := repo.AppendRunEvent(ctx, run.ID, "run.progress", "info", "{}")
@@ -0,0 +1,122 @@
package sqlite_test
import (
"context"
"sync"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/kandev/kandev/internal/office/models"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/testutil"
)
// TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun is the
// two-connection regression for Review round 3, R3-1: FinishRun's write
// used to carry no status guard at all, so a run cancelled on another
// connection between the caller's read and this write would have its
// 'cancelled' status and real finished_at silently overwritten to
// 'finished'/'failed'. Real Postgres row locking forces the interleaving
// this needs: one connection holds an uncommitted cancel on the row,
// FinishRun's write blocks on that row lock, and only proceeds once the
// cancel commits — at which point the guard (status = 'claimed') must see
// the fresh 'cancelled' status and write nothing. Skips unless
// KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) {
dsn := testutil.PostgresDSNFromEnv(t)
db := testutil.OpenIsolatedPostgres(t, dsn)
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
t.Fatalf("init task repo: %v", err)
}
officeRepo, err := officesqlite.NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init office repo: %v", err)
}
repo := officeRepo.RunsRepository()
ctx := context.Background()
var schema string
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
t.Fatalf("read isolated schema name: %v", err)
}
cancellerDB, err := sqlx.Open("pgx", dsn)
if err != nil {
t.Fatalf("open canceller connection: %v", err)
}
cancellerDB.SetMaxOpenConns(1)
cancellerDB.SetMaxIdleConns(1)
t.Cleanup(func() { _ = cancellerDB.Close() })
if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil {
t.Fatalf("set canceller search_path: %v", err)
}
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
if err := repo.CreateRun(ctx, run); err != nil {
t.Fatalf("create run: %v", err)
}
now := time.Now().UTC()
if _, err := db.ExecContext(ctx, db.Rebind(
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
), now, run.ID); err != nil {
t.Fatalf("seed claimed: %v", err)
}
tx, err := cancellerDB.BeginTxx(ctx, nil)
if err != nil {
t.Fatalf("begin canceller tx: %v", err)
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, cancellerDB.Rebind(
`UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`,
), "too late", now, run.ID); err != nil {
t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err)
}
var (
wg sync.WaitGroup
finishedRun *models.Run
finishErr error
finishDone = make(chan struct{})
commitErr error
)
outcome := "processed"
wg.Add(1)
go func() {
defer wg.Done()
defer close(finishDone)
finishedRun, finishErr = repo.FinishRun(ctx, run.ID, "finished", &outcome)
}()
// Give FinishRun's write time to reach Postgres and block on the row
// lock the canceller's uncommitted transaction holds, before that
// transaction commits. Without this window the two goroutines could
// run in either order and the test would not exercise the lock-wait
// path this bug depends on.
select {
case <-finishDone:
t.Fatal("FinishRun returned before the canceller committed; it never blocked on the row lock")
case <-time.After(200 * time.Millisecond):
}
commitErr = tx.Commit()
wg.Wait()
if commitErr != nil {
t.Fatalf("canceller commit: %v", commitErr)
}
if finishErr != nil {
t.Fatalf("finish run: %v", finishErr)
}
if finishedRun != nil {
t.Fatalf("finished run = %+v, want nil: a run that committed cancelled must not be reported finished", finishedRun)
}
if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" {
t.Fatalf("run status = %q, want cancelled (the concurrent finish must not have overwritten it)", got)
}
}
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@@ -41,13 +42,13 @@ func (r *Repository) CreateRunTx(ctx context.Context, tx *sqlx.Tx, req *models.R
id, agent_profile_id, reason, payload, status, coalesced_count,
idempotency_key, context_snapshot, capabilities, input_snapshot,
output_summary, failure_reason, session_id, retry_count, scheduled_retry_at,
requested_at, error_message, cancel_reason, continuation_scope
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
requested_at, error_message, cancel_reason, continuation_scope, causation_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`), req.ID, req.AgentProfileID, req.Reason, req.Payload, req.Status,
req.CoalescedCount, req.IdempotencyKey, req.ContextSnapshot,
req.Capabilities, req.InputSnapshot, req.OutputSummary, req.FailureReason,
req.SessionID, req.RetryCount, req.ScheduledRetryAt, req.RequestedAt,
req.ErrorMessage, req.CancelReason, req.ContinuationScope)
req.ErrorMessage, req.CancelReason, req.ContinuationScope, req.CausationID)
return err
}
@@ -103,6 +104,35 @@ func (r *Repository) UpdateRunRuntimeSnapshot(
return err
}
// SetRunSessionID persists the session id a launch produced. A no-op
// when sessionID is empty — the caller counts that as a without-session
// launch rather than clobbering whatever the column already held.
// Guarded to status = 'claimed' so a launch that loses a race against a
// concurrent cancel or terminal write cannot mutate an already-terminal
// row's session id after the fact: without the guard, a terminal-shape
// classification already recorded off the empty session id would drift
// from what the row shows on a later read. Returns whether the row was
// still claimed so the caller can distinguish a real write from a
// stale run id or a lost race.
func (r *Repository) SetRunSessionID(
ctx context.Context, runID, sessionID string,
) (bool, error) {
if sessionID == "" {
return false, nil
}
res, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE runs SET session_id = ? WHERE id = ? AND status = 'claimed'
`), sessionID, runID)
if err != nil {
return false, err
}
n, err := res.RowsAffected()
if err != nil {
return false, err
}
return n > 0, nil
}
// UpdateRunPromptArtifacts persists the assembled prompt the agent
// received and the continuation-summary content prepended at dispatch.
// Called from the scheduler-integration after BuildAgentPrompt completes
@@ -189,12 +219,32 @@ func (r *Repository) ClaimRun(ctx context.Context, agentInstanceID string) (*mod
// outcome from a different transition. outcome is nil for the failed path
// and for callers with no established semantic label (docs/specs/
// task-delivery-ledger/spec.md, "Office run outcome").
func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) error {
// FinishRun writes the terminal status/outcome and returns the row as it
// stands immediately after that write, via the same statement (RETURNING),
// so a caller classifying the transition (office_loop_terminal_total) never
// depends on a separate read succeeding independently of the write that
// persisted it. Guarded to status = 'claimed', the same status every caller
// reaches this from (ClaimNextEligibleRun then processRun/an event
// subscriber): without the guard, a cancel that commits between the
// caller's read and this write would have its 'cancelled' status and
// finished_at overwritten by this transition. Returns (nil, nil) for an
// unknown id or a run no longer claimed (already terminal by another
// writer): zero rows changed, so there is nothing to classify.
func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) (*models.Run, error) {
now := time.Now().UTC()
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE runs SET status = ?, outcome = ?, finished_at = ? WHERE id = ?
`), status, outcome, now, id)
return err
var run models.Run
err := r.db.QueryRowxContext(ctx, r.db.Rebind(`
UPDATE runs SET status = ?, outcome = ?, finished_at = ?
WHERE id = ? AND status = 'claimed'
RETURNING *
`), status, outcome, now, id).StructScan(&run)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &run, nil
}
// GetRunByID returns the run row for a given ID. Returns sql.ErrNoRows when unknown.
@@ -275,7 +275,7 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) {
before := time.Now().UTC().Add(-time.Second)
run := mustCreateRun(t, repo, fullRun())
if err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil {
if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil {
t.Fatalf("finish run: %v", err)
}
got := mustGetRun(t, repo, run.ID)
@@ -288,6 +288,35 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) {
}
}
// TestFinishRun_ReturnsFullRowWithTheReaderClosed pins the property a
// terminal-shape classifier depends on: FinishRun's own RETURNING
// clause carries every field needed to classify the transition (here,
// SessionID), so a caller never needs a separate read through r.ro to
// get it.
func TestFinishRun_ReturnsFullRowWithTheReaderClosed(t *testing.T) {
repo, _, reader := newTestRepoWithHandles(t)
ctx := context.Background()
run := mustCreateRun(t, repo, fullRun())
if err := reader.Close(); err != nil {
t.Fatalf("close reader: %v", err)
}
got, err := repo.FinishRun(ctx, run.ID, "finished", nil)
if err != nil {
t.Fatalf("finish run with the reader closed: %v", err)
}
if got == nil {
t.Fatal("finish run returned a nil row")
}
checkString(t, "status", string(got.Status), "finished")
checkString(t, "session_id", got.SessionID, "sess-7")
checkString(t, "agent_profile_id", got.AgentProfileID, "agent-42")
if got.FinishedAt == nil {
t.Fatalf("finished_at = nil, want a stamp")
}
}
// TestSetRunStatusForTest_WritesStatusAndBothTimestamps covers the
// harness affordance used by the E2E seeder, including its NULL case.
func TestSetRunStatusForTest_WritesStatusAndBothTimestamps(t *testing.T) {
@@ -0,0 +1,117 @@
package sqlite_test
import (
"context"
"testing"
"github.com/kandev/kandev/internal/office/models"
)
// AC-OFFICE-LOOP-LIVENESS-002.7/.8/.10: SetRunSessionID persists the
// session id a launch produced, is a no-op on an empty id (so a caller
// can distinguish "wrote nothing" from a real write), and — given the
// single-launch-in-flight invariant — an unconditional non-empty write
// is "last non-empty wins".
func TestSetRunSessionID_PersistsNonEmptyID(t *testing.T) {
repo := newTestRepo(t)
run := &models.Run{ID: "run-session-1", AgentProfileID: "agent-1", Status: "claimed"}
mustCreateRun(t, repo, run)
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-abc")
if err != nil {
t.Fatalf("SetRunSessionID: %v", err)
}
if !wrote {
t.Fatal("expected wrote=true for existing run with non-empty session id")
}
got := mustGetRun(t, repo, run.ID)
if got.SessionID != "sess-abc" {
t.Fatalf("session_id = %q, want sess-abc", got.SessionID)
}
}
func TestSetRunSessionID_EmptyIDIsNoop(t *testing.T) {
repo := newTestRepo(t)
run := &models.Run{ID: "run-session-2", AgentProfileID: "agent-1", Status: "claimed", SessionID: "sess-existing"}
mustCreateRun(t, repo, run)
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "")
if err != nil {
t.Fatalf("SetRunSessionID: %v", err)
}
if wrote {
t.Fatal("expected wrote=false for empty session id")
}
got := mustGetRun(t, repo, run.ID)
if got.SessionID != "sess-existing" {
t.Fatalf("session_id = %q, want unchanged sess-existing", got.SessionID)
}
}
func TestSetRunSessionID_UnknownRunIsNoop(t *testing.T) {
repo := newTestRepo(t)
wrote, err := repo.SetRunSessionID(context.Background(), "run-does-not-exist", "sess-x")
if err != nil {
t.Fatalf("SetRunSessionID: %v", err)
}
if wrote {
t.Fatal("expected wrote=false for unknown run id")
}
}
// A cancel that lands between claim and this write already recorded a
// terminal shape off the run's session id at that instant; letting this
// write land anyway would make the persisted row disagree with the
// classification that was already counted for it.
func TestSetRunSessionID_LostRaceAgainstConcurrentCancelIsNoop(t *testing.T) {
repo := newTestRepo(t)
run := &models.Run{ID: "run-session-4", AgentProfileID: "agent-1", Status: "claimed"}
mustCreateRun(t, repo, run)
cancelled, err := repo.CancelRun(context.Background(), run.ID, "lost_race")
if err != nil {
t.Fatalf("CancelRun: %v", err)
}
if !cancelled {
t.Fatal("expected CancelRun to cancel the still-claimed run")
}
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-late")
if err != nil {
t.Fatalf("SetRunSessionID: %v", err)
}
if wrote {
t.Fatal("expected wrote=false once the run is no longer claimed")
}
got := mustGetRun(t, repo, run.ID)
if got.SessionID != "" {
t.Fatalf("session_id = %q, want unchanged empty string on the cancelled row", got.SessionID)
}
if got.Status != "cancelled" {
t.Fatalf("status = %q, want cancelled", got.Status)
}
}
// AC-OFFICE-LOOP-LIVENESS-002.10: a later, non-empty write overwrites an
// earlier one — "last non-empty wins" for the relaunch-keeps-latest case.
func TestSetRunSessionID_LaterWriteOverwritesEarlier(t *testing.T) {
repo := newTestRepo(t)
run := &models.Run{ID: "run-session-3", AgentProfileID: "agent-1", Status: "claimed"}
mustCreateRun(t, repo, run)
if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-first"); err != nil {
t.Fatalf("first SetRunSessionID: %v", err)
}
if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-second"); err != nil {
t.Fatalf("second SetRunSessionID: %v", err)
}
got := mustGetRun(t, repo, run.ID)
if got.SessionID != "sess-second" {
t.Fatalf("session_id = %q, want sess-second", got.SessionID)
}
}