feat(office): make the unattended loop legible end-to-end (#3613)
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
@@ -260,6 +260,7 @@ func (a *routineWakeupAdapter) CreateWakeupRequest(
|
||||
Reason: req.Reason,
|
||||
Payload: req.Payload,
|
||||
RequestedAt: req.RequestedAt,
|
||||
CausationID: req.CausationID,
|
||||
}
|
||||
if req.IdempotencyKey != "" {
|
||||
row.IdempotencyKey = sql.NullString{String: req.IdempotencyKey, Valid: true}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
officemodels "github.com/kandev/kandev/internal/office/models"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
officeroutines "github.com/kandev/kandev/internal/office/routines"
|
||||
officeservice "github.com/kandev/kandev/internal/office/service"
|
||||
schedulercron "github.com/kandev/kandev/internal/scheduler/cron"
|
||||
tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
workflowrepo "github.com/kandev/kandev/internal/workflow/repository"
|
||||
@@ -48,6 +49,10 @@ func startCronScheduler(
|
||||
routines := schedulercron.NewRoutinesHandler(routineTicker, nil, log)
|
||||
loop := schedulercron.NewLoop(schedulercron.DefaultTickInterval, log,
|
||||
heartbeat, budget, routines, officeRecovery, parentWakeReconciler)
|
||||
// AC-OFFICE-LOOP-LIVENESS-003.6: published once, at boot, so a flat
|
||||
// office_loop_cron_tick_total is distinguishable from a restarted
|
||||
// process versus a genuinely stopped loop.
|
||||
officeservice.RecordLoopProcessStarted(time.Now().UTC().Format(time.RFC3339))
|
||||
loop.Start(ctx)
|
||||
log.Info("phase 5 cron loop started",
|
||||
zap.Duration("interval", schedulercron.DefaultTickInterval))
|
||||
|
||||
@@ -106,6 +106,7 @@ import (
|
||||
officeskills "github.com/kandev/kandev/internal/office/skills"
|
||||
officewakeup "github.com/kandev/kandev/internal/office/wakeup"
|
||||
orchexecutor "github.com/kandev/kandev/internal/orchestrator/executor"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
|
||||
// Runs queue (Phase 3 of task-model-unification)
|
||||
runsscheduler "github.com/kandev/kandev/internal/runs/scheduler"
|
||||
@@ -1474,6 +1475,10 @@ func wireOfficeSvcsDependencies(
|
||||
orchestratorSvc.SetOfficeTaskStatusUpdater(services.OfficeSvcs.Dashboard)
|
||||
// Wire the office service as the retry canceller for task reassignment.
|
||||
services.OfficeSvcs.Dashboard.SetRetryCanceller(services.Office)
|
||||
// Wire the office service as the terminal-shape recorder so a
|
||||
// dashboard-driven displaced-run cancellation still counts toward
|
||||
// office_loop_terminal_total.
|
||||
services.OfficeSvcs.Dashboard.SetTerminalShapeRecorder(services.Office)
|
||||
// Wire the office service as the task canceller for status→cancelled hard-cancels.
|
||||
services.OfficeSvcs.Dashboard.SetTaskCanceller(services.Office)
|
||||
// Route the Office "No parent" mutation through the canonical task detach
|
||||
@@ -1573,6 +1578,13 @@ type schedulerTaskStarterAdapter struct {
|
||||
orch *orchestrator.Service
|
||||
}
|
||||
|
||||
// The production routed-launch adapter is required to satisfy
|
||||
// TaskStarterWithSession — no silent fallback to a session-less launch
|
||||
// (AC-OFFICE-LOOP-LIVENESS-002.7). A compile-time assertion pins this
|
||||
// harder than a runtime wiring test: dropping the method fails the
|
||||
// build, not just a test run.
|
||||
var _ officescheduler.TaskStarterWithSession = (*schedulerTaskStarterAdapter)(nil)
|
||||
|
||||
// StartTask starts a task on the orchestrator with the given launch parameters.
|
||||
func (a *schedulerTaskStarterAdapter) StartTask(
|
||||
ctx context.Context,
|
||||
@@ -1594,6 +1606,32 @@ func (a *schedulerTaskStarterAdapter) StartTaskWithRoute(
|
||||
launch officescheduler.LaunchContext,
|
||||
route officescheduler.RouteOverride,
|
||||
) error {
|
||||
_, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route)
|
||||
return err
|
||||
}
|
||||
|
||||
// StartTaskWithRouteReturningSession implements
|
||||
// officescheduler.TaskStarterWithSession so the routed launch path can
|
||||
// persist the session id it started (AC-OFFICE-LOOP-LIVENESS-002.7).
|
||||
func (a *schedulerTaskStarterAdapter) StartTaskWithRouteReturningSession(
|
||||
ctx context.Context,
|
||||
taskID, agentProfileID string,
|
||||
launch officescheduler.LaunchContext,
|
||||
route officescheduler.RouteOverride,
|
||||
) (string, error) {
|
||||
execution, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route)
|
||||
if err != nil || execution == nil {
|
||||
return "", err
|
||||
}
|
||||
return execution.SessionID, nil
|
||||
}
|
||||
|
||||
func (a *schedulerTaskStarterAdapter) startTaskWithRoute(
|
||||
ctx context.Context,
|
||||
taskID, agentProfileID string,
|
||||
launch officescheduler.LaunchContext,
|
||||
route officescheduler.RouteOverride,
|
||||
) (*orchexecutor.TaskExecution, error) {
|
||||
return a.orch.StartTaskWithRoute(ctx, taskID, agentProfileID,
|
||||
orchexecutor.LaunchContext{
|
||||
ExecutorID: launch.ExecutorID,
|
||||
@@ -2069,19 +2107,101 @@ func backfillAgentDefaultSkills(
|
||||
}
|
||||
}
|
||||
|
||||
// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in the
|
||||
// officeservice.TaskStarterWithLaunchContextFunc adapter. Extracted from
|
||||
// initOfficeServices to keep that function under the funlen cap.
|
||||
// newOfficeTaskStarter wraps orchestratorSvc.StartTaskWithEnvAndSkills in an
|
||||
// adapter that also implements officeservice.TaskStarterWithLaunchContextSession,
|
||||
// so the direct (non-routed) launch path forwards per-run skill additions and
|
||||
// persists the session id the orchestrator's *executor.TaskExecution already
|
||||
// carries (AC-OFFICE-LOOP-LIVENESS-002.7) instead of discarding it. Extracted
|
||||
// from initOfficeServices to keep that function under the funlen cap.
|
||||
func newOfficeTaskStarter(orchestratorSvc *orchestrator.Service) officeservice.TaskStarter {
|
||||
return officeservice.TaskStarterWithLaunchContextFunc(
|
||||
func(ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext) error {
|
||||
_, err := orchestratorSvc.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
|
||||
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
|
||||
launch.WorkflowStepID, launch.PlanMode, false, launch.Attachments, launch.Env,
|
||||
launch.AdditionalSkillSlugs)
|
||||
return err
|
||||
},
|
||||
)
|
||||
return &officeOrchestratorTaskStarter{orch: orchestratorSvc}
|
||||
}
|
||||
|
||||
// officeOrchestratorTaskStarter satisfies officeservice.TaskStarter,
|
||||
// TaskStarterWithEnv, TaskStarterWithLaunchContext, TaskStarterWithSession,
|
||||
// and TaskStarterWithLaunchContextSession against the orchestrator service's
|
||||
// direct launch path.
|
||||
type officeOrchestratorTaskStarter struct {
|
||||
orch *orchestrator.Service
|
||||
}
|
||||
|
||||
// The production direct-launch adapter is required to satisfy
|
||||
// TaskStarterWithLaunchContextSession for the same reason as the routed
|
||||
// adapter above (AC-OFFICE-LOOP-LIVENESS-002.7): the session id must be
|
||||
// recorded unconditionally, whether or not the launch also carries skills.
|
||||
var _ officeservice.TaskStarterWithLaunchContextSession = (*officeOrchestratorTaskStarter)(nil)
|
||||
|
||||
// StartTask implements officeservice.TaskStarter.
|
||||
func (a *officeOrchestratorTaskStarter) StartTask(
|
||||
ctx context.Context, taskID, agentProfileID, executorID,
|
||||
executorProfileID string, priority string, prompt, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment,
|
||||
) error {
|
||||
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
|
||||
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, nil, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// StartTaskWithEnv implements officeservice.TaskStarterWithEnv.
|
||||
func (a *officeOrchestratorTaskStarter) StartTaskWithEnv(
|
||||
ctx context.Context, taskID, agentProfileID, executorID,
|
||||
executorProfileID string, priority string, prompt, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
|
||||
) error {
|
||||
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
|
||||
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// StartTaskWithEnvReturningSession implements
|
||||
// officeservice.TaskStarterWithSession.
|
||||
func (a *officeOrchestratorTaskStarter) StartTaskWithEnvReturningSession(
|
||||
ctx context.Context, taskID, agentProfileID, executorID,
|
||||
executorProfileID string, priority string, prompt, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
|
||||
) (string, error) {
|
||||
execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID,
|
||||
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil)
|
||||
if err != nil || execution == nil {
|
||||
return "", err
|
||||
}
|
||||
return execution.SessionID, nil
|
||||
}
|
||||
|
||||
// StartTaskWithLaunchContext implements officeservice.TaskStarterWithLaunchContext.
|
||||
func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContext(
|
||||
ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext,
|
||||
) error {
|
||||
_, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
|
||||
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
|
||||
launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env,
|
||||
launch.AdditionalSkillSlugs)
|
||||
return err
|
||||
}
|
||||
|
||||
// StartTaskWithLaunchContextReturningSession implements
|
||||
// officeservice.TaskStarterWithLaunchContextSession.
|
||||
func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContextReturningSession(
|
||||
ctx context.Context, taskID, agentProfileID string, launch officeservice.LaunchContext,
|
||||
) (string, error) {
|
||||
execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
|
||||
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt,
|
||||
launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env,
|
||||
launch.AdditionalSkillSlugs)
|
||||
if err != nil || execution == nil {
|
||||
return "", err
|
||||
}
|
||||
return execution.SessionID, nil
|
||||
}
|
||||
|
||||
func (a *officeOrchestratorTaskStarter) startTaskWithEnvAndSkills(
|
||||
ctx context.Context, taskID, agentProfileID, executorID,
|
||||
executorProfileID string, priority string, prompt, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment, env map[string]string, additionalSkillSlugs []string,
|
||||
) (*orchexecutor.TaskExecution, error) {
|
||||
return a.orch.StartTaskWithEnvAndSkills(ctx, taskID, agentProfileID,
|
||||
executorID, executorProfileID, priority, prompt,
|
||||
workflowStepID, planMode, false, attachments, env, additionalSkillSlugs)
|
||||
}
|
||||
|
||||
// newAgentAuth wraps officeagents.NewAgentAuth with a dev-mode warning when
|
||||
|
||||
@@ -68,6 +68,8 @@
|
||||
{"file": "internal/office/repository/sqlite/dashboard.go", "symbol": "QueryRunActivity", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
|
||||
{"file": "internal/office/repository/sqlite/labels.go", "symbol": "GetOrCreateLabel", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"},
|
||||
{"file": "internal/office/repository/sqlite/labels.go", "symbol": "AddLabelToTask", "rule": "conflict-syntax", "reason": "The Office repository is explicitly SQLite-only"},
|
||||
{"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "CountEligibleTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"},
|
||||
{"file": "internal/office/repository/sqlite/loop_health.go", "symbol": "ListOverdueOrStrandedTriggers", "rule": "boolean-integer", "reason": "The Office SQLite schema stores trigger enabled as an INTEGER flag"},
|
||||
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "hasFTSTable", "rule": "sqlite-catalog", "reason": "SQLite-only FTS inspection"},
|
||||
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTask", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
|
||||
{"file": "internal/office/repository/sqlite/tasks.go", "symbol": "CheckoutTaskForRun", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"},
|
||||
|
||||
@@ -44,6 +44,7 @@ type Handler struct {
|
||||
gitMgr *configloader.GitManager
|
||||
runDetail RunDetailRepo
|
||||
agentSummary AgentSummaryRepository
|
||||
loopHealth LoopHealthRepo
|
||||
handoff *taskservice.HandoffService
|
||||
guard ActiveSourceChecker
|
||||
logger *logger.Logger
|
||||
@@ -76,6 +77,9 @@ func NewHandler(svc *DashboardService, labelRepo labelFetcher, gitMgr *configloa
|
||||
if r, ok := labelRepo.(AgentSummaryRepository); ok {
|
||||
h.agentSummary = r
|
||||
}
|
||||
if r, ok := labelRepo.(LoopHealthRepo); ok {
|
||||
h.loopHealth = r
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
@@ -127,6 +131,8 @@ func RegisterRoutes(api *gin.RouterGroup, svc *DashboardService, labelRepo label
|
||||
api.GET("/workspaces/:wsId/routing/preview", h.getWorkspaceRoutingPreview)
|
||||
api.GET("/runs/:id/attempts", h.listRunAttempts)
|
||||
api.GET("/agents/:id/route", h.getAgentRoute)
|
||||
|
||||
registerLoopHealthRoutes(api, h)
|
||||
}
|
||||
|
||||
// -- Dashboard --
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// registerLoopHealthRoutes mounts the two REQ-OFFICE-LOOP-LIVENESS
|
||||
// reads on the Office route group. Both inherit AgentAuthMiddleware
|
||||
// and officeWorkspaceScopeMiddleware from the group they're mounted
|
||||
// on; :wsId is their only id, so neither needs a
|
||||
// officeParamScopeResolvers entry.
|
||||
func registerLoopHealthRoutes(api *gin.RouterGroup, h *Handler) {
|
||||
api.GET("/workspaces/:wsId/loop-health", h.getLoopHealth)
|
||||
api.GET("/workspaces/:wsId/loop-counters", h.getLoopCounters)
|
||||
}
|
||||
|
||||
// getLoopHealth serves GET /workspaces/:wsId/loop-health
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once here and
|
||||
// threaded through every predicate (AC-004.14). Any unreadable input
|
||||
// fails the whole read with 503 rather than a partial verdict
|
||||
// (AC-004.10), incrementing office_loop_liveness_degraded_total by
|
||||
// the failing input's reason.
|
||||
func (h *Handler) getLoopHealth(c *gin.Context) {
|
||||
if h.loopHealth == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: "loop health not configured"})
|
||||
return
|
||||
}
|
||||
wsID := c.Param("wsId")
|
||||
now := time.Now().UTC()
|
||||
|
||||
resp, err := EvaluateLoopHealth(c.Request.Context(), h.loopHealth, wsID, now)
|
||||
if err != nil {
|
||||
h.respondLoopHealthError(c, wsID, err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func (h *Handler) respondLoopHealthError(c *gin.Context, workspaceID string, err error) {
|
||||
var degraded *LoopHealthDegradedError
|
||||
reason := ReasonRunReadFailed
|
||||
if errors.As(err, °raded) {
|
||||
reason = degraded.Reason
|
||||
}
|
||||
service.IncLoopLivenessDegraded(workspaceID, reason)
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{errorKey: err.Error(), "reason": reason})
|
||||
}
|
||||
|
||||
// getLoopCounters serves GET /workspaces/:wsId/loop-counters
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-003). Reads process-wide expvar state, so
|
||||
// it never fails or 503s — an empty workspace simply reads all zeros.
|
||||
func (h *Handler) getLoopCounters(c *gin.Context) {
|
||||
wsID := c.Param("wsId")
|
||||
c.JSON(http.StatusOK, ReadLoopCounters(wsID))
|
||||
}
|
||||
|
||||
// LoopHealthRepoImpl returns the wired loop-health repo (may be nil).
|
||||
// Exposed for tests that need to confirm the wiring.
|
||||
func (h *Handler) LoopHealthRepoImpl() LoopHealthRepo { return h.loopHealth }
|
||||
@@ -0,0 +1,161 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"strings"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// loopCounterMapNames lists every office_loop_* expvar.Map counter
|
||||
// name (REQ-OFFICE-LOOP-LIVENESS-003, "Counters" table). Zero-filled
|
||||
// at counter-name granularity (AC-003.8's malformed-key note and the
|
||||
// design's "Zero-filling is at counter-name granularity" rule): a
|
||||
// name absent from a reader's response would be indistinguishable
|
||||
// from "never observed".
|
||||
var loopCounterMapNames = []string{
|
||||
"office_loop_trigger_claimed_total",
|
||||
"office_loop_routine_run_total",
|
||||
"office_loop_wakeup_created_total",
|
||||
"office_loop_run_claimed_total",
|
||||
"office_loop_launch_total",
|
||||
"office_loop_launch_without_session_total",
|
||||
"office_loop_session_persist_failed_total",
|
||||
"office_loop_terminal_total",
|
||||
"office_loop_last_run_at_write_failed_total",
|
||||
"office_loop_liveness_degraded_total",
|
||||
}
|
||||
|
||||
// LoopCountersResponse is the GET .../loop-counters payload. workspace
|
||||
// carries every counter above labelled to the requesting workspace
|
||||
// (secondary labels like source/disposition/shape/reason preserved,
|
||||
// but only as observed); process carries the cron-tick counter, the
|
||||
// two published instants, and the _unattributed totals — identical
|
||||
// for every reader, no workspace's data (AC-003.5, AC-003.8).
|
||||
type LoopCountersResponse struct {
|
||||
Workspace map[string]map[string]int64 `json:"workspace"`
|
||||
Process LoopCountersProcessDTO `json:"process"`
|
||||
}
|
||||
|
||||
// LoopCountersProcessDTO is the process-scoped block of the counter
|
||||
// read: never filtered by workspace, identical for every caller.
|
||||
type LoopCountersProcessDTO struct {
|
||||
CronTickTotal int64 `json:"cron_tick_total"`
|
||||
CronTickAt string `json:"cron_tick_at"`
|
||||
ProcessStartedAt string `json:"process_started_at"`
|
||||
Unattributed map[string]map[string]int64 `json:"unattributed"`
|
||||
Malformed map[string]map[string]int64 `json:"malformed"`
|
||||
}
|
||||
|
||||
// ReadLoopCounters parses every office_loop_* expvar.Map back into a
|
||||
// workspace-filtered, zero-filled response (AC-003.4, AC-003.5,
|
||||
// AC-003.7). A key that fails to parse into the expected "k=v;k=v"
|
||||
// label shape lands in the process block's malformed bucket rather
|
||||
// than being dropped or attributed to any workspace, the same
|
||||
// treatment AC-003.8 gives an unresolved workspace.
|
||||
func ReadLoopCounters(workspaceID string) LoopCountersResponse {
|
||||
resp := LoopCountersResponse{
|
||||
Workspace: make(map[string]map[string]int64, len(loopCounterMapNames)),
|
||||
Process: LoopCountersProcessDTO{
|
||||
Unattributed: make(map[string]map[string]int64),
|
||||
Malformed: make(map[string]map[string]int64),
|
||||
},
|
||||
}
|
||||
for _, name := range loopCounterMapNames {
|
||||
resp.Workspace[name] = map[string]int64{}
|
||||
}
|
||||
|
||||
if v := expvar.Get("office_loop_cron_tick_total"); v != nil {
|
||||
if m, ok := v.(*expvar.Map); ok {
|
||||
resp.Process.CronTickTotal = expvarMapTotal(m)
|
||||
}
|
||||
}
|
||||
if v := expvar.Get("office_loop_cron_tick_at"); v != nil {
|
||||
resp.Process.CronTickAt = v.String()
|
||||
resp.Process.CronTickAt = strings.Trim(resp.Process.CronTickAt, `"`)
|
||||
}
|
||||
if v := expvar.Get("office_loop_process_started_at"); v != nil {
|
||||
resp.Process.ProcessStartedAt = strings.Trim(v.String(), `"`)
|
||||
}
|
||||
|
||||
for _, name := range loopCounterMapNames {
|
||||
v := expvar.Get(name)
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
partitionLoopCounterMap(&resp, name, m, workspaceID)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// partitionLoopCounterMap walks one counter's labelled entries and
|
||||
// routes each into the workspace block, the process unattributed
|
||||
// bucket, or the process malformed bucket.
|
||||
func partitionLoopCounterMap(resp *LoopCountersResponse, mapName string, m *expvar.Map, workspaceID string) {
|
||||
m.Do(func(kv expvar.KeyValue) {
|
||||
iv, ok := kv.Value.(*expvar.Int)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
labels, ok := parseLoopCounterKey(kv.Key)
|
||||
if !ok {
|
||||
addLoopCounterBucket(resp.Process.Malformed, mapName, kv.Key, iv.Value())
|
||||
return
|
||||
}
|
||||
ws, hasWorkspace := labels["workspace"]
|
||||
switch {
|
||||
case hasWorkspace && ws == workspaceID:
|
||||
resp.Workspace[mapName][kv.Key] = iv.Value()
|
||||
case hasWorkspace && ws == service.LoopUnattributedWorkspace:
|
||||
addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value())
|
||||
case hasWorkspace:
|
||||
// A different workspace's value — filtered out entirely
|
||||
// (AC-003.5): not returned in any form to this reader.
|
||||
default:
|
||||
// No workspace label at all (process-only counter reached
|
||||
// via this generic loop, or a future label-less counter) —
|
||||
// treat as unattributed rather than silently dropped.
|
||||
addLoopCounterBucket(resp.Process.Unattributed, mapName, kv.Key, iv.Value())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func addLoopCounterBucket(bucket map[string]map[string]int64, mapName, key string, value int64) {
|
||||
if bucket[mapName] == nil {
|
||||
bucket[mapName] = map[string]int64{}
|
||||
}
|
||||
bucket[mapName][key] = value
|
||||
}
|
||||
|
||||
// parseLoopCounterKey parses the "k1=v1;k2=v2" expvar map key back
|
||||
// into labels. Total parse (AC-003.8's argument extended to this
|
||||
// read): any segment that doesn't split into exactly one "=" fails
|
||||
// the whole key rather than silently dropping that label.
|
||||
func parseLoopCounterKey(key string) (map[string]string, bool) {
|
||||
if key == "" {
|
||||
return map[string]string{}, true
|
||||
}
|
||||
labels := make(map[string]string)
|
||||
for _, part := range strings.Split(key, ";") {
|
||||
kv := strings.SplitN(part, "=", 2)
|
||||
if len(kv) != 2 || kv[0] == "" {
|
||||
return nil, false
|
||||
}
|
||||
labels[kv[0]] = kv[1]
|
||||
}
|
||||
return labels, true
|
||||
}
|
||||
|
||||
func expvarMapTotal(m *expvar.Map) int64 {
|
||||
var total int64
|
||||
m.Do(func(kv expvar.KeyValue) {
|
||||
if iv, ok := kv.Value.(*expvar.Int); ok {
|
||||
total += iv.Value()
|
||||
}
|
||||
})
|
||||
return total
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
func TestReadLoopCounters_FiltersToRequestingWorkspace(t *testing.T) {
|
||||
wsA := "ws-counters-a"
|
||||
wsB := "ws-counters-b"
|
||||
service.IncLoopRunClaimed(wsA)
|
||||
service.IncLoopRunClaimed(wsB)
|
||||
|
||||
resp := ReadLoopCounters(wsA)
|
||||
|
||||
got := resp.Workspace["office_loop_run_claimed_total"]
|
||||
if _, ok := got[service.LoopMetricLabel("workspace", wsB)]; ok {
|
||||
t.Errorf("workspace block contains another workspace's key: %+v", got)
|
||||
}
|
||||
if v, ok := got[service.LoopMetricLabel("workspace", wsA)]; !ok || v < 1 {
|
||||
t.Errorf("workspace block missing own key, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLoopCounters_UnobservedCounterIsZeroFilledNotOmitted(t *testing.T) {
|
||||
resp := ReadLoopCounters("ws-counters-never-seen")
|
||||
|
||||
for _, name := range loopCounterMapNames {
|
||||
if _, ok := resp.Workspace[name]; !ok {
|
||||
t.Errorf("counter %q missing from workspace block, want zero-filled empty map", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLoopCounters_UnattributedAndProcessInstantsPresent(t *testing.T) {
|
||||
service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace)
|
||||
service.IncLoopCronTick("2026-05-01T12:00:00Z")
|
||||
service.RecordLoopProcessStarted("2026-05-01T11:00:00Z")
|
||||
|
||||
resp := ReadLoopCounters("ws-counters-some-other-workspace")
|
||||
|
||||
unattr := resp.Process.Unattributed["office_loop_trigger_claimed_total"]
|
||||
if _, ok := unattr[service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; !ok {
|
||||
t.Errorf("unattributed bucket missing entry, got %+v", unattr)
|
||||
}
|
||||
if _, ok := resp.Workspace["office_loop_trigger_claimed_total"][service.LoopMetricLabel("workspace", service.LoopUnattributedWorkspace)]; ok {
|
||||
t.Errorf("unattributed key leaked into workspace block: %+v", resp.Workspace["office_loop_trigger_claimed_total"])
|
||||
}
|
||||
if resp.Process.CronTickTotal < 1 {
|
||||
t.Errorf("cron tick total = %d, want >= 1", resp.Process.CronTickTotal)
|
||||
}
|
||||
if resp.Process.CronTickAt == "" {
|
||||
t.Error("cron tick at is empty, want the published instant")
|
||||
}
|
||||
if resp.Process.ProcessStartedAt == "" {
|
||||
t.Error("process started at is empty, want the published instant")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadLoopCounters_MalformedKeyCountsIntoMalformedBucket(t *testing.T) {
|
||||
m := expvar.Get("office_loop_run_claimed_total").(*expvar.Map)
|
||||
m.Add("not-a-valid-label", 1)
|
||||
|
||||
resp := ReadLoopCounters("ws-counters-malformed-probe")
|
||||
|
||||
malformed := resp.Process.Malformed["office_loop_run_claimed_total"]
|
||||
if v, ok := malformed["not-a-valid-label"]; !ok || v < 1 {
|
||||
t.Errorf("malformed bucket missing key, got %+v", malformed)
|
||||
}
|
||||
if _, ok := resp.Workspace["office_loop_run_claimed_total"]["not-a-valid-label"]; ok {
|
||||
t.Error("malformed key leaked into workspace block")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// Thresholds (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.8 — echoed verbatim
|
||||
// in every response so a caller never has to hardcode them).
|
||||
const (
|
||||
officeLoopTriggerOverdueGrace = 3 * time.Minute
|
||||
officeLoopTriggerStrandedGrace = 5 * time.Minute
|
||||
officeLoopRunQueuedGrace = 2 * time.Minute
|
||||
officeLoopRunClaimedGrace = 10 * time.Minute
|
||||
officeLoopEvaluationWindow = 24 * time.Hour
|
||||
officeLoopEvidenceCap = 50
|
||||
)
|
||||
|
||||
// Verdict values, closed set (AC-004.1), precedence order
|
||||
// unknown -> dead -> degraded -> not_armed -> healthy (AC-004.2).
|
||||
const (
|
||||
VerdictUnknown = "unknown"
|
||||
VerdictDead = "dead"
|
||||
VerdictDegraded = "degraded"
|
||||
VerdictNotArmed = "not_armed"
|
||||
VerdictHealthy = "healthy"
|
||||
)
|
||||
|
||||
// Degraded reasons (AC-004.10), the enumerated set
|
||||
// office_loop_liveness_degraded_total{reason} is labelled with. Per
|
||||
// OPERATOR DECISION F31, the silent-success evidence read and the
|
||||
// terminal-shape count read both map to ReasonTerminalReadFailed —
|
||||
// both ultimately answer "what shape did this run end in", so a
|
||||
// caller diagnosing a 503 reads one reason for either query rather
|
||||
// than a distinction that doesn't help them act.
|
||||
const (
|
||||
ReasonActivationReadFailed = "activation_read_failed"
|
||||
ReasonTriggerReadFailed = "trigger_read_failed"
|
||||
ReasonRunReadFailed = "run_read_failed"
|
||||
ReasonTerminalReadFailed = "terminal_read_failed"
|
||||
)
|
||||
|
||||
// LoopHealthDegradedError names which input could not be read
|
||||
// (AC-004.10). The HTTP layer maps this to a 503 and increments
|
||||
// office_loop_liveness_degraded_total{reason}.
|
||||
type LoopHealthDegradedError struct {
|
||||
Reason string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *LoopHealthDegradedError) Error() string {
|
||||
return "loop health degraded (" + e.Reason + "): " + e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *LoopHealthDegradedError) Unwrap() error { return e.Err }
|
||||
|
||||
// LoopHealthRepo is the persistence surface EvaluateLoopHealth needs.
|
||||
// Implemented by *sqlite.Repository.
|
||||
type LoopHealthRepo interface {
|
||||
LoopLivenessActivationChecked() (at time.Time, published bool, err error)
|
||||
CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error)
|
||||
ListOverdueOrStrandedTriggers(
|
||||
ctx context.Context, workspaceID string, now time.Time,
|
||||
overdueGrace, strandedGrace time.Duration, limit int,
|
||||
) ([]sqlite.TriggerHealthRow, int, error)
|
||||
ListStuckRuns(
|
||||
ctx context.Context, workspaceID string, now time.Time,
|
||||
queuedGrace, claimedGrace time.Duration, limit int,
|
||||
) ([]sqlite.StuckRunRow, int, error)
|
||||
ListSilentSuccesses(
|
||||
ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int,
|
||||
) ([]sqlite.SilentSuccessRow, int, error)
|
||||
ListTerminalRunsInWindow(
|
||||
ctx context.Context, workspaceID string, windowStart time.Time,
|
||||
) ([]sqlite.TerminalRunShapeInputRow, error)
|
||||
}
|
||||
|
||||
// TriggerEvidenceDTO is one overdue or stranded trigger row (AC-004.7).
|
||||
type TriggerEvidenceDTO struct {
|
||||
TriggerID string `json:"trigger_id"`
|
||||
RoutineID string `json:"routine_id"`
|
||||
RoutineName string `json:"routine_name"`
|
||||
Condition string `json:"condition"`
|
||||
NextRunAt *time.Time `json:"next_run_at,omitempty"`
|
||||
LastFiredAt *time.Time `json:"last_fired_at,omitempty"`
|
||||
}
|
||||
|
||||
// StuckRunEvidenceDTO is one claimed-stuck or queued-stuck run row.
|
||||
type StuckRunEvidenceDTO struct {
|
||||
RunID string `json:"run_id"`
|
||||
AgentProfileID string `json:"agent_profile_id"`
|
||||
Status string `json:"status"`
|
||||
Condition string `json:"condition"`
|
||||
StuckSince *time.Time `json:"stuck_since,omitempty"`
|
||||
}
|
||||
|
||||
// SilentSuccessEvidenceDTO is one finished/processed/sessionless run.
|
||||
type SilentSuccessEvidenceDTO struct {
|
||||
RunID string `json:"run_id"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
// EvidenceListDTO wraps a capped list with its untruncated total
|
||||
// (AC-004.9): "capped" is true exactly when total > len(rows).
|
||||
type triggerEvidenceListDTO struct {
|
||||
Rows []TriggerEvidenceDTO `json:"rows"`
|
||||
Total int `json:"total"`
|
||||
Cap int `json:"cap"`
|
||||
Capped bool `json:"capped"`
|
||||
}
|
||||
|
||||
type stuckRunEvidenceListDTO struct {
|
||||
Rows []StuckRunEvidenceDTO `json:"rows"`
|
||||
Total int `json:"total"`
|
||||
Cap int `json:"cap"`
|
||||
Capped bool `json:"capped"`
|
||||
}
|
||||
|
||||
type silentSuccessEvidenceListDTO struct {
|
||||
Rows []SilentSuccessEvidenceDTO `json:"rows"`
|
||||
Total int `json:"total"`
|
||||
Cap int `json:"cap"`
|
||||
Capped bool `json:"capped"`
|
||||
}
|
||||
|
||||
// LoopHealthThresholdsDTO echoes every threshold and the evaluation
|
||||
// window used to reach the verdict (AC-004.8), in seconds so the
|
||||
// response carries no ambiguous duration string.
|
||||
type LoopHealthThresholdsDTO struct {
|
||||
TriggerOverdueGraceSeconds int `json:"trigger_overdue_grace_seconds"`
|
||||
TriggerStrandedGraceSeconds int `json:"trigger_stranded_grace_seconds"`
|
||||
RunQueuedGraceSeconds int `json:"run_queued_grace_seconds"`
|
||||
RunClaimedGraceSeconds int `json:"run_claimed_grace_seconds"`
|
||||
EvaluationWindowSeconds int `json:"evaluation_window_seconds"`
|
||||
EvidenceCap int `json:"evidence_cap"`
|
||||
}
|
||||
|
||||
// LoopHealthResponse is the GET .../loop-health payload.
|
||||
type LoopHealthResponse struct {
|
||||
Verdict string `json:"verdict"`
|
||||
EvaluatedAt time.Time `json:"evaluated_at"`
|
||||
Thresholds LoopHealthThresholdsDTO `json:"thresholds"`
|
||||
EligibleTriggerCount int `json:"eligible_trigger_count"`
|
||||
Triggers triggerEvidenceListDTO `json:"triggers"`
|
||||
StuckRuns stuckRunEvidenceListDTO `json:"stuck_runs"`
|
||||
SilentSuccesses silentSuccessEvidenceListDTO `json:"silent_successes"`
|
||||
TerminalShapeCounts map[string]int `json:"terminal_shape_counts"`
|
||||
EvaluationWindowStart time.Time `json:"evaluation_window_start"`
|
||||
}
|
||||
|
||||
// allTerminalShapes lists every TerminalShape so the response
|
||||
// zero-fills unobserved shapes rather than omitting their key,
|
||||
// mirroring the counter read's zero-fill-by-name rule.
|
||||
var allTerminalShapes = []service.TerminalShape{
|
||||
service.ShapePreActivation,
|
||||
service.ShapeLaunchedCompleted,
|
||||
service.ShapeLaunchedFailed,
|
||||
service.ShapeSilentSuccess,
|
||||
service.ShapeUnlaunchedSkipped,
|
||||
service.ShapeUnlaunchedFailed,
|
||||
service.ShapeUnclassified,
|
||||
}
|
||||
|
||||
// EvaluateLoopHealth answers "is the loop alive" for one workspace
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-004). now is captured once by the caller
|
||||
// and passed to every predicate so the trigger, run and terminal reads
|
||||
// cannot disagree about the instant they were evaluated against
|
||||
// (AC-004.14). Side-effect free (AC-004.11): every call is a read.
|
||||
//
|
||||
// On any unreadable input, returns a *LoopHealthDegradedError naming
|
||||
// the failing input rather than a partial verdict (AC-004.10).
|
||||
func EvaluateLoopHealth(
|
||||
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
|
||||
) (*LoopHealthResponse, error) {
|
||||
thresholds := LoopHealthThresholdsDTO{
|
||||
TriggerOverdueGraceSeconds: int(officeLoopTriggerOverdueGrace.Seconds()),
|
||||
TriggerStrandedGraceSeconds: int(officeLoopTriggerStrandedGrace.Seconds()),
|
||||
RunQueuedGraceSeconds: int(officeLoopRunQueuedGrace.Seconds()),
|
||||
RunClaimedGraceSeconds: int(officeLoopRunClaimedGrace.Seconds()),
|
||||
EvaluationWindowSeconds: int(officeLoopEvaluationWindow.Seconds()),
|
||||
EvidenceCap: officeLoopEvidenceCap,
|
||||
}
|
||||
windowStart := now.Add(-officeLoopEvaluationWindow)
|
||||
|
||||
activationAt, published, err := repo.LoopLivenessActivationChecked()
|
||||
if err != nil {
|
||||
return nil, &LoopHealthDegradedError{Reason: ReasonActivationReadFailed, Err: err}
|
||||
}
|
||||
if !published {
|
||||
return &LoopHealthResponse{
|
||||
Verdict: VerdictUnknown,
|
||||
EvaluatedAt: now,
|
||||
Thresholds: thresholds,
|
||||
Triggers: triggerEvidenceListDTO{Rows: []TriggerEvidenceDTO{}, Cap: officeLoopEvidenceCap},
|
||||
StuckRuns: stuckRunEvidenceListDTO{Rows: []StuckRunEvidenceDTO{}, Cap: officeLoopEvidenceCap},
|
||||
SilentSuccesses: silentSuccessEvidenceListDTO{Rows: []SilentSuccessEvidenceDTO{}, Cap: officeLoopEvidenceCap},
|
||||
TerminalShapeCounts: zeroFilledShapeCounts(),
|
||||
EvaluationWindowStart: windowStart,
|
||||
}, nil
|
||||
}
|
||||
|
||||
triggerEvidence, eligibleCount, err := evaluateLoopHealthTriggers(ctx, repo, workspaceID, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stuckEvidence, err := evaluateLoopHealthStuckRuns(ctx, repo, workspaceID, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
silentEvidence, err := evaluateLoopHealthSilentSuccesses(ctx, repo, workspaceID, windowStart, activationAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shapeCounts, err := evaluateLoopHealthTerminalShapes(ctx, repo, workspaceID, windowStart, activationAt, published)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
verdict := decideLoopHealthVerdict(triggerEvidence.Total, eligibleCount, stuckEvidence.Total, silentEvidence.Total)
|
||||
|
||||
return &LoopHealthResponse{
|
||||
Verdict: verdict,
|
||||
EvaluatedAt: now,
|
||||
Thresholds: thresholds,
|
||||
EligibleTriggerCount: eligibleCount,
|
||||
Triggers: triggerEvidence,
|
||||
StuckRuns: stuckEvidence,
|
||||
SilentSuccesses: silentEvidence,
|
||||
TerminalShapeCounts: shapeCounts,
|
||||
EvaluationWindowStart: windowStart,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// decideLoopHealthVerdict applies the AC-004.2 precedence
|
||||
// unknown -> dead -> degraded -> not_armed -> healthy. unknown is
|
||||
// decided by the caller before evidence is even read, so this
|
||||
// function only sees the four remaining branches.
|
||||
func decideLoopHealthVerdict(triggerTotal, eligibleCount, stuckTotal, silentTotal int) string {
|
||||
switch {
|
||||
case triggerTotal > 0:
|
||||
return VerdictDead
|
||||
case stuckTotal > 0 || silentTotal > 0:
|
||||
return VerdictDegraded
|
||||
case eligibleCount == 0:
|
||||
return VerdictNotArmed
|
||||
default:
|
||||
return VerdictHealthy
|
||||
}
|
||||
}
|
||||
|
||||
func evaluateLoopHealthTriggers(
|
||||
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
|
||||
) (triggerEvidenceListDTO, int, error) {
|
||||
eligibleCount, err := repo.CountEligibleTriggers(ctx, workspaceID)
|
||||
if err != nil {
|
||||
return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err}
|
||||
}
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, workspaceID, now, officeLoopTriggerOverdueGrace, officeLoopTriggerStrandedGrace, officeLoopEvidenceCap,
|
||||
)
|
||||
if err != nil {
|
||||
return triggerEvidenceListDTO{}, 0, &LoopHealthDegradedError{Reason: ReasonTriggerReadFailed, Err: err}
|
||||
}
|
||||
dtos := make([]TriggerEvidenceDTO, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
dtos = append(dtos, TriggerEvidenceDTO{
|
||||
TriggerID: r.TriggerID,
|
||||
RoutineID: r.RoutineID,
|
||||
RoutineName: r.RoutineName,
|
||||
Condition: r.Condition,
|
||||
NextRunAt: r.NextRunAt,
|
||||
LastFiredAt: r.LastFiredAt,
|
||||
})
|
||||
}
|
||||
return triggerEvidenceListDTO{
|
||||
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
|
||||
}, eligibleCount, nil
|
||||
}
|
||||
|
||||
func evaluateLoopHealthStuckRuns(
|
||||
ctx context.Context, repo LoopHealthRepo, workspaceID string, now time.Time,
|
||||
) (stuckRunEvidenceListDTO, error) {
|
||||
rows, total, err := repo.ListStuckRuns(
|
||||
ctx, workspaceID, now, officeLoopRunQueuedGrace, officeLoopRunClaimedGrace, officeLoopEvidenceCap,
|
||||
)
|
||||
if err != nil {
|
||||
return stuckRunEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonRunReadFailed, Err: err}
|
||||
}
|
||||
dtos := make([]StuckRunEvidenceDTO, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
dtos = append(dtos, StuckRunEvidenceDTO{
|
||||
RunID: r.RunID, AgentProfileID: r.AgentProfileID, Status: r.Status,
|
||||
Condition: r.Condition, StuckSince: r.StuckSince,
|
||||
})
|
||||
}
|
||||
return stuckRunEvidenceListDTO{
|
||||
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// evaluateLoopHealthSilentSuccesses reports ReasonTerminalReadFailed
|
||||
// on failure, not ReasonRunReadFailed — OPERATOR DECISION F31.
|
||||
func evaluateLoopHealthSilentSuccesses(
|
||||
ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart, activationAt time.Time,
|
||||
) (silentSuccessEvidenceListDTO, error) {
|
||||
rows, total, err := repo.ListSilentSuccesses(ctx, workspaceID, windowStart, activationAt, officeLoopEvidenceCap)
|
||||
if err != nil {
|
||||
return silentSuccessEvidenceListDTO{}, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err}
|
||||
}
|
||||
dtos := make([]SilentSuccessEvidenceDTO, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
dtos = append(dtos, SilentSuccessEvidenceDTO{
|
||||
RunID: r.RunID, RequestedAt: r.RequestedAt, FinishedAt: r.FinishedAt,
|
||||
})
|
||||
}
|
||||
return silentSuccessEvidenceListDTO{
|
||||
Rows: dtos, Total: total, Cap: officeLoopEvidenceCap, Capped: total > len(dtos),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func evaluateLoopHealthTerminalShapes(
|
||||
ctx context.Context, repo LoopHealthRepo, workspaceID string, windowStart time.Time,
|
||||
activationAt time.Time, published bool,
|
||||
) (map[string]int, error) {
|
||||
rows, err := repo.ListTerminalRunsInWindow(ctx, workspaceID, windowStart)
|
||||
if err != nil {
|
||||
return nil, &LoopHealthDegradedError{Reason: ReasonTerminalReadFailed, Err: err}
|
||||
}
|
||||
counts := zeroFilledShapeCounts()
|
||||
for _, r := range rows {
|
||||
shape := service.ClassifyTerminalRun(r.Status, r.Outcome, r.SessionID, r.RequestedAt, activationAt, published)
|
||||
counts[string(shape)]++
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
func zeroFilledShapeCounts() map[string]int {
|
||||
counts := make(map[string]int, len(allTerminalShapes))
|
||||
for _, s := range allTerminalShapes {
|
||||
counts[string(s)] = 0
|
||||
}
|
||||
return counts
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package dashboard_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"expvar"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// degradedExpvarInt reads one office_loop_liveness_degraded_total cell
|
||||
// by its full label key, mirroring the terminalShapeExpvarInt helper
|
||||
// pattern used for the same expvar.Map shape elsewhere in this suite.
|
||||
func degradedExpvarInt(t *testing.T, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_loop_liveness_degraded_total")
|
||||
if v == nil {
|
||||
t.Fatal("expvar map office_loop_liveness_degraded_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatal("office_loop_liveness_degraded_total is not a Map")
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("office_loop_liveness_degraded_total[%q] is not an Int", key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// AC-003.4: the loop-health and loop-counters routes are reachable
|
||||
// with no dev-mode/pprof env var set — newTestDeps's router mounts
|
||||
// them with no such gate, matching production registration.
|
||||
func TestGetLoopHealth_ReachableWithoutDevMode(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
deps.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLoopCounters_ReachableWithoutDevMode(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/ws-route-check/loop-counters", nil)
|
||||
w := httptest.NewRecorder()
|
||||
deps.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.10: a read failure through the real, database-backed
|
||||
// LoopHealthRepo (not the pure-Go fake used by loop_health_test.go)
|
||||
// must reach the HTTP layer as a 503 carrying a reason field, and
|
||||
// must increment office_loop_liveness_degraded_total by that reason
|
||||
// — exercising the full handler wiring, not just EvaluateLoopHealth
|
||||
// in isolation.
|
||||
func TestGetLoopHealth_ReadFailureRespondsServiceUnavailableAndIncrementsCounter(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
wsID := "ws-loop-health-db-closed"
|
||||
key := service.LoopMetricLabel("workspace", wsID, "reason", "activation_read_failed")
|
||||
before := degradedExpvarInt(t, key)
|
||||
|
||||
if err := deps.db.Close(); err != nil {
|
||||
t.Fatalf("close db: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/office/workspaces/"+wsID+"/loop-health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
deps.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want 503: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(w.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode body: %v", err)
|
||||
}
|
||||
if body["reason"] != "activation_read_failed" {
|
||||
t.Errorf("reason = %v, want activation_read_failed", body["reason"])
|
||||
}
|
||||
if _, ok := body["error"]; !ok {
|
||||
t.Errorf("body = %+v, want an error field", body)
|
||||
}
|
||||
|
||||
after := degradedExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Errorf("degraded counter delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
)
|
||||
|
||||
// fakeLoopHealthRepo is a hand-rolled LoopHealthRepo for exercising
|
||||
// EvaluateLoopHealth's verdict precedence and failure-reason mapping
|
||||
// without a database.
|
||||
type fakeLoopHealthRepo struct {
|
||||
activationAt time.Time
|
||||
activationPublished bool
|
||||
activationErr error
|
||||
|
||||
eligibleCount int
|
||||
eligibleErr error
|
||||
|
||||
triggerRows []sqlite.TriggerHealthRow
|
||||
triggerTotal int
|
||||
triggerErr error
|
||||
|
||||
stuckRows []sqlite.StuckRunRow
|
||||
stuckTotal int
|
||||
stuckErr error
|
||||
|
||||
silentRows []sqlite.SilentSuccessRow
|
||||
silentTotal int
|
||||
silentErr error
|
||||
|
||||
terminalRows []sqlite.TerminalRunShapeInputRow
|
||||
terminalErr error
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) LoopLivenessActivationChecked() (time.Time, bool, error) {
|
||||
return f.activationAt, f.activationPublished, f.activationErr
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) CountEligibleTriggers(context.Context, string) (int, error) {
|
||||
return f.eligibleCount, f.eligibleErr
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) ListOverdueOrStrandedTriggers(
|
||||
context.Context, string, time.Time, time.Duration, time.Duration, int,
|
||||
) ([]sqlite.TriggerHealthRow, int, error) {
|
||||
return f.triggerRows, f.triggerTotal, f.triggerErr
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) ListStuckRuns(
|
||||
context.Context, string, time.Time, time.Duration, time.Duration, int,
|
||||
) ([]sqlite.StuckRunRow, int, error) {
|
||||
return f.stuckRows, f.stuckTotal, f.stuckErr
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) ListSilentSuccesses(
|
||||
context.Context, string, time.Time, time.Time, int,
|
||||
) ([]sqlite.SilentSuccessRow, int, error) {
|
||||
return f.silentRows, f.silentTotal, f.silentErr
|
||||
}
|
||||
|
||||
func (f *fakeLoopHealthRepo) ListTerminalRunsInWindow(
|
||||
context.Context, string, time.Time,
|
||||
) ([]sqlite.TerminalRunShapeInputRow, error) {
|
||||
return f.terminalRows, f.terminalErr
|
||||
}
|
||||
|
||||
func baseFakeRepo(now time.Time) *fakeLoopHealthRepo {
|
||||
return &fakeLoopHealthRepo{
|
||||
activationAt: now.Add(-48 * time.Hour),
|
||||
activationPublished: true,
|
||||
eligibleCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_UnpublishedActivationIsUnknown(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.activationPublished = false
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictUnknown {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown)
|
||||
}
|
||||
if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 {
|
||||
t.Errorf("expected empty evidence lists when unknown, got %+v", resp)
|
||||
}
|
||||
for shape, count := range resp.TerminalShapeCounts {
|
||||
if count != 0 {
|
||||
t.Errorf("shape %q = %d, want 0 (zero-filled)", shape, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_OverdueTriggerIsDead(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}}
|
||||
repo.triggerTotal = 1
|
||||
// Even with a stuck run also present, dead outranks degraded.
|
||||
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1"}}
|
||||
repo.stuckTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictDead {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDead)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_StuckRunIsDegraded(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}}
|
||||
repo.stuckTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictDegraded {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_SilentSuccessIsDegraded(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.silentRows = []sqlite.SilentSuccessRow{{RunID: "r1", RequestedAt: now.Add(-time.Hour)}}
|
||||
repo.silentTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictDegraded {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_NoEligibleTriggerIsNotArmed(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.eligibleCount = 0
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-empty", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictNotArmed {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictNotArmed)
|
||||
}
|
||||
if len(resp.Triggers.Rows) != 0 || len(resp.StuckRuns.Rows) != 0 || len(resp.SilentSuccesses.Rows) != 0 {
|
||||
t.Errorf("expected empty evidence for a workspace with nothing wrong, got %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_NothingWrongIsHealthy(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictHealthy {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictHealthy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_EchoesThresholds(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
want := LoopHealthThresholdsDTO{
|
||||
TriggerOverdueGraceSeconds: 180,
|
||||
TriggerStrandedGraceSeconds: 300,
|
||||
RunQueuedGraceSeconds: 120,
|
||||
RunClaimedGraceSeconds: 600,
|
||||
EvaluationWindowSeconds: 86400,
|
||||
EvidenceCap: 50,
|
||||
}
|
||||
if resp.Thresholds != want {
|
||||
t.Errorf("thresholds = %+v, want %+v", resp.Thresholds, want)
|
||||
}
|
||||
wantWindowStart := now.Add(-24 * time.Hour)
|
||||
if !resp.EvaluationWindowStart.Equal(wantWindowStart) {
|
||||
t.Errorf("window start = %v, want %v", resp.EvaluationWindowStart, wantWindowStart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_ClassifiesTerminalRunsIntoShapeCounts(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
processed := "processed"
|
||||
repo.terminalRows = []sqlite.TerminalRunShapeInputRow{
|
||||
{Status: "finished", Outcome: &processed, SessionID: "s1", RequestedAt: now.Add(-time.Hour)},
|
||||
{Status: "finished", Outcome: &processed, SessionID: "", RequestedAt: now.Add(-time.Hour)},
|
||||
}
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.TerminalShapeCounts["launched_completed"] != 1 {
|
||||
t.Errorf("launched_completed = %d, want 1", resp.TerminalShapeCounts["launched_completed"])
|
||||
}
|
||||
if resp.TerminalShapeCounts["silent_success"] != 1 {
|
||||
t.Errorf("silent_success = %d, want 1", resp.TerminalShapeCounts["silent_success"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_ActivationReadFailureDegrades(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.activationErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonActivationReadFailed)
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_TriggerReadFailureDegrades(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.triggerErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonTriggerReadFailed)
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_EligibleTriggerCountReadFailureDegrades(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.eligibleErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonTriggerReadFailed)
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_StuckRunReadFailureDegrades(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.stuckErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonRunReadFailed)
|
||||
}
|
||||
|
||||
// OPERATOR DECISION F31: both the silent-success read and the
|
||||
// terminal-shape read map to ReasonTerminalReadFailed.
|
||||
func TestEvaluateLoopHealth_SilentSuccessReadFailureDegradesWithTerminalReason(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.silentErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonTerminalReadFailed)
|
||||
}
|
||||
|
||||
func TestEvaluateLoopHealth_TerminalShapeReadFailureDegradesWithTerminalReason(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.terminalErr = errors.New("boom")
|
||||
|
||||
_, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
assertDegradedReason(t, err, ReasonTerminalReadFailed)
|
||||
}
|
||||
|
||||
// AC-004.2: unknown outranks dead. An unpublished activation must win
|
||||
// even when overdue-trigger evidence (the dead condition) is also
|
||||
// present in the same read.
|
||||
func TestEvaluateLoopHealth_UnknownOutranksDead(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.activationPublished = false
|
||||
repo.triggerRows = []sqlite.TriggerHealthRow{{TriggerID: "t1", Condition: "overdue"}}
|
||||
repo.triggerTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictUnknown {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictUnknown)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.2: degraded outranks not_armed. Zero eligible triggers (the
|
||||
// not_armed condition) must not suppress a stuck run already in
|
||||
// flight from before the workspace went unarmed.
|
||||
func TestEvaluateLoopHealth_DegradedOutranksNotArmed(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
repo.eligibleCount = 0
|
||||
repo.stuckRows = []sqlite.StuckRunRow{{RunID: "r1", Status: "claimed"}}
|
||||
repo.stuckTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
if resp.Verdict != VerdictDegraded {
|
||||
t.Errorf("verdict = %q, want %q", resp.Verdict, VerdictDegraded)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.7: evidence rows carry the offending row's identifying data,
|
||||
// not just a nonzero count that flips the verdict.
|
||||
func TestEvaluateLoopHealth_EvidenceRowsArePopulated(t *testing.T) {
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
repo := baseFakeRepo(now)
|
||||
nextRunAt := now.Add(time.Hour)
|
||||
repo.triggerRows = []sqlite.TriggerHealthRow{{
|
||||
TriggerID: "trig-1", RoutineID: "rout-1", RoutineName: "Nightly sync",
|
||||
Condition: "overdue", NextRunAt: &nextRunAt,
|
||||
}}
|
||||
repo.triggerTotal = 1
|
||||
stuckSince := now.Add(-20 * time.Minute)
|
||||
repo.stuckRows = []sqlite.StuckRunRow{{
|
||||
RunID: "run-1", AgentProfileID: "agent-1", Status: "claimed",
|
||||
Condition: "claimed_stuck", StuckSince: &stuckSince,
|
||||
}}
|
||||
repo.stuckTotal = 1
|
||||
repo.silentRows = []sqlite.SilentSuccessRow{{
|
||||
RunID: "run-2", RequestedAt: now.Add(-time.Hour),
|
||||
}}
|
||||
repo.silentTotal = 1
|
||||
|
||||
resp, err := EvaluateLoopHealth(context.Background(), repo, "ws-a", now)
|
||||
if err != nil {
|
||||
t.Fatalf("evaluate: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Triggers.Rows) != 1 {
|
||||
t.Fatalf("trigger rows = %d, want 1", len(resp.Triggers.Rows))
|
||||
}
|
||||
gotTrigger := resp.Triggers.Rows[0]
|
||||
if gotTrigger.TriggerID != "trig-1" || gotTrigger.RoutineID != "rout-1" ||
|
||||
gotTrigger.RoutineName != "Nightly sync" || gotTrigger.Condition != "overdue" {
|
||||
t.Errorf("trigger row = %+v, want trig-1/rout-1/Nightly sync/overdue", gotTrigger)
|
||||
}
|
||||
if gotTrigger.NextRunAt == nil || !gotTrigger.NextRunAt.Equal(nextRunAt) {
|
||||
t.Errorf("trigger row next_run_at = %v, want %v", gotTrigger.NextRunAt, nextRunAt)
|
||||
}
|
||||
|
||||
if len(resp.StuckRuns.Rows) != 1 {
|
||||
t.Fatalf("stuck run rows = %d, want 1", len(resp.StuckRuns.Rows))
|
||||
}
|
||||
gotStuck := resp.StuckRuns.Rows[0]
|
||||
if gotStuck.RunID != "run-1" || gotStuck.AgentProfileID != "agent-1" ||
|
||||
gotStuck.Status != "claimed" || gotStuck.Condition != "claimed_stuck" {
|
||||
t.Errorf("stuck run row = %+v, want run-1/agent-1/claimed/claimed_stuck", gotStuck)
|
||||
}
|
||||
if gotStuck.StuckSince == nil || !gotStuck.StuckSince.Equal(stuckSince) {
|
||||
t.Errorf("stuck run row stuck_since = %v, want %v", gotStuck.StuckSince, stuckSince)
|
||||
}
|
||||
|
||||
if len(resp.SilentSuccesses.Rows) != 1 {
|
||||
t.Fatalf("silent success rows = %d, want 1", len(resp.SilentSuccesses.Rows))
|
||||
}
|
||||
if got := resp.SilentSuccesses.Rows[0]; got.RunID != "run-2" {
|
||||
t.Errorf("silent success row = %+v, want run-2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func assertDegradedReason(t *testing.T, err error, wantReason string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatal("expected an error, got nil")
|
||||
}
|
||||
var degraded *LoopHealthDegradedError
|
||||
if !errors.As(err, °raded) {
|
||||
t.Fatalf("error = %v, want *LoopHealthDegradedError", err)
|
||||
}
|
||||
if degraded.Reason != wantReason {
|
||||
t.Errorf("reason = %q, want %q", degraded.Reason, wantReason)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
taskmodels "github.com/kandev/kandev/internal/task/models"
|
||||
workflowmodels "github.com/kandev/kandev/internal/workflow/models"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
@@ -99,8 +100,12 @@ type Repository interface {
|
||||
GetTaskWorkflowStepID(ctx context.Context, taskID string) (string, error)
|
||||
// CancelDisplacedParticipantRun cancels the run(s) the step-entry
|
||||
// fan-out queued for agentProfileID at (taskID, stepID). Used after a
|
||||
// claim displaces an agent from a role.
|
||||
CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error)
|
||||
// claim displaces an agent from a role. Returns the rows actually
|
||||
// cancelled so the caller can record each one's loop-liveness
|
||||
// terminal shape.
|
||||
CancelDisplacedParticipantRun(
|
||||
ctx context.Context, taskID, stepID, agentProfileID string,
|
||||
) ([]runssqlite.CancelledRun, error)
|
||||
IsTaskWorkflowStepTerminal(ctx context.Context, taskID string) (terminal, hasStep bool, err error)
|
||||
}
|
||||
|
||||
@@ -152,6 +157,17 @@ type RetryCanceller interface {
|
||||
CancelPendingRetriesForTask(ctx context.Context, taskID string) error
|
||||
}
|
||||
|
||||
// TerminalShapeRecorder classifies and counts a loop-liveness terminal
|
||||
// shape (office_loop_terminal_total, REQ-OFFICE-LOOP-LIVENESS-005) for
|
||||
// runs the dashboard package cancelled directly. Mirrors the RetryCanceller
|
||||
// / TaskCanceller pattern above: the classification logic (activation
|
||||
// instant, workspace resolution) is stateful and already lives once on
|
||||
// *office/service.Service, so this seam reuses it instead of a second,
|
||||
// divergence-prone copy in dashboard.
|
||||
type TerminalShapeRecorder interface {
|
||||
RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun)
|
||||
}
|
||||
|
||||
// ProjectBudgetEvaluator evaluates project-scoped budget policies for a
|
||||
// destination project. Wired to costs.CostService.EvaluateProjectBudget so
|
||||
// reassigning a task into a project re-checks that project's policies —
|
||||
@@ -376,34 +392,35 @@ type WorkspaceSettings struct {
|
||||
|
||||
// DashboardService provides dashboard, inbox, activity, run, and task-search business logic.
|
||||
type DashboardService struct {
|
||||
repo Repository
|
||||
logger *logger.Logger
|
||||
activity shared.ActivityLogger
|
||||
agents shared.AgentReader
|
||||
costs shared.CostChecker
|
||||
permissions shared.PermissionLister // optional; nil means no permission items in inbox
|
||||
settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable
|
||||
governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable
|
||||
eb bus.EventBus // optional; nil means no events are published
|
||||
retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign
|
||||
taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled
|
||||
taskDetacher TaskDetacher // optional; canonical empty-parent mutation
|
||||
taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated
|
||||
sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal
|
||||
reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations
|
||||
engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine
|
||||
approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs
|
||||
skillLister SkillLister // optional; nil means skill_count is always 0
|
||||
routineLister RoutineLister // optional; nil means routine_count is always 0
|
||||
failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries
|
||||
failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources
|
||||
markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint
|
||||
decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints
|
||||
routingProvider RoutingProvider // optional; nil disables /routing endpoints (503)
|
||||
attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses
|
||||
runResolver RunResolver // optional; nil means status-change activity rows have no run_id
|
||||
assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization
|
||||
projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies
|
||||
repo Repository
|
||||
logger *logger.Logger
|
||||
activity shared.ActivityLogger
|
||||
agents shared.AgentReader
|
||||
costs shared.CostChecker
|
||||
permissions shared.PermissionLister // optional; nil means no permission items in inbox
|
||||
settingsProvider SettingsProvider // optional; nil means settings endpoints are unavailable
|
||||
governanceStore GovernanceSettingsStore // optional; nil means governance settings are unavailable
|
||||
eb bus.EventBus // optional; nil means no events are published
|
||||
retryCanceller RetryCanceller // optional; nil means retries are not cancelled on reassign
|
||||
terminalShapeRecorder TerminalShapeRecorder // optional; nil means displaced-run cancellations aren't counted in office_loop_terminal_total
|
||||
taskCanceller TaskCanceller // optional; used to hard-cancel sessions on status→cancelled
|
||||
taskDetacher TaskDetacher // optional; canonical empty-parent mutation
|
||||
taskLifecycle TaskLifecyclePublisher // optional; nil means status changes don't publish canonical task.updated
|
||||
sessionTerm SessionTerminator // optional; flips office session rows to COMPLETED on participation removal
|
||||
reactivity ReactivityApplier // optional; runs the office reactivity pipeline on mutations
|
||||
engineDispatcher shared.WorkflowEngineDispatcher // optional; synchronously routes comment triggers through the engine
|
||||
approvalQueuer ApprovalReactivityQueuer // optional; queues approval-flow runs
|
||||
skillLister SkillLister // optional; nil means skill_count is always 0
|
||||
routineLister RoutineLister // optional; nil means routine_count is always 0
|
||||
failureNotifier FailureNotifier // optional; nil means assignee changes don't auto-dismiss inbox entries
|
||||
failureInbox FailureInboxSource // optional; nil disables the new agent_run_failed / agent_paused_after_failures inbox sources
|
||||
markFixed MarkFixedHandler // optional; nil disables the dismiss endpoint
|
||||
decisions DecisionStore // workflow-domain decisions store (ADR 0005 Wave E); nil disables decision endpoints
|
||||
routingProvider RoutingProvider // optional; nil disables /routing endpoints (503)
|
||||
attemptLister RouteAttemptLister // optional; nil disables attempt embedding on run-detail responses
|
||||
runResolver RunResolver // optional; nil means status-change activity rows have no run_id
|
||||
assigneeWriter HumanAssigneeWriter // optional; nil rejects human-assignee writes rather than skipping authorization
|
||||
projectBudget ProjectBudgetEvaluator // optional; nil means reassignment doesn't re-evaluate the destination project's budget policies
|
||||
// officeSessionIdentity gates RecordAgentDecision's use of the caller's
|
||||
// own session id. Defaults false (zero value); set via
|
||||
// SetOfficeSessionIdentity, wired from features.officeSessionIdentity.
|
||||
@@ -516,6 +533,13 @@ func (s *DashboardService) SetRetryCanceller(c RetryCanceller) {
|
||||
s.retryCanceller = c
|
||||
}
|
||||
|
||||
// SetTerminalShapeRecorder wires the seam used to count a loop-liveness
|
||||
// terminal shape for a run this package cancelled directly (a displaced
|
||||
// participant's queued run).
|
||||
func (s *DashboardService) SetTerminalShapeRecorder(r TerminalShapeRecorder) {
|
||||
s.terminalShapeRecorder = r
|
||||
}
|
||||
|
||||
// SetRunResolver wires the seam used to attribute status-change activity
|
||||
// rows back to the office run that produced them. Optional; when unset,
|
||||
// rows are logged with an empty run_id.
|
||||
|
||||
@@ -501,12 +501,17 @@ func (s *DashboardService) cancelDisplacedRun(ctx context.Context, taskID, stepI
|
||||
if displacedAgentID == "" {
|
||||
return
|
||||
}
|
||||
if _, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID); err != nil {
|
||||
cancelled, err := s.repo.CancelDisplacedParticipantRun(ctx, taskID, stepID, displacedAgentID)
|
||||
if err != nil {
|
||||
s.logger.Warn("cancel displaced participant run failed",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("step_id", stepID),
|
||||
zap.String("agent_profile_id", displacedAgentID),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
if s.terminalShapeRecorder != nil {
|
||||
s.terminalShapeRecorder.RecordCancelledRunTerminalShapes(ctx, cancelled)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package dashboard_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/dashboard"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
// recordingTerminalShapeRecorder captures RecordCancelledRunTerminalShapes
|
||||
// calls so tests can verify a displaced-participant cancellation reaches
|
||||
// office_loop_terminal_total's classification path.
|
||||
type recordingTerminalShapeRecorder struct {
|
||||
calls [][]runssqlite.CancelledRun
|
||||
}
|
||||
|
||||
func (r *recordingTerminalShapeRecorder) RecordCancelledRunTerminalShapes(
|
||||
_ context.Context, cancelled []runssqlite.CancelledRun,
|
||||
) {
|
||||
r.calls = append(r.calls, cancelled)
|
||||
}
|
||||
|
||||
var _ dashboard.TerminalShapeRecorder = (*recordingTerminalShapeRecorder)(nil)
|
||||
|
||||
// TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape is
|
||||
// the Testing round 3 regression test. cancelDisplacedRun calls
|
||||
// repo.CancelDisplacedParticipantRun directly, which bypassed
|
||||
// office_loop_terminal_total entirely — the same bypass class Review
|
||||
// round 1 fixed for the single-run CancelRun call sites. The setup mirrors
|
||||
// TestAddTaskReviewer_ClaimingAutoSeatCancelsDisplacedRunAndLogsActivity;
|
||||
// this test additionally asserts the terminal-shape recorder is invoked
|
||||
// with the displaced run's cancelled row.
|
||||
func TestAddTaskReviewer_ClaimingAutoSeatRecordsDisplacedRunTerminalShape(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
recorder := &recordingTerminalShapeRecorder{}
|
||||
deps.svc.SetTerminalShapeRecorder(recorder)
|
||||
insertTestTask(t, deps.db, "claim-shape1", "ws-claim-shape", "C", "todo", 2)
|
||||
stepID := "step-claim-shape1"
|
||||
|
||||
if _, err := deps.db.Exec(`
|
||||
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
|
||||
VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape', '', datetime('now'), datetime('now'))
|
||||
`); err != nil {
|
||||
t.Fatalf("seed claiming agent profile: %v", err)
|
||||
}
|
||||
|
||||
if _, err := deps.db.Exec(`
|
||||
INSERT INTO workflow_step_participants
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance)
|
||||
VALUES ('auto-seat-claim-shape1', ?, 'claim-shape1', 'reviewer', 'agent-auto', 1, 0, 'auto')
|
||||
`, stepID); err != nil {
|
||||
t.Fatalf("seed auto seat: %v", err)
|
||||
}
|
||||
|
||||
run := &models.Run{
|
||||
AgentProfileID: "agent-auto",
|
||||
Reason: "task_assigned",
|
||||
Payload: `{"task_id":"claim-shape1","workflow_step_id":"` + stepID + `"}`,
|
||||
Status: models.RunStatusQueued,
|
||||
CoalescedCount: 1,
|
||||
}
|
||||
if err := deps.repo.CreateRun(context.Background(), run); err != nil {
|
||||
t.Fatalf("seed displaced run: %v", err)
|
||||
}
|
||||
|
||||
body := `{"agent_profile_id":"agent-claiming"}`
|
||||
req := httptest.NewRequest(http.MethodPost,
|
||||
"/api/v1/office/tasks/claim-shape1/reviewers", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
deps.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
if len(recorder.calls) != 1 {
|
||||
t.Fatalf("terminal shape recorder calls = %d, want 1", len(recorder.calls))
|
||||
}
|
||||
if len(recorder.calls[0]) != 1 || recorder.calls[0][0].ID != run.ID {
|
||||
t.Fatalf("recorded cancelled rows = %+v, want [%s]", recorder.calls[0], run.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetTerminalShapeRecorder_NilTolerated keeps the wiring optional: when
|
||||
// no recorder is registered, the displaced-run cancellation itself must
|
||||
// still succeed (dashboard tests run without the office service wired in).
|
||||
func TestSetTerminalShapeRecorder_NilTolerated(t *testing.T) {
|
||||
deps := newTestDeps(t)
|
||||
insertTestTask(t, deps.db, "claim-shape-nil", "ws-claim-shape-nil", "C", "todo", 2)
|
||||
stepID := "step-claim-shape-nil"
|
||||
|
||||
if _, err := deps.db.Exec(`
|
||||
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
|
||||
VALUES ('agent-claiming', '', 'agent-claiming', 'agent-claiming', 'ws-claim-shape-nil', '', datetime('now'), datetime('now'))
|
||||
`); err != nil {
|
||||
t.Fatalf("seed claiming agent profile: %v", err)
|
||||
}
|
||||
if _, err := deps.db.Exec(`
|
||||
INSERT INTO workflow_step_participants
|
||||
(id, step_id, task_id, role, agent_profile_id, decision_required, position, provenance)
|
||||
VALUES ('auto-seat-claim-shape-nil', ?, 'claim-shape-nil', 'reviewer', 'agent-auto', 1, 0, 'auto')
|
||||
`, stepID); err != nil {
|
||||
t.Fatalf("seed auto seat: %v", err)
|
||||
}
|
||||
run := &models.Run{
|
||||
AgentProfileID: "agent-auto",
|
||||
Reason: "task_assigned",
|
||||
Payload: `{"task_id":"claim-shape-nil","workflow_step_id":"` + stepID + `"}`,
|
||||
Status: models.RunStatusQueued,
|
||||
CoalescedCount: 1,
|
||||
}
|
||||
if err := deps.repo.CreateRun(context.Background(), run); err != nil {
|
||||
t.Fatalf("seed displaced run: %v", err)
|
||||
}
|
||||
|
||||
body := `{"agent_profile_id":"agent-claiming"}`
|
||||
req := httptest.NewRequest(http.MethodPost,
|
||||
"/api/v1/office/tasks/claim-shape-nil/reviewers", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
deps.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want 201: %s", w.Code, w.Body.String())
|
||||
}
|
||||
gotRun, err := deps.repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get displaced run: %v", err)
|
||||
}
|
||||
if gotRun.Status != "cancelled" {
|
||||
t.Fatalf("displaced run status = %q, want cancelled", gotRun.Status)
|
||||
}
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func TestReconcileAgentWorkingStatus_ClearsTerminalOwner(t *testing.T) {
|
||||
if _, err := repo.MarkAgentWorking(ctx, agent.ID, claimed.ID); err != nil {
|
||||
t.Fatalf("mark agent working: %v", err)
|
||||
}
|
||||
if err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil {
|
||||
if _, err := repo.FinishRun(ctx, claimed.ID, "finished", nil); err != nil {
|
||||
t.Fatalf("finish run: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -485,6 +485,13 @@ type Run struct {
|
||||
// EarliestRetryAt is the earliest moment a parked run should be re-
|
||||
// resolved. Set only when at least one degraded route is auto-retryable.
|
||||
EarliestRetryAt *time.Time `json:"earliest_retry_at,omitempty" db:"earliest_retry_at"`
|
||||
|
||||
// CausationID is copied from the agent_wakeup_requests row that
|
||||
// created this run (REQ-OFFICE-LOOP-LIVENESS-002). "" means
|
||||
// uncorrelated — either a legacy pre-migration row or a run created
|
||||
// off a wake that never carried an id. Never a join/group-by key
|
||||
// without excluding "" first.
|
||||
CausationID string `json:"causation_id,omitempty" db:"causation_id"`
|
||||
}
|
||||
|
||||
// RouteAttempt records one provider attempt inside a Run. Each fallback
|
||||
@@ -598,6 +605,11 @@ type RoutineRun struct {
|
||||
StartedAt *time.Time `json:"started_at" db:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at" db:"completed_at"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
// CausationID is minted once per fire in dispatchRoutineRun
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-002) — the origin id every wakeup
|
||||
// request and run this fire produces carries forward. "" only for
|
||||
// rows written before this feature.
|
||||
CausationID string `json:"causation_id,omitempty" db:"causation_id"`
|
||||
}
|
||||
|
||||
// ApprovalType constants for approval request types.
|
||||
|
||||
@@ -226,7 +226,7 @@ func TestMarkAgentWorking_SuccessorTakeover(t *testing.T) {
|
||||
|
||||
// Run A leaves 'claimed' (finishes) but its own clearAgentWorking has not
|
||||
// run yet — the un-transacted window handleAgentCompleted leaves open.
|
||||
if err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil {
|
||||
if _, err := repo.FinishRun(ctx, claimedA.ID, string(models.RunStatusFinished), nil); err != nil {
|
||||
t.Fatalf("finish run A: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,7 @@ func (r *Repository) initSchema() error {
|
||||
return fmt.Errorf("required office migration: %w", err)
|
||||
}
|
||||
r.activateRunOutcome()
|
||||
r.activateLoopLiveness()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -413,6 +414,9 @@ func (r *Repository) createRunTables() error {
|
||||
-- re-deriving it against a context_snapshot a coalesced wakeup
|
||||
-- may have since patched.
|
||||
continuation_scope TEXT NOT NULL DEFAULT '',
|
||||
-- causation_id (office-loop-liveness): copied from the wakeup
|
||||
-- request that created this run. '' means uncorrelated.
|
||||
causation_id TEXT NOT NULL DEFAULT '',
|
||||
requested_at TIMESTAMP NOT NULL,
|
||||
claimed_at TIMESTAMP,
|
||||
finished_at TIMESTAMP
|
||||
@@ -492,6 +496,7 @@ func (r *Repository) createRoutineTables() error {
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
causation_id TEXT NOT NULL DEFAULT '',
|
||||
FOREIGN KEY (routine_id) REFERENCES office_routines(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_office_routine_runs_active_fingerprint
|
||||
@@ -733,7 +738,8 @@ func (r *Repository) createAgentWakeupRequestTable() error {
|
||||
run_id TEXT NOT NULL DEFAULT '',
|
||||
requested_at TIMESTAMP NOT NULL,
|
||||
claimed_at TIMESTAMP,
|
||||
finished_at TIMESTAMP
|
||||
finished_at TIMESTAMP,
|
||||
causation_id TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_wakeup_agent_status ON agent_wakeup_requests(agent_profile_id, status);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_wakeup_idempotency ON agent_wakeup_requests(idempotency_key)
|
||||
|
||||
@@ -62,6 +62,7 @@ func (r *Repository) runMigrations() error {
|
||||
r.migrateParentWakeReceiptColumns()
|
||||
r.migrate.Apply("task_workspace_groups.ownership_generation",
|
||||
`ALTER TABLE task_workspace_groups ADD COLUMN ownership_generation INTEGER NOT NULL DEFAULT 1`)
|
||||
r.migrateLoopLivenessCausationID()
|
||||
if err := r.migrate.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -83,6 +84,35 @@ func (r *Repository) backfillRoutineTriggerTimezones() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateLoopLivenessCausationID adds the causation_id correlation
|
||||
// column to all three rows in the wake-to-run chain
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-002): the routine fire, the wakeup request
|
||||
// it produces, and the run the wakeup request produces. "" rather than
|
||||
// NULL, because every existing row predates this feature and a reader
|
||||
// must treat "legacy" and "uncorrelated" identically. Each column gets
|
||||
// its own partial index (excluding "") so a backward correlation walk
|
||||
// from a run to its originating fire is a direct lookup on every
|
||||
// dialect, not a table scan — NFR-2 is symmetric with the forward walk.
|
||||
func (r *Repository) migrateLoopLivenessCausationID() {
|
||||
_ = r.migrate.Apply("office_routine_runs.causation_id",
|
||||
`ALTER TABLE office_routine_runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
|
||||
_ = r.migrate.Apply("idx_office_routine_runs_causation_id",
|
||||
`CREATE INDEX IF NOT EXISTS idx_office_routine_runs_causation_id
|
||||
ON office_routine_runs(causation_id) WHERE causation_id != ''`)
|
||||
|
||||
_ = r.migrate.Apply("agent_wakeup_requests.causation_id",
|
||||
`ALTER TABLE agent_wakeup_requests ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
|
||||
_ = r.migrate.Apply("idx_agent_wakeup_requests_causation_id",
|
||||
`CREATE INDEX IF NOT EXISTS idx_agent_wakeup_requests_causation_id
|
||||
ON agent_wakeup_requests(causation_id) WHERE causation_id != ''`)
|
||||
|
||||
_ = r.migrate.Apply("runs.causation_id",
|
||||
`ALTER TABLE runs ADD COLUMN causation_id TEXT NOT NULL DEFAULT ''`)
|
||||
_ = r.migrate.Apply("idx_runs_causation_id",
|
||||
`CREATE INDEX IF NOT EXISTS idx_runs_causation_id
|
||||
ON runs(causation_id) WHERE causation_id != ''`)
|
||||
}
|
||||
|
||||
// migrateContinuationScope adds runs.continuation_scope for databases
|
||||
// created before WO-16's claim-time scope persistence. Existing rows receive
|
||||
// a scope from their stored context snapshot so queued or claimed taskless
|
||||
|
||||
@@ -266,20 +266,33 @@ func (r *Repository) SetWorkspaceAgentFailureThreshold(
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkRunFailed sets a run row to status=failed with a verbatim
|
||||
// error message and stamps finished_at. Idempotent: re-marking is a
|
||||
// no-op when the run is already in the same state.
|
||||
// MarkRunFailed sets a run row to status=failed with a verbatim error
|
||||
// message and stamps finished_at. Guarded to status = 'claimed'
|
||||
// (Review round 3, R3-1): every caller reaches this after reading a
|
||||
// claimed run, so without the guard a cancel that commits in between
|
||||
// would have its 'cancelled' status and finished_at overwritten by
|
||||
// this write. Returns wrote=false — the same "nothing changed" signal
|
||||
// runs/repository/sqlite.FinishRun gives its callers — when the row
|
||||
// was no longer claimed, whether because it was already marked failed
|
||||
// or because it reached a different terminal state first.
|
||||
func (r *Repository) MarkRunFailed(
|
||||
ctx context.Context, runID, errorMessage string,
|
||||
) error {
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
) (bool, error) {
|
||||
res, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE runs
|
||||
SET status = 'failed',
|
||||
error_message = ?,
|
||||
finished_at = COALESCE(finished_at, ?)
|
||||
WHERE id = ?
|
||||
WHERE id = ? AND status = 'claimed'
|
||||
`), errorMessage, time.Now().UTC(), runID)
|
||||
return err
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// DismissInboxItem records a per-user dismissal. Idempotent — a second
|
||||
|
||||
@@ -225,9 +225,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
|
||||
"2026-01-02 09:00:00", "2026-01-02 10:00:00", "2026-01-02 11:00:00", "{}")
|
||||
seedSummaryRun(t, repo, "mr-other", "agent-a", "queued", "2026-01-03 09:00:00", "", "", "{}")
|
||||
|
||||
if err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded"); err != nil {
|
||||
wrote, err := repo.MarkRunFailed(ctx, "mr-open", "provider exploded")
|
||||
if err != nil {
|
||||
t.Fatalf("MarkRunFailed: %v", err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Error("wrote = false, want true (mr-open was claimed)")
|
||||
}
|
||||
run, err := repo.GetRun(ctx, "mr-open")
|
||||
if err != nil {
|
||||
t.Fatalf("GetRun: %v", err)
|
||||
@@ -242,10 +246,21 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
|
||||
t.Error("FinishedAt is nil, want it stamped")
|
||||
}
|
||||
|
||||
// Re-marking is idempotent and must not move an existing finished_at.
|
||||
if err := repo.MarkRunFailed(ctx, "mr-done", "later failure"); err != nil {
|
||||
// Review round 3, R3-1: MarkRunFailed is guarded to status = 'claimed',
|
||||
// so a run already in a different terminal state (here, 'finished') is
|
||||
// left untouched — a guarded no-op, not a re-stamp. This is a
|
||||
// deliberate change from the pre-guard "idempotent re-mark" contract:
|
||||
// unconditionally re-marking an already-terminal run is exactly the
|
||||
// TOCTOU bug this guard exists to close (a concurrent cancel/finish
|
||||
// must never have its real terminal state overwritten by a stale
|
||||
// failure write).
|
||||
wrote, err = repo.MarkRunFailed(ctx, "mr-done", "later failure")
|
||||
if err != nil {
|
||||
t.Fatalf("MarkRunFailed (already finished): %v", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Error("wrote = true, want false (mr-done was already finished, not claimed)")
|
||||
}
|
||||
run, err = repo.GetRun(ctx, "mr-done")
|
||||
if err != nil {
|
||||
t.Fatalf("GetRun: %v", err)
|
||||
@@ -253,8 +268,11 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
|
||||
if run.FinishedAt == nil || run.FinishedAt.Format("2006-01-02 15:04:05") != "2026-01-02 11:00:00" {
|
||||
t.Errorf("FinishedAt = %v, want the original 2026-01-02 11:00:00", run.FinishedAt)
|
||||
}
|
||||
if run.ErrorMessage != "later failure" {
|
||||
t.Errorf("ErrorMessage = %q, want it overwritten", run.ErrorMessage)
|
||||
if run.ErrorMessage != "" {
|
||||
t.Errorf("ErrorMessage = %q, want it left untouched (guarded no-op)", run.ErrorMessage)
|
||||
}
|
||||
if run.Status != "finished" {
|
||||
t.Errorf("Status = %q, want finished (unchanged)", run.Status)
|
||||
}
|
||||
|
||||
// An untouched run keeps its status.
|
||||
@@ -266,9 +284,13 @@ func TestMarkRunFailed_StampsMessageAndPreservesFinishedAt(t *testing.T) {
|
||||
}
|
||||
|
||||
// Marking a missing run is a silent no-op, not an error.
|
||||
if err := repo.MarkRunFailed(ctx, "no-such-run", "x"); err != nil {
|
||||
wrote, err = repo.MarkRunFailed(ctx, "no-such-run", "x")
|
||||
if err != nil {
|
||||
t.Errorf("MarkRunFailed(missing) = %v, want nil", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Error("wrote = true, want false (no such run)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboxDismissals_UpsertQueryAndList(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/db/dialect"
|
||||
)
|
||||
|
||||
// TriggerHealthRow is one overdue or stranded cron trigger
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-004). Condition names which predicate
|
||||
// matched so a single evidence list can carry both kinds (AC-004.7).
|
||||
type TriggerHealthRow struct {
|
||||
TriggerID string `db:"id"`
|
||||
RoutineID string `db:"routine_id"`
|
||||
RoutineName string `db:"routine_name"`
|
||||
NextRunAt *time.Time `db:"next_run_at"`
|
||||
LastFiredAt *time.Time `db:"last_fired_at"`
|
||||
Condition string `db:"-"`
|
||||
}
|
||||
|
||||
// CountEligibleTriggers reports how many enabled cron triggers exist
|
||||
// for the workspace, regardless of due state — the input to the
|
||||
// not_armed verdict (AC-004.6): zero means no trigger is scheduled at
|
||||
// all, not that none happen to be due right now.
|
||||
func (r *Repository) CountEligibleTriggers(ctx context.Context, workspaceID string) (int, error) {
|
||||
var count int
|
||||
err := r.ro.GetContext(ctx, &count, r.ro.Rebind(`
|
||||
SELECT COUNT(*) FROM office_routine_triggers t
|
||||
JOIN office_routines rt ON rt.id = t.routine_id
|
||||
WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ?
|
||||
`), workspaceID)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// ListOverdueOrStrandedTriggers returns eligible cron triggers that are
|
||||
// overdue (armed but past grace) or stranded (claimed, or never fired
|
||||
// at all, and not re-armed past grace), capped at limit, with the
|
||||
// untruncated total from the same statement (AC-004.9, AC-004.12). The
|
||||
// stranded clock is the claim instant COALESCE(last_fired_at,
|
||||
// created_at), not last_fired_at alone: a trigger that has never fired
|
||||
// (e.g. a malformed cron_expression that never got a next_run_at)
|
||||
// ages from its own creation, not from the epoch. A trigger claimed
|
||||
// less than strandedGrace ago is a claiming trigger — excluded here
|
||||
// rather than given a third label (AC-004.15).
|
||||
func (r *Repository) ListOverdueOrStrandedTriggers(
|
||||
ctx context.Context, workspaceID string, now time.Time,
|
||||
overdueGrace, strandedGrace time.Duration, limit int,
|
||||
) ([]TriggerHealthRow, int, error) {
|
||||
overdueBefore := now.Add(-overdueGrace)
|
||||
strandedBefore := now.Add(-strandedGrace)
|
||||
|
||||
type row struct {
|
||||
TriggerHealthRow
|
||||
Total int `db:"total"`
|
||||
}
|
||||
var rows []row
|
||||
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
|
||||
SELECT t.id AS id, t.routine_id AS routine_id, rt.name AS routine_name,
|
||||
t.next_run_at AS next_run_at, t.last_fired_at AS last_fired_at,
|
||||
COUNT(*) OVER () AS total
|
||||
FROM office_routine_triggers t
|
||||
JOIN office_routines rt ON rt.id = t.routine_id
|
||||
WHERE t.kind = 'cron' AND t.enabled = 1 AND rt.workspace_id = ?
|
||||
AND (
|
||||
(t.next_run_at IS NOT NULL AND t.next_run_at <= ?)
|
||||
OR (t.next_run_at IS NULL AND COALESCE(t.last_fired_at, t.created_at) <= ?)
|
||||
)
|
||||
ORDER BY (t.next_run_at IS NULL) DESC, t.next_run_at ASC, t.id ASC
|
||||
LIMIT ?
|
||||
`), workspaceID, overdueBefore, strandedBefore, limit)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]TriggerHealthRow, 0, len(rows))
|
||||
total := 0
|
||||
for _, rr := range rows {
|
||||
hr := rr.TriggerHealthRow
|
||||
if hr.NextRunAt == nil {
|
||||
hr.Condition = "stranded"
|
||||
} else {
|
||||
hr.Condition = "overdue"
|
||||
}
|
||||
out = append(out, hr)
|
||||
total = rr.Total
|
||||
}
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// StuckRunRow is one queued or claimed run the scheduler should have
|
||||
// progressed but has not (REQ-OFFICE-LOOP-LIVENESS-004). StuckSince is
|
||||
// the clock the row aged from: claimed_at for a claimed run,
|
||||
// COALESCE(scheduled_retry_at, requested_at) for a queued one.
|
||||
type StuckRunRow struct {
|
||||
RunID string `db:"id"`
|
||||
AgentProfileID string `db:"agent_profile_id"`
|
||||
Status string `db:"status"`
|
||||
StuckSince *time.Time `db:"stuck_since"`
|
||||
Condition string `db:"-"`
|
||||
}
|
||||
|
||||
// stuckRunsUnionSQL is the shared claimed+queued candidate set behind
|
||||
// ListStuckRuns' stuck_since column, unchanged by dialect: both source
|
||||
// expressions (`claimed_at`, `COALESCE(scheduled_retry_at, requested_at)`)
|
||||
// are plain TIMESTAMP-typed columns on both engines.
|
||||
const stuckRunsUnionSQL = `
|
||||
SELECT r.id, r.agent_profile_id, r.status, r.claimed_at AS stuck_since
|
||||
FROM runs r
|
||||
JOIN agent_profiles a ON a.id = r.agent_profile_id
|
||||
WHERE a.workspace_id = ? AND r.status = 'claimed'
|
||||
AND (r.claimed_at IS NULL OR r.claimed_at <= ?)
|
||||
UNION ALL
|
||||
SELECT r.id, r.agent_profile_id, r.status,
|
||||
COALESCE(r.scheduled_retry_at, r.requested_at) AS stuck_since
|
||||
FROM runs r
|
||||
JOIN agent_profiles a ON a.id = r.agent_profile_id
|
||||
WHERE a.workspace_id = ? AND r.status = 'queued'
|
||||
AND r.current_route_attempt_seq = 0
|
||||
AND r.routing_blocked_status IS NULL
|
||||
AND COALESCE(r.scheduled_retry_at, r.requested_at) <= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM runs s
|
||||
WHERE s.agent_profile_id = r.agent_profile_id AND s.status = 'claimed'
|
||||
)
|
||||
`
|
||||
|
||||
// stuckSinceColumnExpr and stuckSinceLayout keep stuck_since portable
|
||||
// across dialects. stuck_since is a computed UNION ALL column, so it
|
||||
// carries no declared type for the driver to auto-parse into
|
||||
// time.Time (unlike a direct table column read): on SQLite, strftime()
|
||||
// normalizes it to a fixed ISO-8601 UTC string that the scan target
|
||||
// (sql.NullString) parses by hand below. Postgres has no strftime;
|
||||
// selecting the raw TIMESTAMP column there returns a native time.Time
|
||||
// that database/sql's Scan already formats into an RFC3339Nano string
|
||||
// on assignment to *sql.NullString, so one scan type and one dialect
|
||||
// switch cover both engines without a second query or row type.
|
||||
func stuckSinceColumnExpr(driver string) string {
|
||||
if dialect.IsPostgres(driver) {
|
||||
return "w.stuck_since"
|
||||
}
|
||||
return "strftime('%Y-%m-%dT%H:%M:%fZ', w.stuck_since)"
|
||||
}
|
||||
|
||||
func stuckSinceLayout(driver string) string {
|
||||
if dialect.IsPostgres(driver) {
|
||||
return time.RFC3339Nano
|
||||
}
|
||||
return "2006-01-02T15:04:05.000Z"
|
||||
}
|
||||
|
||||
// ListStuckRuns returns claimed-stuck and queued-stuck runs in one
|
||||
// list, each row naming which condition it matched, ordered claimed
|
||||
// first then by stuck instant (NULL first), capped with the same-
|
||||
// statement untruncated total (AC-004.9, AC-004.12). Neither predicate
|
||||
// is windowed (AC-004.5): a run stuck longer than
|
||||
// officeLoopEvaluationWindow is the one most worth surfacing.
|
||||
//
|
||||
// The queued half mirrors ClaimNextEligibleRun's own filter
|
||||
// (AC-004.17) as an accepted limitation: it restricts to
|
||||
// current_route_attempt_seq = 0, so a routed run that has dispatched
|
||||
// at least once and requeued reads as silent rather than stuck. That
|
||||
// gap is intentional (OPERATOR DECISION F29/F30) — fixing it here
|
||||
// would let this read disagree with what the scheduler would actually
|
||||
// claim next.
|
||||
func (r *Repository) ListStuckRuns(
|
||||
ctx context.Context, workspaceID string, now time.Time,
|
||||
queuedGrace, claimedGrace time.Duration, limit int,
|
||||
) ([]StuckRunRow, int, error) {
|
||||
claimedBefore := now.Add(-claimedGrace)
|
||||
queuedBefore := now.Add(-queuedGrace)
|
||||
driver := r.ro.DriverName()
|
||||
|
||||
type row struct {
|
||||
ID string `db:"id"`
|
||||
AgentProfileID string `db:"agent_profile_id"`
|
||||
Status string `db:"status"`
|
||||
StuckSince sql.NullString `db:"stuck_since"`
|
||||
Total int `db:"total"`
|
||||
}
|
||||
var rows []row
|
||||
query := fmt.Sprintf(`
|
||||
SELECT w.id AS id, w.agent_profile_id AS agent_profile_id, w.status AS status,
|
||||
%s AS stuck_since,
|
||||
COUNT(*) OVER () AS total
|
||||
FROM (%s) w
|
||||
ORDER BY (w.status = 'claimed') DESC, (w.stuck_since IS NULL) DESC, w.stuck_since ASC, w.id ASC
|
||||
LIMIT ?
|
||||
`, stuckSinceColumnExpr(driver), stuckRunsUnionSQL)
|
||||
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(query),
|
||||
workspaceID, claimedBefore, workspaceID, queuedBefore, limit)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
layout := stuckSinceLayout(driver)
|
||||
out := make([]StuckRunRow, 0, len(rows))
|
||||
total := 0
|
||||
for _, rr := range rows {
|
||||
hr := StuckRunRow{RunID: rr.ID, AgentProfileID: rr.AgentProfileID, Status: rr.Status}
|
||||
if rr.StuckSince.Valid {
|
||||
parsed, perr := time.Parse(layout, rr.StuckSince.String)
|
||||
if perr != nil {
|
||||
return nil, 0, fmt.Errorf("parse stuck_since %q: %w", rr.StuckSince.String, perr)
|
||||
}
|
||||
hr.StuckSince = &parsed
|
||||
}
|
||||
if hr.Status == "claimed" {
|
||||
hr.Condition = "claimed_stuck"
|
||||
} else {
|
||||
hr.Condition = "queued_stuck"
|
||||
}
|
||||
out = append(out, hr)
|
||||
total = rr.Total
|
||||
}
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// SilentSuccessRow is one finished run that asserted success but never
|
||||
// named a session (AC-005.4 made an evidence row).
|
||||
type SilentSuccessRow struct {
|
||||
RunID string `db:"id"`
|
||||
RequestedAt time.Time `db:"requested_at"`
|
||||
FinishedAt *time.Time `db:"finished_at"`
|
||||
}
|
||||
|
||||
// ListSilentSuccesses returns finished/processed/sessionless runs whose
|
||||
// COALESCE(finished_at, requested_at) falls in [windowStart, now),
|
||||
// requested at or after the activation instant (AC-005.6: a run
|
||||
// predating activation is pre_activation, never a silent success),
|
||||
// newest first, capped with the same-statement untruncated total
|
||||
// (AC-004.7, AC-004.9, AC-004.16).
|
||||
func (r *Repository) ListSilentSuccesses(
|
||||
ctx context.Context, workspaceID string, windowStart, activationAt time.Time, limit int,
|
||||
) ([]SilentSuccessRow, int, error) {
|
||||
type row struct {
|
||||
SilentSuccessRow
|
||||
Total int `db:"total"`
|
||||
}
|
||||
var rows []row
|
||||
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
|
||||
SELECT r.id AS id, r.requested_at AS requested_at, r.finished_at AS finished_at,
|
||||
COUNT(*) OVER () AS total
|
||||
FROM runs r
|
||||
JOIN agent_profiles a ON a.id = r.agent_profile_id
|
||||
WHERE a.workspace_id = ? AND r.status = 'finished' AND r.outcome = 'processed'
|
||||
AND (r.session_id IS NULL OR r.session_id = '')
|
||||
AND r.requested_at >= ?
|
||||
AND COALESCE(r.finished_at, r.requested_at) >= ?
|
||||
ORDER BY COALESCE(r.finished_at, r.requested_at) DESC, r.id ASC
|
||||
LIMIT ?
|
||||
`), workspaceID, activationAt, windowStart, limit)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out := make([]SilentSuccessRow, 0, len(rows))
|
||||
total := 0
|
||||
for _, rr := range rows {
|
||||
out = append(out, rr.SilentSuccessRow)
|
||||
total = rr.Total
|
||||
}
|
||||
return out, total, nil
|
||||
}
|
||||
|
||||
// TerminalRunShapeInputRow carries exactly the columns
|
||||
// service.ClassifyTerminalRun needs, for every terminal run (not
|
||||
// queued, not claimed) in the evaluation window (AC-004.16). The
|
||||
// caller classifies and buckets; this method reads no free text
|
||||
// (AC-005.7) and is side-effect free.
|
||||
type TerminalRunShapeInputRow struct {
|
||||
Status string `db:"status"`
|
||||
Outcome *string `db:"outcome"`
|
||||
SessionID string `db:"session_id"`
|
||||
RequestedAt time.Time `db:"requested_at"`
|
||||
FinishedAt *time.Time `db:"finished_at"`
|
||||
}
|
||||
|
||||
// ListTerminalRunsInWindow returns every terminal run for the
|
||||
// workspace whose COALESCE(finished_at, requested_at) falls in
|
||||
// [windowStart, now). Unbounded by officeLoopEvidenceCap: this feeds
|
||||
// the terminal-shape counts, not an evidence list.
|
||||
func (r *Repository) ListTerminalRunsInWindow(
|
||||
ctx context.Context, workspaceID string, windowStart time.Time,
|
||||
) ([]TerminalRunShapeInputRow, error) {
|
||||
var rows []TerminalRunShapeInputRow
|
||||
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
|
||||
SELECT r.status AS status, r.outcome AS outcome, r.session_id AS session_id,
|
||||
r.requested_at AS requested_at, r.finished_at AS finished_at
|
||||
FROM runs r
|
||||
JOIN agent_profiles a ON a.id = r.agent_profile_id
|
||||
WHERE a.workspace_id = ? AND r.status NOT IN ('queued', 'claimed')
|
||||
AND COALESCE(r.finished_at, r.requested_at) >= ?
|
||||
`), workspaceID, windowStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rows == nil {
|
||||
rows = []TerminalRunShapeInputRow{}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// AC-004.15 / Claim-instant terminology (Review round 3, R3-3): a
|
||||
// trigger that has never fired (last_fired_at IS NULL, e.g. a
|
||||
// malformed cron_expression that never got a next_run_at) is still
|
||||
// claiming, not stranded, until strandedGrace has elapsed since its
|
||||
// own created_at — the spec's claim instant is
|
||||
// COALESCE(last_fired_at, created_at), not last_fired_at alone. The
|
||||
// old predicate treated any never-fired trigger as immediately
|
||||
// eligible for stranded, regardless of how recently it was created.
|
||||
func TestListOverdueOrStrandedTriggers_NeverFiredTriggerNotStrandedBeforeCreatedGrace(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "NeverFired")
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: nil,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
// Created 1 minute before now: well inside the 5-minute stranded
|
||||
// grace, never fired, never armed with a next_run_at.
|
||||
createdAt := now.Add(-1 * time.Minute)
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`,
|
||||
createdAt, trigger.ID); err != nil {
|
||||
t.Fatalf("seed never-fired state: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 || len(rows) != 0 {
|
||||
t.Fatalf("got %d rows, total %d, want 0/0 (a trigger created 1m ago is still claiming, not stranded)", len(rows), total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOverdueOrStrandedTriggers_NeverFiredTriggerStrandedAfterCreatedGrace(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "NeverFiredStale")
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: nil,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
// Created 6 minutes before now: past the 5-minute stranded grace,
|
||||
// never fired, never armed.
|
||||
createdAt := now.Add(-6 * time.Minute)
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = NULL, created_at = ? WHERE id = ?`,
|
||||
createdAt, trigger.ID); err != nil {
|
||||
t.Fatalf("seed never-fired stale state: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1 (a trigger created 6m ago that never fired must be stranded)", len(rows), total)
|
||||
}
|
||||
if rows[0].Condition != "stranded" {
|
||||
t.Errorf("condition = %q, want stranded", rows[0].Condition)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
)
|
||||
|
||||
func seedFinishedRun(
|
||||
t *testing.T, repo *sqlite.Repository, db *sqlx.DB,
|
||||
agentID, outcome, sessionID string, requestedAt time.Time, finishedAt *time.Time,
|
||||
) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
run := &models.Run{
|
||||
AgentProfileID: agentID, Reason: "task_assigned", Payload: "{}",
|
||||
Status: "queued", CoalescedCount: 1, SessionID: sessionID,
|
||||
}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(
|
||||
`UPDATE runs SET status = 'finished', outcome = ?, requested_at = ?, finished_at = ? WHERE id = ?`,
|
||||
outcome, requestedAt, finishedAt, run.ID,
|
||||
); err != nil {
|
||||
t.Fatalf("finish run: %v", err)
|
||||
}
|
||||
return run.ID
|
||||
}
|
||||
|
||||
func TestListSilentSuccesses_ReturnsFinishedProcessedSessionlessRun(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
finishedAt := now.Add(-time.Hour)
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt)
|
||||
|
||||
windowStart := now.Add(-24 * time.Hour)
|
||||
activationAt := now.Add(-48 * time.Hour)
|
||||
rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-005.6, applied to the AC-004.5/.7 evidence list: a legacy run
|
||||
// requested before the activation instant is pre_activation, never a
|
||||
// silent success, even when it is otherwise indistinguishable from one
|
||||
// and falls inside the evaluation window.
|
||||
func TestListSilentSuccesses_PreActivationRunExcluded(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
finishedAt := now.Add(-time.Hour)
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-2*time.Hour), &finishedAt)
|
||||
|
||||
windowStart := now.Add(-24 * time.Hour)
|
||||
activationAt := now.Add(-time.Minute) // activated after this run was requested
|
||||
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (run predates activation, must not read as silent success)", total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.16: a terminal run with a NULL finished_at still counts
|
||||
// within the window, aged from requested_at via COALESCE.
|
||||
func TestListSilentSuccesses_NullFinishedAtStillInsideWindow(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-time.Hour), nil)
|
||||
|
||||
windowStart := now.Add(-24 * time.Hour)
|
||||
activationAt := now.Add(-48 * time.Hour)
|
||||
rows, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1 (NULL finished_at falls back to requested_at)", len(rows), total)
|
||||
}
|
||||
if rows[0].FinishedAt != nil {
|
||||
t.Errorf("FinishedAt = %v, want nil", rows[0].FinishedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSilentSuccesses_LaunchedRunIsNotSilent(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
finishedAt := now.Add(-time.Hour)
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt)
|
||||
|
||||
windowStart := now.Add(-24 * time.Hour)
|
||||
activationAt := now.Add(-48 * time.Hour)
|
||||
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (a run with a session id is not silent)", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSilentSuccesses_OutsideWindowExcluded(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
finishedAt := now.Add(-30 * time.Hour)
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "", now.Add(-31*time.Hour), &finishedAt)
|
||||
|
||||
windowStart := now.Add(-24 * time.Hour)
|
||||
activationAt := now.Add(-48 * time.Hour)
|
||||
_, total, err := repo.ListSilentSuccesses(context.Background(), "ws-a", windowStart, activationAt, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (outside the 24h evaluation window)", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTerminalRunsInWindow_ExcludesQueuedAndClaimed(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, queued); err != nil {
|
||||
t.Fatalf("create queued: %v", err)
|
||||
}
|
||||
claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, claimed); err != nil {
|
||||
t.Fatalf("create claimed: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, claimed.ID); err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
finishedAt := now.Add(-time.Hour)
|
||||
seedFinishedRun(t, repo, db, "agent-a", "processed", "session-1", now.Add(-2*time.Hour), &finishedAt)
|
||||
|
||||
rows, err := repo.ListTerminalRunsInWindow(ctx, "ws-a", now.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, want 1 (queued/claimed excluded)", len(rows))
|
||||
}
|
||||
if rows[0].Status != "finished" {
|
||||
t.Errorf("status = %q, want finished", rows[0].Status)
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
// TestPostgresListStuckRuns is the PostgreSQL twin of the ListStuckRuns cases
|
||||
// in loop_health_stuck_runs_test.go. ListStuckRuns' stuck_since column is
|
||||
// computed by a UNION ALL over two different source columns, which loses its
|
||||
// declared type on read — the original fix normalized it with SQLite-only
|
||||
// strftime(), a function Postgres does not have, so this exercised the same
|
||||
// query shape (a claimed-stuck and a queued-stuck row, one list, ordered) end
|
||||
// to end against a real Postgres backend. Skips unless
|
||||
// KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresListStuckRuns(t *testing.T) {
|
||||
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("init settings store: %v", err)
|
||||
}
|
||||
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo: %v", err)
|
||||
}
|
||||
repo, err := sqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init office repo: %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Postgres timestamp columns store microsecond precision; truncate
|
||||
// before the round-trip so the later Equal() comparisons aren't
|
||||
// comparing against nanosecond digits the column never persisted.
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
// Two distinct agent profiles: sharing one would put the queued run
|
||||
// behind a claimed sibling on the same agent_profile_id, excluding
|
||||
// it under AC-OFFICE-LOOP-LIVENESS-004.17 (a busy agent's steady
|
||||
// state, not a stall) — this test wants both rows reported.
|
||||
// agent_profiles.agent_id carries a foreign key to agents(id),
|
||||
// enforced on Postgres (not on SQLite's default connection).
|
||||
for _, agentID := range []string{"agent-a", "agent-b"} {
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`INSERT INTO agents (id, name, created_at, updated_at) VALUES (?, ?, ?, ?)`,
|
||||
), agentID, agentID, now, now); err != nil {
|
||||
t.Fatalf("seed agent %s: %v", agentID, err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(`
|
||||
INSERT INTO agent_profiles (id, agent_id, name, agent_display_name, workspace_id, role, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`), agentID, agentID, agentID, agentID, "ws-pg-stuck", "developer", now, now); err != nil {
|
||||
t.Fatalf("seed agent profile %s: %v", agentID, err)
|
||||
}
|
||||
}
|
||||
|
||||
claimed := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, claimed); err != nil {
|
||||
t.Fatalf("create claimed run: %v", err)
|
||||
}
|
||||
longAgo := now.Add(-72 * time.Hour)
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
|
||||
), longAgo, claimed.ID); err != nil {
|
||||
t.Fatalf("seed claimed state: %v", err)
|
||||
}
|
||||
|
||||
queued := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, queued); err != nil {
|
||||
t.Fatalf("create queued run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`UPDATE runs SET requested_at = ? WHERE id = ?`,
|
||||
), old, queued.ID); err != nil {
|
||||
t.Fatalf("backdate requested_at: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-pg-stuck", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("ListStuckRuns: %v", err)
|
||||
}
|
||||
if total != 2 || len(rows) != 2 {
|
||||
t.Fatalf("got %d rows, total %d, want 2/2", len(rows), total)
|
||||
}
|
||||
// Claimed-stuck outranks queued-stuck regardless of relative age
|
||||
// (loop_health.go's ORDER BY (status = 'claimed') DESC ...).
|
||||
if rows[0].RunID != claimed.ID || rows[0].Condition != "claimed_stuck" {
|
||||
t.Errorf("rows[0] = %+v, want claimed_stuck for %s", rows[0], claimed.ID)
|
||||
}
|
||||
if rows[0].StuckSince == nil || !rows[0].StuckSince.Equal(longAgo) {
|
||||
t.Errorf("rows[0].StuckSince = %v, want %v", rows[0].StuckSince, longAgo)
|
||||
}
|
||||
if rows[1].RunID != queued.ID || rows[1].Condition != "queued_stuck" {
|
||||
t.Errorf("rows[1] = %+v, want queued_stuck for %s", rows[1], queued.ID)
|
||||
}
|
||||
if rows[1].StuckSince == nil || !rows[1].StuckSince.Equal(old) {
|
||||
t.Errorf("rows[1].StuckSince = %v, want %v", rows[1].StuckSince, old)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
func TestListStuckRuns_ClaimedStuckReportedRegardlessOfAge(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
// Claimed a very long time ago — the predicate is not windowed
|
||||
// (AC-004.5): a run stuck longer than the evaluation window is
|
||||
// still reported.
|
||||
longAgo := now.Add(-72 * time.Hour)
|
||||
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, longAgo, run.ID); err != nil {
|
||||
t.Fatalf("seed claimed state: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
|
||||
}
|
||||
if rows[0].Condition != "claimed_stuck" {
|
||||
t.Errorf("condition = %q, want claimed_stuck", rows[0].Condition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListStuckRuns_QueuedStuckPositiveControl(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil {
|
||||
t.Fatalf("backdate requested_at: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1 (positive control must be reported)", len(rows), total)
|
||||
}
|
||||
if rows[0].Condition != "queued_stuck" {
|
||||
t.Errorf("condition = %q, want queued_stuck", rows[0].Condition)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.17 exclusion 1: a future scheduled_retry_at is a deferral,
|
||||
// not a stall, even though requested_at is old — the COALESCE handles
|
||||
// this without a separate branch.
|
||||
func TestListStuckRuns_ExcludesFutureScheduledRetry(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
future := now.Add(5 * time.Minute)
|
||||
if _, err := db.Exec(
|
||||
`UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`,
|
||||
old, future, run.ID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed future retry: %v", err)
|
||||
}
|
||||
|
||||
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (future scheduled_retry_at is a deferral)", total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.17 exclusion 2: a non-NULL routing_blocked_status is a parked
|
||||
// run — parked for a human, not stuck.
|
||||
func TestListStuckRuns_ExcludesRoutingBlockedRun(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
if _, err := db.Exec(
|
||||
`UPDATE runs SET requested_at = ?, routing_blocked_status = 'waiting_for_provider_capacity' WHERE id = ?`,
|
||||
old, run.ID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed routing blocked: %v", err)
|
||||
}
|
||||
|
||||
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (routing-blocked run is parked, not stuck)", total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.17 exclusion 3: a claimed sibling on the same agent_profile_id
|
||||
// means the queue is correctly holding this run back — a busy agent's
|
||||
// steady state, not a stall.
|
||||
func TestListStuckRuns_ExcludesRunWithClaimedSibling(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
sibling := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, sibling); err != nil {
|
||||
t.Fatalf("create sibling: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, now, sibling.ID); err != nil {
|
||||
t.Fatalf("claim sibling: %v", err)
|
||||
}
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, old, run.ID); err != nil {
|
||||
t.Fatalf("backdate requested_at: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
// The claimed sibling itself is not "stuck" either (claimed_at is
|
||||
// fresh), so the workspace should report nothing at all.
|
||||
if total != 0 || len(rows) != 0 {
|
||||
t.Fatalf("got %d rows, total %d, want 0/0 (busy agent is not a stall)", len(rows), total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.17 aging: a run whose scheduled_retry_at has JUST passed is
|
||||
// aged from that instant, not from requested_at.
|
||||
func TestListStuckRuns_AgesFromScheduledRetryAtOncePassed(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
// requested_at is recent (would not itself be stuck), but
|
||||
// scheduled_retry_at passed well over the grace period ago.
|
||||
recent := now.Add(-30 * time.Second)
|
||||
pastRetry := now.Add(-3 * time.Minute)
|
||||
if _, err := db.Exec(
|
||||
`UPDATE runs SET requested_at = ?, scheduled_retry_at = ? WHERE id = ?`,
|
||||
recent, pastRetry, run.ID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed past retry: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1 (aged from scheduled_retry_at)", len(rows), total)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.17 (F29/F30 accepted limitation): a queued run that has
|
||||
// dispatched at least once (current_route_attempt_seq > 0) reads as
|
||||
// silent rather than stuck, mirroring ClaimNextEligibleRun's own gap
|
||||
// rather than fixing it here.
|
||||
func TestListStuckRuns_ExcludesRunWithNonZeroRouteAttemptSeq(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
|
||||
run := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
old := now.Add(-10 * time.Minute)
|
||||
if _, err := db.Exec(
|
||||
`UPDATE runs SET requested_at = ?, current_route_attempt_seq = 1 WHERE id = ?`,
|
||||
old, run.ID,
|
||||
); err != nil {
|
||||
t.Fatalf("seed routed attempt: %v", err)
|
||||
}
|
||||
|
||||
_, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("total = %d, want 0 (accepted limitation: a dispatched-once run reads as silent)", total)
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering: a claimed-stuck run outranks an older queued-stuck one,
|
||||
// and a NULL claimed_at row sorts first among claimed rows.
|
||||
func TestListStuckRuns_ClaimedOutranksOlderQueuedAndNullClaimedAtSortsFirst(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
seedAgentProfile(t, db, "agent-a", "ws-a")
|
||||
seedAgentProfile(t, db, "agent-b", "ws-a")
|
||||
seedAgentProfile(t, db, "agent-c", "ws-a")
|
||||
|
||||
// Oldest queued-stuck run.
|
||||
queued := &models.Run{AgentProfileID: "agent-a", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, queued); err != nil {
|
||||
t.Fatalf("create queued: %v", err)
|
||||
}
|
||||
veryOld := now.Add(-60 * time.Minute)
|
||||
if _, err := db.Exec(`UPDATE runs SET requested_at = ? WHERE id = ?`, veryOld, queued.ID); err != nil {
|
||||
t.Fatalf("backdate queued: %v", err)
|
||||
}
|
||||
|
||||
// Claimed-stuck run with a recorded claimed_at, younger than the
|
||||
// queued one but still past the claimed grace.
|
||||
claimedWithAge := &models.Run{AgentProfileID: "agent-b", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, claimedWithAge); err != nil {
|
||||
t.Fatalf("create claimed: %v", err)
|
||||
}
|
||||
claimedAt := now.Add(-15 * time.Minute)
|
||||
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`, claimedAt, claimedWithAge.ID); err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
|
||||
// Claimed-stuck run with NULL claimed_at.
|
||||
claimedNull := &models.Run{AgentProfileID: "agent-c", Reason: "task_assigned", Payload: "{}", Status: "queued", CoalescedCount: 1}
|
||||
if err := repo.CreateRun(ctx, claimedNull); err != nil {
|
||||
t.Fatalf("create claimed-null: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`UPDATE runs SET status = 'claimed', claimed_at = NULL WHERE id = ?`, claimedNull.ID); err != nil {
|
||||
t.Fatalf("claim with null claimed_at: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListStuckRuns(ctx, "ws-a", now, 2*time.Minute, 10*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 3 || len(rows) != 3 {
|
||||
t.Fatalf("got %d rows, total %d, want 3/3", len(rows), total)
|
||||
}
|
||||
if rows[0].RunID != claimedNull.ID {
|
||||
t.Errorf("rows[0] = %s, want the NULL-claimed_at row first", rows[0].RunID)
|
||||
}
|
||||
if rows[1].RunID != claimedWithAge.ID {
|
||||
t.Errorf("rows[1] = %s, want the aged claimed row second", rows[1].RunID)
|
||||
}
|
||||
if rows[2].RunID != queued.ID {
|
||||
t.Errorf("rows[2] = %s, want the queued row last despite being oldest", rows[2].RunID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// seedCronTrigger creates a routine and one cron trigger for it,
|
||||
// returning the trigger id. next_run_at/last_fired_at are set with a
|
||||
// raw UPDATE afterward since CreateRoutineTrigger only accepts a
|
||||
// caller-supplied NextRunAt at creation and never a NULL/backdated
|
||||
// LastFiredAt.
|
||||
func seedCronTrigger(t *testing.T, repo interface {
|
||||
CreateRoutine(ctx context.Context, r *models.Routine) error
|
||||
CreateRoutineTrigger(ctx context.Context, tr *models.RoutineTrigger) error
|
||||
}, workspaceID, name string) *models.Routine {
|
||||
t.Helper()
|
||||
routine := &models.Routine{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: name,
|
||||
TaskTemplate: "",
|
||||
Status: "active",
|
||||
ConcurrencyPolicy: "always_create",
|
||||
}
|
||||
if err := repo.CreateRoutine(context.Background(), routine); err != nil {
|
||||
t.Fatalf("create routine: %v", err)
|
||||
}
|
||||
return routine
|
||||
}
|
||||
|
||||
func TestListOverdueOrStrandedTriggers_ReturnsOverdueArmedTrigger(t *testing.T) {
|
||||
repo, db := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "Overdue")
|
||||
overdueAt := now.Add(-4 * time.Minute)
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
|
||||
}
|
||||
if rows[0].Condition != "overdue" {
|
||||
t.Errorf("condition = %q, want overdue", rows[0].Condition)
|
||||
}
|
||||
if rows[0].RoutineName != "Overdue" {
|
||||
t.Errorf("routine_name = %q, want Overdue", rows[0].RoutineName)
|
||||
}
|
||||
_ = db
|
||||
}
|
||||
|
||||
func TestListOverdueOrStrandedTriggers_ReturnsStrandedClaimedTrigger(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "Stranded")
|
||||
future := now.Add(time.Minute)
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &future,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
// Claim it: next_run_at -> NULL, last_fired_at -> 6 minutes ago,
|
||||
// past the 5-minute stranded grace.
|
||||
claimedAt := now.Add(-6 * time.Minute)
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
|
||||
claimedAt, trigger.ID); err != nil {
|
||||
t.Fatalf("seed stranded state: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("got %d rows, total %d, want 1/1", len(rows), total)
|
||||
}
|
||||
if rows[0].Condition != "stranded" {
|
||||
t.Errorf("condition = %q, want stranded", rows[0].Condition)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-004.15: a trigger claimed less than strandedGrace ago is a
|
||||
// claiming trigger — excluded from the evidence list entirely, not
|
||||
// given a third label.
|
||||
func TestListOverdueOrStrandedTriggers_ExcludesClaimingTrigger(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "Claiming")
|
||||
future := now.Add(time.Minute)
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &future,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
claimedAt := now.Add(-1 * time.Minute)
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
|
||||
claimedAt, trigger.ID); err != nil {
|
||||
t.Fatalf("seed claiming state: %v", err)
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(
|
||||
ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if total != 0 || len(rows) != 0 {
|
||||
t.Fatalf("got %d rows, total %d, want 0/0 (claiming trigger must be excluded)", len(rows), total)
|
||||
}
|
||||
|
||||
eligible, err := repo.CountEligibleTriggers(ctx, "ws-a")
|
||||
if err != nil {
|
||||
t.Fatalf("count eligible: %v", err)
|
||||
}
|
||||
if eligible != 1 {
|
||||
t.Errorf("eligible count = %d, want 1 (still an eligible trigger, just not overdue/stranded)", eligible)
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering: stranded (NULL next_run_at) sorts before overdue.
|
||||
func TestListOverdueOrStrandedTriggers_StrandedSortsBeforeOverdue(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
overdueRoutine := seedCronTrigger(t, repo, "ws-a", "Overdue")
|
||||
overdueAt := now.Add(-10 * time.Minute)
|
||||
overdueTrigger := &models.RoutineTrigger{
|
||||
RoutineID: overdueRoutine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, overdueTrigger); err != nil {
|
||||
t.Fatalf("create overdue trigger: %v", err)
|
||||
}
|
||||
|
||||
strandedRoutine := seedCronTrigger(t, repo, "ws-a", "Stranded")
|
||||
future := now.Add(time.Minute)
|
||||
strandedTrigger := &models.RoutineTrigger{
|
||||
RoutineID: strandedRoutine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &future,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, strandedTrigger); err != nil {
|
||||
t.Fatalf("create stranded trigger: %v", err)
|
||||
}
|
||||
if _, err := repo.ExecRaw(ctx,
|
||||
`UPDATE office_routine_triggers SET next_run_at = NULL, last_fired_at = ? WHERE id = ?`,
|
||||
now.Add(-10*time.Minute), strandedTrigger.ID); err != nil {
|
||||
t.Fatalf("seed stranded state: %v", err)
|
||||
}
|
||||
|
||||
rows, _, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("got %d rows, want 2", len(rows))
|
||||
}
|
||||
if rows[0].Condition != "stranded" || rows[1].Condition != "overdue" {
|
||||
t.Errorf("order = [%s, %s], want [stranded, overdue]", rows[0].Condition, rows[1].Condition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListOverdueOrStrandedTriggers_CapsAndReportsTotal(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
routine := seedCronTrigger(t, repo, "ws-a", "R")
|
||||
overdueAt := now.Add(-10 * time.Minute)
|
||||
trigger := &models.RoutineTrigger{
|
||||
RoutineID: routine.ID, Kind: "cron", CronExpression: "* * * * *",
|
||||
Timezone: "UTC", Enabled: true, NextRunAt: &overdueAt,
|
||||
}
|
||||
if err := repo.CreateRoutineTrigger(ctx, trigger); err != nil {
|
||||
t.Fatalf("create trigger %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
rows, total, err := repo.ListOverdueOrStrandedTriggers(ctx, "ws-a", now, 3*time.Minute, 5*time.Minute, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("got %d rows, want capped at 2", len(rows))
|
||||
}
|
||||
if total != 3 {
|
||||
t.Fatalf("total = %d, want 3 (untruncated)", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountEligibleTriggers_ZeroForEmptyWorkspace(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
count, err := repo.CountEligibleTriggers(context.Background(), "ws-empty")
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("count = %d, want 0", count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/persistence"
|
||||
)
|
||||
|
||||
// loopLivenessActivationKey is the kandev_meta key published once a
|
||||
// boot has verified all three causation_id columns exist
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-002/005). See docs/specs/office/system-design/
|
||||
// loop-liveness.md, "Activation".
|
||||
const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at"
|
||||
|
||||
// causationIDColumn is the column all three activation probes check.
|
||||
const causationIDColumn = "causation_id"
|
||||
|
||||
// activateLoopLiveness writes loopLivenessActivationKey only after a
|
||||
// positive probe confirms all three causation_id columns exist. Mirrors
|
||||
// activateRunOutcome: the migration runner swallows failures at WARN,
|
||||
// so this probe is what stands between a failed migration and a reader
|
||||
// wrongly believing correlation and terminal-shape classification are
|
||||
// live. WriteMetaKeyIfAbsent makes the write replay-safe — the instant
|
||||
// is never overwritten once set, because every run requested before it
|
||||
// must classify as pre_activation forever, not just until the next
|
||||
// boot. Any failure here is logged and swallowed — activation is
|
||||
// best-effort published fact, never a boot-blocking requirement.
|
||||
func (r *Repository) activateLoopLiveness() {
|
||||
if err := persistence.EnsureMetaTable(r.db); err != nil {
|
||||
r.logActivationWarn("ensure kandev_meta table failed", err)
|
||||
return
|
||||
}
|
||||
for _, probe := range []struct{ table, column string }{
|
||||
{"office_routine_runs", causationIDColumn},
|
||||
{"agent_wakeup_requests", causationIDColumn},
|
||||
{"runs", causationIDColumn},
|
||||
} {
|
||||
exists, err := columnExists(r.db, probe.table, probe.column)
|
||||
if err != nil {
|
||||
r.logActivationWarn("loop liveness schema probe failed", err)
|
||||
return
|
||||
}
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := persistence.WriteMetaKeyIfAbsent(
|
||||
r.db, loopLivenessActivationKey, time.Now().UTC().Format(time.RFC3339),
|
||||
); err != nil {
|
||||
r.logActivationWarn("write loop liveness activation key failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
// LoopLivenessActivation returns the published activation instant and
|
||||
// whether it has been published at all. A run requested before the
|
||||
// instant, or read when activation was never published, must classify
|
||||
// as pre_activation (AC-005.6) rather than risk a false silent_success.
|
||||
// Read failures are swallowed here (logged only) because every existing
|
||||
// caller uses this for a safe-default classification, not a decision
|
||||
// that must fail loud; the /loop-health read that must distinguish
|
||||
// "unpublished" from "unreadable" uses LoopLivenessActivationChecked.
|
||||
func (r *Repository) LoopLivenessActivation() (at time.Time, published bool) {
|
||||
at, published, err := r.LoopLivenessActivationChecked()
|
||||
if err != nil && r.log != nil {
|
||||
r.log.Warn("read loop liveness activation key failed", zap.Error(err))
|
||||
}
|
||||
return at, published
|
||||
}
|
||||
|
||||
// LoopLivenessActivationChecked is LoopLivenessActivation with the read
|
||||
// or parse error surfaced instead of swallowed, for a caller that must
|
||||
// fail loud (REQ-OFFICE-LOOP-LIVENESS-004, AC-004.10) rather than let a
|
||||
// broken kandev_meta read silently render as "not yet activated".
|
||||
func (r *Repository) LoopLivenessActivationChecked() (at time.Time, published bool, err error) {
|
||||
raw, err := persistence.ReadMetaKey(r.db, loopLivenessActivationKey)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if raw == "" {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
return parsed, true, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/persistence"
|
||||
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
const loopLivenessActivationKey = "telemetry.office_loop_liveness.activated_at"
|
||||
|
||||
// TestLoopLivenessMigration_AddsCausationIDColumns covers
|
||||
// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: all three
|
||||
// causation_id columns exist and default to ” (not NULL) on a
|
||||
// freshly-migrated database.
|
||||
func TestLoopLivenessMigration_AddsCausationIDColumns(t *testing.T) {
|
||||
_, db := newTestRepoWithDB(t)
|
||||
|
||||
for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} {
|
||||
var notnull int
|
||||
var dflt sql.NullString
|
||||
rows, err := db.Queryx("PRAGMA table_info(" + table + ")")
|
||||
if err != nil {
|
||||
t.Fatalf("table_info(%s): %v", table, err)
|
||||
}
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid, nn, pk int
|
||||
var name, ctype string
|
||||
var d sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &ctype, &nn, &d, &pk); err != nil {
|
||||
t.Fatalf("scan table_info(%s): %v", table, err)
|
||||
}
|
||||
if name == "causation_id" {
|
||||
found = true
|
||||
notnull = nn
|
||||
dflt = d
|
||||
}
|
||||
}
|
||||
_ = rows.Close()
|
||||
if !found {
|
||||
t.Fatalf("%s.causation_id column missing", table)
|
||||
}
|
||||
if notnull != 1 {
|
||||
t.Fatalf("%s.causation_id NOT NULL = %d, want 1", table, notnull)
|
||||
}
|
||||
if !dflt.Valid || dflt.String != "''" {
|
||||
t.Fatalf("%s.causation_id default = %v, want ''", table, dflt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopLivenessMigration_CausationIDIndexesArePartial covers
|
||||
// REQ-OFFICE-LOOP-LIVENESS-002's Persistence section: each
|
||||
// causation_id index exists and is partial (WHERE causation_id !=
|
||||
// ”), not a plain index over every row — a full index would defeat
|
||||
// the point of excluding the common empty-string case from the
|
||||
// lookup path. sqlite_master.sql is the only way to see the stored
|
||||
// WHERE predicate; PRAGMA index_list/index_info do not expose it.
|
||||
func TestLoopLivenessMigration_CausationIDIndexesArePartial(t *testing.T) {
|
||||
_, db := newTestRepoWithDB(t)
|
||||
|
||||
cases := []struct {
|
||||
table string
|
||||
indexName string
|
||||
}{
|
||||
{"office_routine_runs", "idx_office_routine_runs_causation_id"},
|
||||
{"agent_wakeup_requests", "idx_agent_wakeup_requests_causation_id"},
|
||||
{"runs", "idx_runs_causation_id"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
var stored sql.NullString
|
||||
err := db.QueryRow(
|
||||
`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?`, tc.indexName,
|
||||
).Scan(&stored)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: look up index %s: %v", tc.table, tc.indexName, err)
|
||||
}
|
||||
if !stored.Valid || stored.String == "" {
|
||||
t.Fatalf("%s: index %s missing or has no stored definition", tc.table, tc.indexName)
|
||||
}
|
||||
if !strings.Contains(stored.String, "WHERE") || !strings.Contains(stored.String, "causation_id != ''") {
|
||||
t.Fatalf("%s: index %s definition = %q, want a WHERE causation_id != '' partial predicate",
|
||||
tc.table, tc.indexName, stored.String)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe mirrors
|
||||
// TestRunOutcomeActivation_WrittenOnceAfterSchemaProbe: the key is
|
||||
// written only after a positive probe of all three columns, and a
|
||||
// replayed boot never re-writes it.
|
||||
func TestLoopLivenessActivation_WrittenOnceAfterSchemaProbe(t *testing.T) {
|
||||
_, db := newTestRepoWithDB(t)
|
||||
|
||||
val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
|
||||
if err != nil {
|
||||
t.Fatalf("read activation key: %v", err)
|
||||
}
|
||||
if val == "" {
|
||||
t.Fatal("expected telemetry.office_loop_liveness.activated_at to be written after first boot")
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, val); err != nil {
|
||||
t.Fatalf("activation value %q is not RFC3339: %v", val, err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(db.Rebind(
|
||||
`UPDATE kandev_meta SET value = ? WHERE key = ?`,
|
||||
), "sentinel-value", loopLivenessActivationKey); err != nil {
|
||||
t.Fatalf("seed sentinel: %v", err)
|
||||
}
|
||||
|
||||
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("replay init: %v", err)
|
||||
}
|
||||
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("replay init twice: %v", err)
|
||||
}
|
||||
|
||||
got, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
|
||||
if err != nil {
|
||||
t.Fatalf("read activation key after replay: %v", err)
|
||||
}
|
||||
if got != "sentinel-value" {
|
||||
t.Fatalf("activation key = %q after replay, want unchanged sentinel-value", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoopLivenessActivation_ReadHelperRoundTrips covers
|
||||
// Repository.LoopLivenessActivation: published=true with the parsed
|
||||
// instant after boot, and published=false when the key is absent.
|
||||
func TestLoopLivenessActivation_ReadHelperRoundTrips(t *testing.T) {
|
||||
repo, _ := newTestRepoWithDB(t)
|
||||
|
||||
at, published := repo.LoopLivenessActivation()
|
||||
if !published {
|
||||
t.Fatal("expected activation to be published after boot")
|
||||
}
|
||||
if at.IsZero() {
|
||||
t.Fatal("expected a non-zero activation instant")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostgresLoopLivenessMigration_AddsColumnsAndActivates is the
|
||||
// PostgreSQL twin required by ADR 0027. Skips unless
|
||||
// KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresLoopLivenessMigration_AddsColumnsAndActivates(t *testing.T) {
|
||||
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
|
||||
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo: %v", err)
|
||||
}
|
||||
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init postgres schema: %v", err)
|
||||
}
|
||||
if _, err := sqlite.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("replay postgres schema: %v", err)
|
||||
}
|
||||
|
||||
for _, table := range []string{"office_routine_runs", "agent_wakeup_requests", "runs"} {
|
||||
var dataType string
|
||||
err := db.QueryRow(`
|
||||
SELECT data_type FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = $1 AND column_name = 'causation_id'
|
||||
`, table).Scan(&dataType)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect %s.causation_id: %v", table, err)
|
||||
}
|
||||
if dataType != "text" {
|
||||
t.Fatalf("%s.causation_id data type = %q, want text", table, dataType)
|
||||
}
|
||||
}
|
||||
|
||||
val, err := persistence.ReadMetaKey(db, loopLivenessActivationKey)
|
||||
if err != nil {
|
||||
t.Fatalf("read activation key: %v", err)
|
||||
}
|
||||
if val == "" {
|
||||
t.Fatal("expected activation key to be written on postgres")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
// TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun is
|
||||
// the two-connection regression for Review round 3, R3-1: MarkRunFailed's
|
||||
// write used to carry no status guard at all, so a run cancelled on
|
||||
// another connection between HandleAgentFailure's read and this write
|
||||
// would have its 'cancelled' status and real finished_at silently
|
||||
// overwritten to 'failed'. Real Postgres row locking forces the
|
||||
// interleaving this needs: one connection holds an uncommitted cancel on
|
||||
// the row, MarkRunFailed's write blocks on that row lock, and only
|
||||
// proceeds once the cancel commits — at which point the guard
|
||||
// (status = 'claimed') must see the fresh 'cancelled' status and write
|
||||
// nothing. Skips unless KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresMarkRunFailed_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) {
|
||||
dsn := testutil.PostgresDSNFromEnv(t)
|
||||
db := testutil.OpenIsolatedPostgres(t, dsn)
|
||||
|
||||
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo: %v", err)
|
||||
}
|
||||
repo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init office repo: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
var schema string
|
||||
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
|
||||
t.Fatalf("read isolated schema name: %v", err)
|
||||
}
|
||||
cancellerDB, err := sqlx.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open canceller connection: %v", err)
|
||||
}
|
||||
cancellerDB.SetMaxOpenConns(1)
|
||||
cancellerDB.SetMaxIdleConns(1)
|
||||
t.Cleanup(func() { _ = cancellerDB.Close() })
|
||||
if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil {
|
||||
t.Fatalf("set canceller search_path: %v", err)
|
||||
}
|
||||
|
||||
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
|
||||
), now, run.ID); err != nil {
|
||||
t.Fatalf("seed claimed: %v", err)
|
||||
}
|
||||
|
||||
tx, err := cancellerDB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin canceller tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
if _, err := tx.ExecContext(ctx, cancellerDB.Rebind(
|
||||
`UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`,
|
||||
), "too late", now, run.ID); err != nil {
|
||||
t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
wrote bool
|
||||
markErr error
|
||||
markDone = make(chan struct{})
|
||||
commitErr error
|
||||
)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer close(markDone)
|
||||
wrote, markErr = repo.MarkRunFailed(ctx, run.ID, "provider exploded")
|
||||
}()
|
||||
|
||||
// Give MarkRunFailed's write time to reach Postgres and block on the
|
||||
// row lock the canceller's uncommitted transaction holds, before that
|
||||
// transaction commits. Without this window the two goroutines could
|
||||
// run in either order and the test would not exercise the lock-wait
|
||||
// path this bug depends on.
|
||||
select {
|
||||
case <-markDone:
|
||||
t.Fatal("MarkRunFailed returned before the canceller committed; it never blocked on the row lock")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
commitErr = tx.Commit()
|
||||
wg.Wait()
|
||||
if commitErr != nil {
|
||||
t.Fatalf("canceller commit: %v", commitErr)
|
||||
}
|
||||
if markErr != nil {
|
||||
t.Fatalf("mark run failed: %v", markErr)
|
||||
}
|
||||
|
||||
if wrote {
|
||||
t.Fatal("wrote = true, want false: a run that committed cancelled must not be reported failed")
|
||||
}
|
||||
if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" {
|
||||
t.Fatalf("run status = %q, want cancelled (the concurrent fail must not have overwritten it)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string {
|
||||
t.Helper()
|
||||
var status string
|
||||
if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil {
|
||||
t.Fatalf("get run status: %v", err)
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -437,8 +437,11 @@ func TestCancelDisplacedParticipantRun_CancelsOnlyTheFanOutRunForThatAgentStep(t
|
||||
if err != nil {
|
||||
t.Fatalf("CancelDisplacedParticipantRun: %v", err)
|
||||
}
|
||||
if cancelled != 1 {
|
||||
t.Fatalf("cancelled = %d, want 1", cancelled)
|
||||
if len(cancelled) != 1 {
|
||||
t.Fatalf("cancelled = %d, want 1", len(cancelled))
|
||||
}
|
||||
if cancelled[0].ID != "run-target" {
|
||||
t.Fatalf("cancelled[0].ID = %q, want run-target", cancelled[0].ID)
|
||||
}
|
||||
|
||||
assertRunStatus(t, repo, "run-target", "cancelled")
|
||||
@@ -460,8 +463,8 @@ func TestCancelDisplacedParticipantRun_NoMatchIsNotAFailure(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("CancelDisplacedParticipantRun: %v", err)
|
||||
}
|
||||
if cancelled != 0 {
|
||||
t.Errorf("cancelled = %d, want 0", cancelled)
|
||||
if len(cancelled) != 0 {
|
||||
t.Errorf("cancelled = %d, want 0", len(cancelled))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/kandev/kandev/internal/db/dialect"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/workflow/models"
|
||||
workflowrepo "github.com/kandev/kandev/internal/workflow/repository"
|
||||
)
|
||||
@@ -672,9 +673,11 @@ const displacedRunCancelReason = "participant_seat_claimed"
|
||||
// SQLite-flavoured and the server dialect needs dialect.JSONExtract, which
|
||||
// is why this selector lives here rather than in the shared runs writer
|
||||
// (mirrors CancelRunsForTasks in tree_holds.go).
|
||||
func (r *Repository) CancelDisplacedParticipantRun(ctx context.Context, taskID, stepID, agentProfileID string) (int64, error) {
|
||||
func (r *Repository) CancelDisplacedParticipantRun(
|
||||
ctx context.Context, taskID, stepID, agentProfileID string,
|
||||
) ([]runssqlite.CancelledRun, error) {
|
||||
if taskID == "" || stepID == "" || agentProfileID == "" {
|
||||
return 0, nil
|
||||
return nil, nil
|
||||
}
|
||||
driver := r.db.DriverName()
|
||||
selector := fmt.Sprintf(
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
func createTestRoutineForLastRun(t *testing.T, repo interface {
|
||||
CreateRoutine(ctx context.Context, routine *models.Routine) error
|
||||
}) *models.Routine {
|
||||
t.Helper()
|
||||
routine := &models.Routine{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "Last Run Test",
|
||||
TaskTemplate: "{}",
|
||||
Status: "active",
|
||||
ConcurrencyPolicy: "skip_if_active",
|
||||
Variables: "{}",
|
||||
}
|
||||
if err := repo.CreateRoutine(context.Background(), routine); err != nil {
|
||||
t.Fatalf("create routine: %v", err)
|
||||
}
|
||||
return routine
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.1/.3/.4/.6: TouchRoutineLastRun advances
|
||||
// last_run_at from NULL, is monotonic against an equal or older instant
|
||||
// (zero-row no-op, not an error), and moves forward for a newer instant.
|
||||
func TestTouchRoutineLastRun_Monotonic(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
ctx := context.Background()
|
||||
routine := createTestRoutineForLastRun(t, repo)
|
||||
|
||||
first := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil {
|
||||
t.Fatalf("touch (initial): %v", err)
|
||||
}
|
||||
got, err := repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.LastRunAt == nil || !got.LastRunAt.Equal(first) {
|
||||
t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, first)
|
||||
}
|
||||
|
||||
// Older instant: no-op.
|
||||
older := first.Add(-time.Hour)
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, older); err != nil {
|
||||
t.Fatalf("touch (older): %v", err)
|
||||
}
|
||||
got, err = repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.Equal(first) {
|
||||
t.Fatalf("last_run_at moved backwards: got %v, want %v", got.LastRunAt, first)
|
||||
}
|
||||
|
||||
// Equal instant: no-op.
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, first); err != nil {
|
||||
t.Fatalf("touch (equal): %v", err)
|
||||
}
|
||||
got, err = repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.Equal(first) {
|
||||
t.Fatalf("last_run_at changed on equal instant: got %v, want %v", got.LastRunAt, first)
|
||||
}
|
||||
|
||||
// Newer instant: advances.
|
||||
newer := first.Add(time.Hour)
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, newer); err != nil {
|
||||
t.Fatalf("touch (newer): %v", err)
|
||||
}
|
||||
got, err = repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !got.LastRunAt.Equal(newer) {
|
||||
t.Fatalf("last_run_at = %v, want %v", got.LastRunAt, newer)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.6: a touch matching zero rows (unknown
|
||||
// routine) is treated as success, not an error.
|
||||
func TestTouchRoutineLastRun_UnknownRoutineIsNoop(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if err := repo.TouchRoutineLastRun(ctx, "does-not-exist", time.Now().UTC()); err != nil {
|
||||
t.Fatalf("touch unknown routine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.5: the write touches only last_run_at (and
|
||||
// updated_at) — every other column is left exactly as it was.
|
||||
func TestTouchRoutineLastRun_TouchesOnlyLastRunAtAndUpdatedAt(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
ctx := context.Background()
|
||||
routine := createTestRoutineForLastRun(t, repo)
|
||||
beforeUpdatedAt := routine.UpdatedAt
|
||||
|
||||
at := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, at); err != nil {
|
||||
t.Fatalf("touch: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Name != routine.Name || got.Status != routine.Status ||
|
||||
got.ConcurrencyPolicy != routine.ConcurrencyPolicy || got.TaskTemplate != routine.TaskTemplate {
|
||||
t.Fatalf("touch mutated a non-last_run_at column: %+v", got)
|
||||
}
|
||||
if !got.UpdatedAt.After(beforeUpdatedAt) {
|
||||
t.Fatalf("updated_at did not advance: got %v, before %v", got.UpdatedAt, beforeUpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.8: UpdateRoutine no longer carries
|
||||
// last_run_at — a caller holding a stale (or nil) snapshot cannot move
|
||||
// the fresh instant this dispatch just wrote backwards or clear it.
|
||||
func TestUpdateRoutine_DoesNotWriteLastRunAt(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
ctx := context.Background()
|
||||
routine := createTestRoutineForLastRun(t, repo)
|
||||
|
||||
fresh := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
if err := repo.TouchRoutineLastRun(ctx, routine.ID, fresh); err != nil {
|
||||
t.Fatalf("touch: %v", err)
|
||||
}
|
||||
|
||||
// A caller holding a stale snapshot (older LastRunAt) must not move
|
||||
// the stored instant backwards via an unrelated field update.
|
||||
stale := fresh.Add(-24 * time.Hour)
|
||||
routine.LastRunAt = &stale
|
||||
routine.Name = "Renamed While Stale"
|
||||
if err := repo.UpdateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("update (stale snapshot): %v", err)
|
||||
}
|
||||
got, err := repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
|
||||
t.Fatalf("stale UpdateRoutine moved last_run_at: got %v, want %v", got.LastRunAt, fresh)
|
||||
}
|
||||
if got.Name != "Renamed While Stale" {
|
||||
t.Fatalf("update did not apply other fields: name = %q", got.Name)
|
||||
}
|
||||
|
||||
// A caller that never populated LastRunAt (nil) must not clear it either.
|
||||
routine.LastRunAt = nil
|
||||
routine.Description = "Updated from nil snapshot"
|
||||
if err := repo.UpdateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("update (nil snapshot): %v", err)
|
||||
}
|
||||
got, err = repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
|
||||
t.Fatalf("nil-snapshot UpdateRoutine cleared last_run_at: got %v, want %v", got.LastRunAt, fresh)
|
||||
}
|
||||
if got.Description != "Updated from nil snapshot" {
|
||||
t.Fatalf("update did not apply other fields: description = %q", got.Description)
|
||||
}
|
||||
|
||||
// A caller holding a snapshot newer than the stored instant must not
|
||||
// advance it either: a stale-or-nil case alone cannot distinguish "the
|
||||
// column is not written" from "the column is written monotonically",
|
||||
// since both pass under a stale or nil snapshot. Only a newer snapshot
|
||||
// tells them apart.
|
||||
newer := fresh.Add(24 * time.Hour)
|
||||
routine.LastRunAt = &newer
|
||||
routine.Name = "Renamed From Newer Snapshot"
|
||||
if err := repo.UpdateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("update (newer snapshot): %v", err)
|
||||
}
|
||||
got, err = repo.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.LastRunAt == nil || !got.LastRunAt.Equal(fresh) {
|
||||
t.Fatalf("newer-snapshot UpdateRoutine advanced last_run_at: got %v, want %v", got.LastRunAt, fresh)
|
||||
}
|
||||
if got.Name != "Renamed From Newer Snapshot" {
|
||||
t.Fatalf("update did not apply other fields: name = %q", got.Name)
|
||||
}
|
||||
}
|
||||
@@ -196,7 +196,13 @@ func (r *Repository) ListRoutines(ctx context.Context, workspaceID string) ([]*m
|
||||
return routines, nil
|
||||
}
|
||||
|
||||
// UpdateRoutine updates an existing routine.
|
||||
// UpdateRoutine updates an existing routine. It deliberately does not
|
||||
// carry last_run_at (AC-OFFICE-LOOP-LIVENESS-001.8): that column only
|
||||
// ever moves forward, written by TouchRoutineLastRun from a dispatch, and
|
||||
// a caller here may be holding a routine loaded before a fire (or one
|
||||
// that never populated the field) — writing it unconditionally would let
|
||||
// that stale or nil snapshot silently move the value backwards or clear
|
||||
// it. routine.LastRunAt stays on the struct for reads.
|
||||
func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine) error {
|
||||
routine.UpdatedAt = time.Now().UTC()
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
@@ -204,12 +210,27 @@ func (r *Repository) UpdateRoutine(ctx context.Context, routine *models.Routine)
|
||||
name = ?, description = ?, task_template = ?,
|
||||
assignee_agent_profile_id = ?, status = ?, concurrency_policy = ?,
|
||||
catch_up_policy = ?, catch_up_max = ?,
|
||||
variables = ?, last_run_at = ?, updated_at = ?
|
||||
variables = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
`), routine.Name, routine.Description, routine.TaskTemplate,
|
||||
routine.AssigneeAgentProfileID, routine.Status, routine.ConcurrencyPolicy,
|
||||
routine.CatchUpPolicy, routine.CatchUpMax,
|
||||
routine.Variables, routine.LastRunAt, routine.UpdatedAt, routine.ID)
|
||||
routine.Variables, routine.UpdatedAt, routine.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
// TouchRoutineLastRun advances office_routines.last_run_at to at,
|
||||
// monotonically: the write is a zero-row no-op, not an error, when the
|
||||
// stored value is already at or after at (AC-OFFICE-LOOP-LIVENESS-001.3,
|
||||
// .4, .6). Deliberately narrower than UpdateRoutine's eleven-column
|
||||
// read-modify-write, which would turn a routine dispatch into a race
|
||||
// against a concurrent UI or config-sync edit (AC-OFFICE-LOOP-LIVENESS-001.5).
|
||||
func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error {
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE office_routines
|
||||
SET last_run_at = ?, updated_at = ?
|
||||
WHERE id = ? AND (last_run_at IS NULL OR last_run_at < ?)
|
||||
`), at, time.Now().UTC(), routineID, at)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -279,11 +300,11 @@ func (r *Repository) CreateRoutineRun(ctx context.Context, run *models.RoutineRu
|
||||
INSERT INTO office_routine_runs (
|
||||
id, routine_id, trigger_id, source, status, trigger_payload,
|
||||
linked_task_id, coalesced_into_run_id, dispatch_fingerprint,
|
||||
started_at, completed_at, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
started_at, completed_at, created_at, causation_id
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`), run.ID, run.RoutineID, run.TriggerID, run.Source, run.Status,
|
||||
run.TriggerPayload, run.LinkedTaskID, run.CoalescedIntoRunID,
|
||||
run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt)
|
||||
run.DispatchFingerprint, run.StartedAt, run.CompletedAt, run.CreatedAt, run.CausationID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -404,18 +425,6 @@ func (r *Repository) GetTaskTerminalStatus(ctx context.Context, taskID string) (
|
||||
}
|
||||
}
|
||||
|
||||
// TouchRoutineLastRun updates only last_run_at (+ updated_at) on a
|
||||
// routine. Deliberately narrower than UpdateRoutine, whose whole-row
|
||||
// snapshot write can revert concurrent edits to other columns (see
|
||||
// UpdateRoutineConfigFields above) — a dispatch in flight should never
|
||||
// clobber a config change made while it ran, or vice versa.
|
||||
func (r *Repository) TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error {
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE office_routines SET last_run_at = ?, updated_at = ? WHERE id = ?
|
||||
`), at, time.Now().UTC(), routineID)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateRunStatus updates a run's status and optionally its linked task.
|
||||
func (r *Repository) UpdateRunStatus(
|
||||
ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string,
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestCancelRun_QueuedRunIsCancelled(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
id := seedCancelRun(t, repo, "t1", "queued", nil, nil)
|
||||
|
||||
if err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil {
|
||||
if _, err := repo.CancelRun(context.Background(), id, "run_too_old"); err != nil {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ func TestCancelRun_ClaimedRunIsCancelled(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
id := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
|
||||
|
||||
if err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil {
|
||||
if _, err := repo.CancelRun(context.Background(), id, "execution_too_old"); err != nil {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestCancelRun_TerminalRunIsUntouched(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
id := seedCancelRun(t, repo, "t1", tc.status, tc.cancelReason, &finished)
|
||||
|
||||
if err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil {
|
||||
if _, err := repo.CancelRun(context.Background(), id, "task_reassigned"); err != nil {
|
||||
t.Fatalf("cancel: %v", err)
|
||||
}
|
||||
|
||||
@@ -155,16 +155,16 @@ func TestCancelRunsWhere_RowsAffected(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("cancel terminal: %v", err)
|
||||
}
|
||||
if rows != 0 {
|
||||
t.Errorf("rows affected = %d, want 0 for an already-terminal run", rows)
|
||||
if len(rows) != 0 {
|
||||
t.Errorf("rows affected = %d, want 0 for an already-terminal run", len(rows))
|
||||
}
|
||||
|
||||
rows, err = repo.CancelRunsWhere(ctx, "tree_cancelled", `id = ?`, queued)
|
||||
if err != nil {
|
||||
t.Fatalf("cancel queued: %v", err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Errorf("rows affected = %d, want 1", rows)
|
||||
if len(rows) != 1 {
|
||||
t.Errorf("rows affected = %d, want 1", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,7 +177,7 @@ func TestBulkCancelRuns_OnlyCancelsEligibleRuns(t *testing.T) {
|
||||
claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
|
||||
done := seedCancelRun(t, repo, "t1", "finished", nil, &finished)
|
||||
|
||||
if err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil {
|
||||
if _, err := repo.BulkCancelRuns(ctx, []string{queued, claimed, done}, "task_reassigned"); err != nil {
|
||||
t.Fatalf("bulk cancel: %v", err)
|
||||
}
|
||||
|
||||
@@ -204,12 +204,12 @@ func TestCancelRunsForTasks_MixedSetCountMatches(t *testing.T) {
|
||||
claimed := seedCancelRun(t, repo, "t1", "claimed", nil, nil)
|
||||
done := seedCancelRun(t, repo, "t1", "finished", nil, &finished)
|
||||
|
||||
count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused")
|
||||
cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_paused")
|
||||
if err != nil {
|
||||
t.Fatalf("cancel for tasks: %v", err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", count)
|
||||
if len(cancelled) != 2 {
|
||||
t.Errorf("count = %d, want 2 (queued + claimed, not the finished run)", len(cancelled))
|
||||
}
|
||||
for _, id := range []string{queued, claimed} {
|
||||
if run := fetchRun(t, repo, id); run.Status != "cancelled" {
|
||||
@@ -229,12 +229,12 @@ func TestCancelRunsForTasks_MatchesPayloadTaskID(t *testing.T) {
|
||||
mine := seedCancelRun(t, repo, "t1", "queued", nil, nil)
|
||||
other := seedCancelRun(t, repo, "t2", "queued", nil, nil)
|
||||
|
||||
count, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled")
|
||||
cancelled, err := repo.CancelRunsForTasks(ctx, []string{"t1"}, "tree_cancelled")
|
||||
if err != nil {
|
||||
t.Fatalf("cancel for tasks: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("count = %d, want 1", count)
|
||||
if len(cancelled) != 1 {
|
||||
t.Fatalf("count = %d, want 1", len(cancelled))
|
||||
}
|
||||
if run := fetchRun(t, repo, mine); run.Status != "cancelled" {
|
||||
t.Errorf("t1 run status = %q, want cancelled", run.Status)
|
||||
@@ -253,15 +253,15 @@ func TestCancelPaths_EmptySliceIsANoOp(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
id := seedCancelRun(t, repo, "t1", "queued", nil, nil)
|
||||
|
||||
if err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil {
|
||||
if _, err := repo.BulkCancelRuns(ctx, nil, "task_reassigned"); err != nil {
|
||||
t.Errorf("bulk cancel with no ids: %v", err)
|
||||
}
|
||||
count, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled")
|
||||
cancelled, err := repo.CancelRunsForTasks(ctx, nil, "tree_cancelled")
|
||||
if err != nil {
|
||||
t.Errorf("cancel for tasks with no task ids: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("count = %d, want 0", count)
|
||||
if len(cancelled) != 0 {
|
||||
t.Errorf("count = %d, want 0", len(cancelled))
|
||||
}
|
||||
if run := fetchRun(t, repo, id); run.Status != "queued" {
|
||||
t.Errorf("unrelated run status = %q, want queued", run.Status)
|
||||
|
||||
@@ -235,14 +235,15 @@ func TestCleanExpired(t *testing.T) {
|
||||
AgentProfileID: "a1",
|
||||
Reason: "task_assigned",
|
||||
Payload: "{}",
|
||||
Status: "queued",
|
||||
Status: "claimed",
|
||||
CoalescedCount: 1,
|
||||
}
|
||||
if err := repo.CreateRun(ctx, req); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
// Finish it.
|
||||
if err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil {
|
||||
// Finish it. FinishRun is guarded to status = 'claimed' (Review round
|
||||
// 3, R3-1), matching the seeded status above.
|
||||
if _, err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
func (r *Repository) createTreeHoldTables() error {
|
||||
@@ -208,23 +209,22 @@ func (r *Repository) CountActiveRunsForTasks(ctx context.Context, taskIDs []stri
|
||||
}
|
||||
|
||||
// CancelRunsForTasks cancels the queued or claimed runs belonging to the
|
||||
// given tasks and returns how many rows were actually cancelled.
|
||||
// given tasks and returns the rows actually cancelled, so a caller can
|
||||
// classify and count each one's loop-liveness terminal shape.
|
||||
//
|
||||
// The terminal write itself lives in the runs repository's CancelRunsWhere,
|
||||
// which applies the queued/claimed guard; only the by-task-id selector stays
|
||||
// here. json_extract is SQLite-flavoured — Postgres is a supported driver
|
||||
// (internal/persistence/provider.go) and would need dialect.JSONExtract — so
|
||||
// keeping it local avoids baking dialect-specific SQL into the shared writer.
|
||||
func (r *Repository) CancelRunsForTasks(ctx context.Context, taskIDs []string, reason string) (int, error) {
|
||||
func (r *Repository) CancelRunsForTasks(
|
||||
ctx context.Context, taskIDs []string, reason string,
|
||||
) ([]runssqlite.CancelledRun, error) {
|
||||
if len(taskIDs) == 0 {
|
||||
return 0, nil
|
||||
return nil, nil
|
||||
}
|
||||
selector, args := taskIDInQuery(`json_extract(payload, '$.task_id') IN (%s)`, taskIDs)
|
||||
rows, err := r.CancelRunsWhere(ctx, reason, selector, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(rows), nil
|
||||
return r.CancelRunsWhere(ctx, reason, selector, args...)
|
||||
}
|
||||
|
||||
func (r *Repository) BulkUpdateTaskState(ctx context.Context, taskIDs []string, state string) error {
|
||||
|
||||
@@ -49,6 +49,11 @@ type WakeupRequest struct {
|
||||
RequestedAt time.Time `db:"requested_at"`
|
||||
ClaimedAt sql.NullTime `db:"claimed_at"`
|
||||
FinishedAt sql.NullTime `db:"finished_at"`
|
||||
// CausationID is copied from the routine fire that produced this
|
||||
// wake, or minted here when the wake has no routine origin
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-002). "" for rows written before this
|
||||
// feature or for a wake with no identifiable origin.
|
||||
CausationID string `db:"causation_id"`
|
||||
}
|
||||
|
||||
// CreateWakeupRequest inserts a new wakeup-request row. When
|
||||
@@ -85,12 +90,12 @@ func (r *Repository) CreateWakeupRequest(ctx context.Context, req *WakeupRequest
|
||||
INSERT INTO agent_wakeup_requests (
|
||||
id, agent_profile_id, source, reason, payload, status,
|
||||
coalesced_count, idempotency_key, run_id,
|
||||
requested_at, claimed_at, finished_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
requested_at, claimed_at, finished_at, causation_id
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`),
|
||||
req.ID, req.AgentProfileID, req.Source, req.Reason, req.Payload, req.Status,
|
||||
req.CoalescedCount, req.IdempotencyKey, req.RunID,
|
||||
req.RequestedAt, req.ClaimedAt, req.FinishedAt,
|
||||
req.RequestedAt, req.ClaimedAt, req.FinishedAt, req.CausationID,
|
||||
)
|
||||
if err != nil && isUniqueConstraintErr(err) {
|
||||
return fmt.Errorf("%w: %v", ErrWakeupIdempotencyConflict, err)
|
||||
@@ -105,7 +110,7 @@ func (r *Repository) GetWakeupRequest(ctx context.Context, id string) (*WakeupRe
|
||||
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
|
||||
SELECT id, agent_profile_id, source, reason, payload, status,
|
||||
coalesced_count, idempotency_key, run_id,
|
||||
requested_at, claimed_at, finished_at
|
||||
requested_at, claimed_at, finished_at, causation_id
|
||||
FROM agent_wakeup_requests
|
||||
WHERE id = ?
|
||||
`), id).StructScan(&row)
|
||||
@@ -125,7 +130,7 @@ func (r *Repository) ListQueuedWakeupRequestsForAgent(
|
||||
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
|
||||
SELECT id, agent_profile_id, source, reason, payload, status,
|
||||
coalesced_count, idempotency_key, run_id,
|
||||
requested_at, claimed_at, finished_at
|
||||
requested_at, claimed_at, finished_at, causation_id
|
||||
FROM agent_wakeup_requests
|
||||
WHERE agent_profile_id = ? AND status = ?
|
||||
ORDER BY requested_at ASC
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routines"
|
||||
)
|
||||
|
||||
// newTestRoutineServiceWithTaskTables mirrors newTestRoutineService but
|
||||
// also creates minimal tasks/task_sessions tables on the same database,
|
||||
// so a heavy-path test can walk the real
|
||||
// office_routine_runs.linked_task_id -> tasks.id -> task_sessions.task_id
|
||||
// chain instead of only asserting the id fields in isolation.
|
||||
func newTestRoutineServiceWithTaskTables(t *testing.T) (*routines.RoutineService, *sqlx.DB) {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repo, err := sqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT ''
|
||||
)`); err != nil {
|
||||
t.Fatalf("create tasks table: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS task_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL
|
||||
)`); err != nil {
|
||||
t.Fatalf("create task_sessions table: %v", err)
|
||||
}
|
||||
|
||||
log := logger.Default()
|
||||
return routines.NewRoutineService(repo, log, &noopActivity{}), db
|
||||
}
|
||||
|
||||
// dbBackedTaskCreator is CreateOfficeTaskInWorkflow's fake for this
|
||||
// file only: unlike fakeTaskCreator (a synthetic id with nothing
|
||||
// behind it), it inserts a real tasks row plus a task_sessions row
|
||||
// naming it, so the produced id is actually reachable by a join —
|
||||
// standing in for auto_start_agent launching the routine's task.
|
||||
type dbBackedTaskCreator struct {
|
||||
db *sqlx.DB
|
||||
taskID string
|
||||
sessID string
|
||||
nextSeq int
|
||||
captured struct{ workflowID string }
|
||||
}
|
||||
|
||||
func (f *dbBackedTaskCreator) CreateOfficeTaskInWorkflow(
|
||||
_ context.Context, workspaceID, _, _, workflowID, title, _ string,
|
||||
) (string, error) {
|
||||
f.nextSeq++
|
||||
f.captured.workflowID = workflowID
|
||||
f.taskID = "task-heavy-causation"
|
||||
f.sessID = "session-heavy-causation"
|
||||
if _, err := f.db.Exec(`INSERT INTO tasks (id, workspace_id, title) VALUES (?, ?, ?)`,
|
||||
f.taskID, workspaceID, title); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := f.db.Exec(`INSERT INTO task_sessions (id, task_id) VALUES (?, ?)`,
|
||||
f.sessID, f.taskID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return f.taskID, nil
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.9: a heavy routine fire's run row carries
|
||||
// both the minted causation id and the linked task id, and that task id
|
||||
// is a real, reachable row — walkable forward to the session the task
|
||||
// launched, the correlation chain a loop-liveness diagnosis depends on.
|
||||
func TestDispatchRoutineRun_HeavyPathCausationIDAndLinkedTaskReachSession(t *testing.T) {
|
||||
svc, db := newTestRoutineServiceWithTaskTables(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := &models.Routine{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "Heavy Causation",
|
||||
TaskTemplate: `{"title":"Heavy {{name}}","description":"D"}`,
|
||||
AssigneeAgentProfileID: "agent-heavy",
|
||||
Status: "active",
|
||||
ConcurrencyPolicy: "always_create",
|
||||
Variables: `{"name":{"default":"routine"}}`,
|
||||
}
|
||||
if err := svc.CreateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("create routine: %v", err)
|
||||
}
|
||||
|
||||
wf := &fakeWorkflowEnsurer{}
|
||||
tc := &dbBackedTaskCreator{db: db}
|
||||
svc.SetWorkflowEnsurer(wf)
|
||||
svc.SetTaskCreator(tc)
|
||||
|
||||
run, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fire manual: %v", err)
|
||||
}
|
||||
if run.CausationID == "" {
|
||||
t.Fatal("expected non-empty causation id on the heavy-path run")
|
||||
}
|
||||
if run.LinkedTaskID != tc.taskID {
|
||||
t.Fatalf("LinkedTaskID = %q, want %q", run.LinkedTaskID, tc.taskID)
|
||||
}
|
||||
|
||||
var sessionID string
|
||||
err = db.QueryRow(`
|
||||
SELECT task_sessions.id
|
||||
FROM office_routine_runs
|
||||
JOIN tasks ON tasks.id = office_routine_runs.linked_task_id
|
||||
JOIN task_sessions ON task_sessions.task_id = tasks.id
|
||||
WHERE office_routine_runs.id = ?
|
||||
`, run.ID).Scan(&sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("two-hop join from routine run to session: %v", err)
|
||||
}
|
||||
if sessionID != tc.sessID {
|
||||
t.Fatalf("joined session id = %q, want %q", sessionID, tc.sessID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.1: dispatchRoutineRun mints one causation
|
||||
// id per fire — non-empty and distinct across separate fires.
|
||||
func TestDispatchRoutineRun_MintsOneCausationIDPerFire(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
routine := createTestRoutine(t, svc, "Causation Mint", "always_create")
|
||||
|
||||
first, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
if first.CausationID == "" {
|
||||
t.Fatal("expected non-empty causation id on first fire")
|
||||
}
|
||||
|
||||
second, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
if second.CausationID == "" {
|
||||
t.Fatal("expected non-empty causation id on second fire")
|
||||
}
|
||||
if second.CausationID == first.CausationID {
|
||||
t.Fatalf("expected distinct causation ids across fires, both were %q", first.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.2: the lightweight (taskless) path copies
|
||||
// the fire's minted causation id onto the WakeupRequest it builds.
|
||||
func TestDispatchRoutineRun_LightweightPathCopiesCausationIDOntoWakeupRequest(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := &models.Routine{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "Lightweight Causation",
|
||||
TaskTemplate: "", // lightweight
|
||||
AssigneeAgentProfileID: "agent-1",
|
||||
Status: "active",
|
||||
ConcurrencyPolicy: "always_create",
|
||||
}
|
||||
if err := svc.CreateRoutine(ctx, routine); err != nil {
|
||||
t.Fatalf("create routine: %v", err)
|
||||
}
|
||||
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
run, err := svc.FireManual(ctx, routine.ID, map[string]string{"name": "alpha"})
|
||||
if err != nil {
|
||||
t.Fatalf("fire manual: %v", err)
|
||||
}
|
||||
if run.CausationID == "" {
|
||||
t.Fatal("expected non-empty causation id on the routine run")
|
||||
}
|
||||
if len(enq.created) != 1 {
|
||||
t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created))
|
||||
}
|
||||
if enq.created[0].CausationID != run.CausationID {
|
||||
t.Fatalf("wakeup request causation id = %q, want %q (copied from the fire)",
|
||||
enq.created[0].CausationID, run.CausationID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// loopCounterInt reads one labelled entry off a registered
|
||||
// office_loop_* expvar map. Missing entries read as zero so a test can
|
||||
// assert a fresh delta without pre-seeding the map.
|
||||
func loopCounterInt(t *testing.T, mapName, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get(mapName)
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map %s not registered", mapName)
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("%s is not a Map", mapName)
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("%s[%q] is not an Int", mapName, key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// REQ-OFFICE-LOOP-LIVENESS-003 / AC-003.1: TickScheduledTriggers counts
|
||||
// one cron loop tick evaluation regardless of whether any trigger is due.
|
||||
func TestTickScheduledTriggers_IncrementsCronTickCounter(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
before := loopCounterInt(t, "office_loop_cron_tick_total", "")
|
||||
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC()); err != nil {
|
||||
t.Fatalf("tick: %v", err)
|
||||
}
|
||||
after := loopCounterInt(t, "office_loop_cron_tick_total", "")
|
||||
if after != before+1 {
|
||||
t.Fatalf("cron_tick delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.1/AC-003.9: a due, successfully-claimed trigger increments
|
||||
// office_loop_trigger_claimed_total labelled by the owning routine's
|
||||
// workspace, at the site the claim persists.
|
||||
func TestTickScheduledTriggers_IncrementsTriggerClaimedCounter(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{
|
||||
RoutineID: routine.ID,
|
||||
Kind: "cron",
|
||||
CronExpression: "* * * * *",
|
||||
Timezone: "UTC",
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID
|
||||
before := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
|
||||
|
||||
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("tick: %v", err)
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("trigger_claimed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.9: a claim that persists must count even when the routine
|
||||
// lookup afterward fails, so the claim can't be attributed to a
|
||||
// workspace. Without the fallback, a claim that already wrote to the
|
||||
// database silently vanished from office_loop_trigger_claimed_total
|
||||
// (Review round 1, R1-4) — the counter must reflect the persisted
|
||||
// write, not the lookup that happens to follow it.
|
||||
func TestTickScheduledTriggers_ClaimedCounterSurvivesRoutineLookupFailure(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
if err := svc.CreateRoutineTrigger(ctx, &models.RoutineTrigger{
|
||||
RoutineID: routine.ID,
|
||||
Kind: "cron",
|
||||
CronExpression: "* * * * *",
|
||||
Timezone: "UTC",
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
// Orphan the trigger: delete the routine it points at without
|
||||
// touching the trigger row, so ClaimTrigger still succeeds but the
|
||||
// subsequent GetRoutineFromConfig fails.
|
||||
if err := svc.DeleteRoutine(ctx, routine.ID); err != nil {
|
||||
t.Fatalf("delete routine: %v", err)
|
||||
}
|
||||
|
||||
key := "workspace=_unattributed"
|
||||
before := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
|
||||
|
||||
if err := svc.TickScheduledTriggers(ctx, time.Now().UTC().Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("tick: %v", err)
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_trigger_claimed_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("trigger_claimed[_unattributed] delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.2: a fire that runs to completion (lightweight, always_create)
|
||||
// counts office_loop_routine_run_total with disposition=task_created.
|
||||
func TestDispatchRoutineRun_CountsTaskCreatedDisposition(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created"
|
||||
before := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("fire manual: %v", err)
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("routine_run[task_created] delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.2: a fire that the concurrency policy skips counts
|
||||
// office_loop_routine_run_total with disposition=skipped, distinct from
|
||||
// the launched fire that decided the skip.
|
||||
func TestDispatchRoutineRun_CountsSkippedDisposition(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
|
||||
svc.SetTaskCreator(&fakeTaskCreator{})
|
||||
|
||||
routine := createTestRoutine(t, svc, "Skip Counter", "skip_if_active")
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=skipped"
|
||||
before := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("routine_run[skipped] delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.2: a fire the concurrency policy coalesces into an existing
|
||||
// active run counts office_loop_routine_run_total with
|
||||
// disposition=coalesced.
|
||||
func TestDispatchRoutineRun_CountsCoalescedDisposition(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
|
||||
svc.SetTaskCreator(&fakeTaskCreator{})
|
||||
|
||||
routine := createTestRoutine(t, svc, "Coalesce Counter", "coalesce_if_active")
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=coalesced"
|
||||
before := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("routine_run[coalesced] delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-003.1: the lightweight path's successfully-created wakeup request
|
||||
// counts office_loop_wakeup_created_total labelled by the row's own
|
||||
// source column, reusing routines.WakeupRequest.Source rather than
|
||||
// inventing a parallel vocabulary.
|
||||
func TestMaterialiseLightweightRoutineRun_IncrementsWakeupCreatedCounter(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
enq := &fakeWakeupEnqueuer{}
|
||||
svc.SetWakeupEnqueuer(enq)
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID + ";source=routine"
|
||||
before := loopCounterInt(t, "office_loop_wakeup_created_total", key)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("fire manual: %v", err)
|
||||
}
|
||||
if len(enq.created) != 1 {
|
||||
t.Fatalf("expected 1 wakeup request created, got %d", len(enq.created))
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_wakeup_created_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("wakeup_created delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routines"
|
||||
)
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.1: last_run_at advances on the coalesced
|
||||
// and skipped concurrency-policy paths too, not only on a fresh
|
||||
// dispatch — the call site runs before applyConcurrencyPolicy can
|
||||
// short-circuit the return.
|
||||
func TestDispatch_AdvancesLastRunAtOnSkippedAndCoalescedPaths(t *testing.T) {
|
||||
t.Run("skipped", func(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
|
||||
svc.SetTaskCreator(&fakeTaskCreator{})
|
||||
routine := createTestRoutine(t, svc, "Skip Last Run", "skip_if_active")
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
afterFirst, err := svc.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if afterFirst.LastRunAt == nil {
|
||||
t.Fatal("expected last_run_at set after first fire")
|
||||
}
|
||||
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
run2, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
if run2.Status != "skipped" {
|
||||
t.Fatalf("second fire status = %q, want skipped", run2.Status)
|
||||
}
|
||||
afterSecond, err := svc.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) {
|
||||
t.Fatalf("last_run_at did not advance on skipped fire: before %v, after %v",
|
||||
afterFirst.LastRunAt, afterSecond.LastRunAt)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("coalesced", func(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
|
||||
svc.SetTaskCreator(&fakeTaskCreator{})
|
||||
routine := createTestRoutine(t, svc, "Coalesce Last Run", "coalesce_if_active")
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err != nil {
|
||||
t.Fatalf("first fire: %v", err)
|
||||
}
|
||||
afterFirst, err := svc.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
run2, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
}
|
||||
if run2.Status != "coalesced" {
|
||||
t.Fatalf("second fire status = %q, want coalesced", run2.Status)
|
||||
}
|
||||
afterSecond, err := svc.GetRoutine(ctx, routine.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !afterSecond.LastRunAt.After(*afterFirst.LastRunAt) {
|
||||
t.Fatalf("last_run_at did not advance on coalesced fire: before %v, after %v",
|
||||
afterFirst.LastRunAt, afterSecond.LastRunAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// failingTouchRepo wraps the real sqlite repository but forces
|
||||
// TouchRoutineLastRun to fail, so the dispatch's error-tolerance
|
||||
// (AC-OFFICE-LOOP-LIVENESS-001.7) can be exercised without a real
|
||||
// storage failure.
|
||||
type failingTouchRepo struct {
|
||||
*sqlite.Repository
|
||||
}
|
||||
|
||||
func (f *failingTouchRepo) TouchRoutineLastRun(context.Context, string, time.Time) error {
|
||||
return errors.New("simulated last_run_at write failure")
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.7: a TouchRoutineLastRun error does not
|
||||
// fail the dispatch — the routine still fires and its run is created.
|
||||
func TestDispatch_SurvivesLastRunAtWriteFailure(t *testing.T) {
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
repo, err := sqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
svc := routines.NewRoutineService(&failingTouchRepo{Repository: repo}, logger.Default(), &noopActivity{})
|
||||
svc.SetWorkflowEnsurer(&fakeWorkflowEnsurer{})
|
||||
svc.SetTaskCreator(&fakeTaskCreator{})
|
||||
|
||||
routine := createTestRoutine(t, svc, "Failing Touch", "always_create")
|
||||
run, err := svc.FireManual(context.Background(), routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fire manual with failing touch: %v", err)
|
||||
}
|
||||
if run.Status != "task_created" {
|
||||
t.Fatalf("status = %q, want task_created", run.Status)
|
||||
}
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routines"
|
||||
)
|
||||
|
||||
// newTestRoutineServiceRejectingRunStatusWrite is newTestRoutineService
|
||||
// plus a trigger that aborts every UPDATE of office_routine_runs.status
|
||||
// to 'task_created' or 'done', so a test can force materialiseRoutineRun's
|
||||
// write to fail (on either the heavy or the lightweight path) without
|
||||
// disturbing CreateRoutineRun (the write whose success is what
|
||||
// office_loop_routine_run_total must count).
|
||||
func newTestRoutineServiceRejectingRunStatusWrite(t *testing.T) *routines.RoutineService {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repo, err := sqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
CREATE TRIGGER reject_run_status_write
|
||||
BEFORE UPDATE OF status ON office_routine_runs
|
||||
WHEN NEW.status IN ('task_created', 'done')
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'run status write rejected for test');
|
||||
END;
|
||||
`); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
|
||||
log := logger.Default()
|
||||
return routines.NewRoutineService(repo, log, &noopActivity{})
|
||||
}
|
||||
|
||||
// AC-003.1 / AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total
|
||||
// must count a routine run CreateRoutineRun genuinely persisted even when
|
||||
// materialiseRoutineRun errors out afterward. Before the fix,
|
||||
// IncLoopRoutineRun was only called inside the applyConcurrencyPolicy and
|
||||
// materialiseRoutineRun success branches — an error from either meant
|
||||
// dispatchRoutineRun returned without incrementing the counter at all,
|
||||
// silently undercounting a fire that genuinely created a persisted row
|
||||
// (the exact defect class this capability exists to catch: the reference
|
||||
// incident was 322 consecutive fires that read as healthy because
|
||||
// nothing counted them correctly).
|
||||
func TestDispatchRoutineRun_CountsRunEvenWhenMaterializeFails(t *testing.T) {
|
||||
svc := newTestRoutineServiceRejectingRunStatusWrite(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
|
||||
key := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed"
|
||||
before := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
|
||||
if _, err := svc.FireManual(ctx, routine.ID, nil); err == nil {
|
||||
t.Fatal("expected FireManual to return an error from the rejected status write")
|
||||
}
|
||||
|
||||
after := loopCounterInt(t, "office_loop_routine_run_total", key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("routine_run[failed] delta = %d, want 1 (the row was genuinely created, must not go uncounted)", after-before)
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
taskservice "github.com/kandev/kandev/internal/task/service"
|
||||
)
|
||||
@@ -38,6 +39,7 @@ type Repository interface {
|
||||
ListRoutines(ctx context.Context, workspaceID string) ([]*Routine, error)
|
||||
UpdateRoutine(ctx context.Context, routine *Routine) error
|
||||
DeleteRoutine(ctx context.Context, id string) error
|
||||
TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error
|
||||
|
||||
CreateRoutineTrigger(ctx context.Context, t *RoutineTrigger) error
|
||||
ListTriggersByRoutineID(ctx context.Context, routineID string) ([]*RoutineTrigger, error)
|
||||
@@ -62,7 +64,6 @@ type Repository interface {
|
||||
// read-then-write race between them.
|
||||
UpdateRunStatusIfTaskCreated(ctx context.Context, runID string, status models.RoutineRunStatus, linkedTaskID string) (bool, error)
|
||||
UpdateRunCoalesced(ctx context.Context, runID, coalescedIntoRunID string) error
|
||||
TouchRoutineLastRun(ctx context.Context, routineID string, at time.Time) error
|
||||
// GetTaskTerminalStatus reads a task's real lifecycle state directly
|
||||
// (not via the TaskMoved event — see applyConcurrencyPolicy) and
|
||||
// reports "" when it is active, "done", "failed", "cancelled", or
|
||||
@@ -95,6 +96,10 @@ type WakeupRequest struct {
|
||||
Payload string
|
||||
IdempotencyKey string
|
||||
RequestedAt time.Time
|
||||
// CausationID is copied from the firing RoutineRun
|
||||
// (AC-OFFICE-LOOP-LIVENESS-002.2) so downstream runs and wakeup
|
||||
// requests correlate back to the fire that produced them.
|
||||
CausationID string
|
||||
}
|
||||
|
||||
// RoutineWorkflowEnsurer materialises (lazily) the routine system
|
||||
@@ -384,6 +389,7 @@ func (s *RoutineService) ListAllRoutineRuns(ctx context.Context, wsID string, li
|
||||
|
||||
// TickScheduledTriggers queries due cron triggers, claims each, and dispatches.
|
||||
func (s *RoutineService) TickScheduledTriggers(ctx context.Context, now time.Time) error {
|
||||
service.IncLoopCronTick(now.Format(time.RFC3339))
|
||||
triggers, err := s.repo.GetDueTriggers(ctx, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get due triggers: %w", err)
|
||||
@@ -407,8 +413,12 @@ func (s *RoutineService) processCronTrigger(ctx context.Context, trigger *Routin
|
||||
}
|
||||
routine, err := s.GetRoutineFromConfig(ctx, trigger.RoutineID)
|
||||
if err != nil {
|
||||
// The claim already persisted, so the counter must still see it
|
||||
// even though the workspace it belongs to can't be resolved.
|
||||
service.IncLoopTriggerClaimed(service.LoopUnattributedWorkspace)
|
||||
return fmt.Errorf("get routine: %w", err)
|
||||
}
|
||||
service.IncLoopTriggerClaimed(routine.WorkspaceID)
|
||||
// Catch-up cap: count how many cron ticks elapsed between the missed
|
||||
// trigger.NextRunAt (inclusive) and now, capped at routine.CatchUpMax.
|
||||
// Mirror of the agent_heartbeat catch-up math, adapted for cron
|
||||
@@ -587,28 +597,65 @@ func (s *RoutineService) dispatchRoutineRun(
|
||||
TriggerPayload: string(payloadJSON),
|
||||
DispatchFingerprint: fingerprint,
|
||||
StartedAt: &now,
|
||||
// CausationID is minted once per fire (AC-OFFICE-LOOP-LIVENESS-002.1)
|
||||
// and copied onto every wakeup request and run this fire produces.
|
||||
CausationID: uuid.New().String(),
|
||||
}
|
||||
if err := s.repo.CreateRoutineRun(ctx, run); err != nil {
|
||||
return nil, fmt.Errorf("create run: %w", err)
|
||||
}
|
||||
|
||||
// AC-003.1/AC-003.9 (Review round 3, R3-4): office_loop_routine_run_total
|
||||
// counts this fire's CreateRoutineRun above — a write that just
|
||||
// succeeded — not whatever applyConcurrencyPolicy/materialiseRoutineRun
|
||||
// do afterward. A single deferred increment fires exactly once no
|
||||
// matter which return path below is taken, labelled by whatever
|
||||
// disposition was reached; it defaults to "failed" (the existing
|
||||
// RoutineRunStatus value, not an invented one) so a fire that created
|
||||
// a real row but then errored is still counted rather than silently
|
||||
// dropped.
|
||||
disposition := string(models.RoutineRunStatusFailed)
|
||||
defer func() {
|
||||
service.IncLoopRoutineRun(routine.WorkspaceID, source, disposition)
|
||||
}()
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.1: advances last_run_at for every
|
||||
// dispatch, whatever disposition the run later reaches — so it must
|
||||
// run before applyConcurrencyPolicy, which can short-circuit into a
|
||||
// coalesced or skipped return. Errors are logged and counted, never
|
||||
// returned (AC-001.7): a routine that fired must not read as having
|
||||
// failed to fire because a bookkeeping write lost a lock.
|
||||
if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, *run.StartedAt); err != nil {
|
||||
service.IncLoopLastRunAtWriteFailed(routine.WorkspaceID)
|
||||
s.logger.Warn("touch routine last_run_at",
|
||||
zap.String("routine_id", routine.ID), zap.Error(err))
|
||||
}
|
||||
|
||||
status, err := s.applyConcurrencyPolicy(ctx, routine, run, fingerprint)
|
||||
if err != nil {
|
||||
return run, err
|
||||
}
|
||||
if status != "" {
|
||||
disposition = string(status)
|
||||
return run, nil
|
||||
}
|
||||
|
||||
if err := s.materialiseRoutineRun(ctx, routine, run, tmpl, title, description, vars, source, idempotencyKey, missedTicks); err != nil {
|
||||
return run, err
|
||||
}
|
||||
|
||||
if err := s.repo.TouchRoutineLastRun(ctx, routine.ID, now); err != nil {
|
||||
s.logger.Warn("touch routine last_run_at",
|
||||
zap.String("routine", routine.Name), zap.Error(err))
|
||||
// A nil error only means materialiseRoutineRun's terminal write
|
||||
// succeeded, not that the write it made was a success: the
|
||||
// lightweight path finalizes with a nil error even when the wakeup
|
||||
// enqueue itself failed, persisting run.Status=failed. Disposition
|
||||
// must reflect that instead of assuming "task_created" (the
|
||||
// dispatched-successfully label, shared by the heavy path and a
|
||||
// lightweight done) whenever this call merely didn't error.
|
||||
if run.Status == models.RoutineRunStatusFailed {
|
||||
disposition = string(models.RoutineRunStatusFailed)
|
||||
} else {
|
||||
disposition = string(models.RoutineRunStatusTaskCreated)
|
||||
}
|
||||
routine.LastRunAt = &now
|
||||
|
||||
s.logger.Info("routine run dispatched",
|
||||
zap.String("routine", routine.Name),
|
||||
zap.String("run_id", run.ID),
|
||||
@@ -713,6 +760,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun(
|
||||
Payload: payloadStr,
|
||||
IdempotencyKey: idemKey,
|
||||
RequestedAt: time.Now().UTC(),
|
||||
CausationID: run.CausationID,
|
||||
}
|
||||
if err := s.wakeup.CreateWakeupRequest(ctx, req); err != nil {
|
||||
if errors.Is(err, ErrWakeupAlreadyRequested) {
|
||||
@@ -724,6 +772,7 @@ func (s *RoutineService) materialiseLightweightRoutineRun(
|
||||
zap.String("routine", routine.Name), zap.Error(err))
|
||||
return s.finalizeLightweightRun(ctx, run, models.RoutineRunStatusFailed)
|
||||
}
|
||||
service.IncLoopWakeupCreated(routine.WorkspaceID, req.Source)
|
||||
if err := s.wakeup.Dispatch(ctx, req.ID); err != nil {
|
||||
s.logger.Warn("dispatch routine wakeup request",
|
||||
zap.String("routine", routine.Name),
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
@@ -716,9 +717,13 @@ func TestDispatch_LastRunAt(t *testing.T) {
|
||||
t.Fatal("expected last_run_at to be set after a materialised fire")
|
||||
}
|
||||
firstStamp := *afterFirst.LastRunAt
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
|
||||
// Second fire is skipped (run1's task is still open) — last_run_at
|
||||
// must not advance for a fire that didn't materialise.
|
||||
// Second fire is skipped (run1's task is still open), but
|
||||
// AC-OFFICE-LOOP-LIVENESS-001.1 advances last_run_at for every
|
||||
// dispatch regardless of disposition — the write runs before
|
||||
// applyConcurrencyPolicy can short-circuit into skipped/coalesced,
|
||||
// so a skip still counts as evidence the loop is alive.
|
||||
run2, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("second fire: %v", err)
|
||||
@@ -730,7 +735,7 @@ func TestDispatch_LastRunAt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get routine: %v", err)
|
||||
}
|
||||
if !afterSecond.LastRunAt.Equal(firstStamp) {
|
||||
t.Errorf("last_run_at changed from %v to %v after a skipped fire", firstStamp, afterSecond.LastRunAt)
|
||||
if !afterSecond.LastRunAt.After(firstStamp) {
|
||||
t.Errorf("last_run_at did not advance on a skipped fire: before %v, after %v", firstStamp, afterSecond.LastRunAt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package routines_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/routines"
|
||||
)
|
||||
|
||||
// failingCreateWakeupEnqueuer fails every CreateWakeupRequest call with a
|
||||
// generic (non-ErrWakeupAlreadyRequested) error, but still lets Dispatch/
|
||||
// FailWakeupRequest succeed trivially — they are never reached once
|
||||
// CreateWakeupRequest itself fails.
|
||||
type failingCreateWakeupEnqueuer struct{ fakeWakeupEnqueuer }
|
||||
|
||||
func (f *failingCreateWakeupEnqueuer) CreateWakeupRequest(context.Context, *routines.WakeupRequest) error {
|
||||
return errors.New("wakeup store unavailable")
|
||||
}
|
||||
|
||||
// TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated is the
|
||||
// PR fixup round 2 regression test (CodeRabbit). materialiseLightweightRoutineRun
|
||||
// finalizes a run as RoutineRunStatusFailed when CreateWakeupRequest errors,
|
||||
// but returns a nil error once that finalizing UPDATE itself succeeds. The
|
||||
// caller (dispatchRoutineRun) used to read that nil error as "materialise
|
||||
// succeeded" and hard-code disposition = task_created regardless of what
|
||||
// status was actually persisted — so a routine run that genuinely failed to
|
||||
// enqueue its wakeup still recorded office_loop_routine_run_total as a
|
||||
// success, corrupting the loop-health evidence this capability exists to
|
||||
// produce.
|
||||
func TestDispatchRoutineRun_FailedWakeupEnqueueDoesNotReportTaskCreated(t *testing.T) {
|
||||
svc := newTestRoutineService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
routine := newLightweightTestRoutine(t, svc)
|
||||
svc.SetWakeupEnqueuer(&failingCreateWakeupEnqueuer{})
|
||||
|
||||
failedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=failed"
|
||||
taskCreatedKey := "workspace=" + routine.WorkspaceID + ";source=manual;disposition=task_created"
|
||||
failedBefore := loopCounterInt(t, "office_loop_routine_run_total", failedKey)
|
||||
taskCreatedBefore := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey)
|
||||
|
||||
run, err := svc.FireManual(ctx, routine.ID, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("FireManual: %v", err)
|
||||
}
|
||||
if run.Status != models.RoutineRunStatusFailed {
|
||||
t.Fatalf("run.Status = %q, want %q — the wakeup enqueue failed and finalizeLightweightRun persisted 'failed'",
|
||||
run.Status, models.RoutineRunStatusFailed)
|
||||
}
|
||||
|
||||
failedAfter := loopCounterInt(t, "office_loop_routine_run_total", failedKey)
|
||||
if failedAfter != failedBefore+1 {
|
||||
t.Fatalf("routine_run[failed] delta = %d, want 1 — a failed wakeup enqueue must not be "+
|
||||
"reported as disposition=task_created", failedAfter-failedBefore)
|
||||
}
|
||||
|
||||
taskCreatedAfter := loopCounterInt(t, "office_loop_routine_run_total", taskCreatedKey)
|
||||
if taskCreatedAfter != taskCreatedBefore {
|
||||
t.Fatalf("routine_run[task_created] delta = %d, want 0 for a run whose wakeup enqueue failed",
|
||||
taskCreatedAfter-taskCreatedBefore)
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,23 @@ import (
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
sqliterepo "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// TaskStarterWithSession optionally returns the id of the agent session
|
||||
// a routed launch created, mirroring service.TaskStarterWithSession for
|
||||
// the legacy launch path (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that
|
||||
// does not implement this falls back to StartTaskWithRoute, and the
|
||||
// run's session id stays empty.
|
||||
type TaskStarterWithSession interface {
|
||||
StartTaskWithRouteReturningSession(
|
||||
ctx context.Context,
|
||||
taskID, agentProfileID string,
|
||||
launch LaunchContext,
|
||||
route RouteOverride,
|
||||
) (sessionID string, err error)
|
||||
}
|
||||
|
||||
// RouteAttemptOutcomeLaunched is the in-flight outcome string the
|
||||
// dispatcher writes when an attempt is appended. It is overwritten with
|
||||
// failed_provider_unavailable / failed_other on classifier verdict.
|
||||
@@ -257,7 +272,7 @@ func (ss *SchedulerService) tryCandidates(
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch)
|
||||
sessionID, launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch)
|
||||
if launchErr == nil {
|
||||
if prev != nil {
|
||||
// We walked past at least one prior candidate — that's
|
||||
@@ -270,7 +285,7 @@ func (ss *SchedulerService) tryCandidates(
|
||||
}
|
||||
ss.recordRouteAttempt(agent.WorkspaceID,
|
||||
string(candidate.ProviderID), metricOutcomeSuccess, "")
|
||||
return ss.handleLaunchSuccess(ctx, run, agent, candidate)
|
||||
return ss.handleLaunchSuccess(ctx, run, agent, candidate, sessionID)
|
||||
}
|
||||
// Emit the failure-side route_attempt before classifying. The
|
||||
// outcome label is refined by handleLaunchFailure.
|
||||
@@ -402,20 +417,20 @@ func (ss *SchedulerService) recordAttemptStart(
|
||||
func (ss *SchedulerService) launchCandidate(
|
||||
ctx context.Context, taskID, agentID string,
|
||||
candidate routing.Candidate, launch LaunchContext,
|
||||
) error {
|
||||
) (string, error) {
|
||||
if taskID == "" {
|
||||
return fmt.Errorf("dispatch: empty task id in run payload")
|
||||
return "", fmt.Errorf("dispatch: empty task id in run payload")
|
||||
}
|
||||
if _, ok := routingerr.InjectedCode(string(candidate.ProviderID)); ok {
|
||||
// Synthesize a launch failure via Classify so injection is
|
||||
// honoured for deterministic E2E specs. Classify short-circuits
|
||||
// to the injected code at the head of its decision chain.
|
||||
return routingerr.Classify(routingerr.Input{
|
||||
return "", routingerr.Classify(routingerr.Input{
|
||||
Phase: routingerr.PhaseProcessStart,
|
||||
ProviderID: string(candidate.ProviderID),
|
||||
})
|
||||
}
|
||||
return ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, RouteOverride{
|
||||
route := RouteOverride{
|
||||
ExecutionProfileID: candidate.ExecutionProfileID,
|
||||
ProviderID: string(candidate.ProviderID),
|
||||
Model: candidate.Model,
|
||||
@@ -423,24 +438,72 @@ func (ss *SchedulerService) launchCandidate(
|
||||
Mode: candidate.Mode,
|
||||
Flags: candidate.Flags,
|
||||
Env: candidate.Env,
|
||||
})
|
||||
}
|
||||
if starter, ok := ss.taskStarter.(TaskStarterWithSession); ok {
|
||||
return starter.StartTaskWithRouteReturningSession(ctx, taskID, agentID, launch, route)
|
||||
}
|
||||
return "", ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, route)
|
||||
}
|
||||
|
||||
// handleLaunchSuccess records the resolved provider/model on the run
|
||||
// row and flips every health scope this candidate touched back to
|
||||
// healthy. Returns (launched=true, nil, nil) on success.
|
||||
// handleLaunchSuccess persists the launched session id, counts the
|
||||
// launch, records the resolved provider/model on the run row, and
|
||||
// flips every health scope this candidate touched back to healthy.
|
||||
// Always returns (launched=true, nil, nil): by the time this is called
|
||||
// launchCandidate has already started a real agent session, so nothing
|
||||
// downstream may report the launch as failed or leave it uncounted
|
||||
// (AC-002.7, AC-003.9). The session id and the launch counter are
|
||||
// persisted first and unconditionally — they are the correlation this
|
||||
// capability exists to preserve — before the best-effort
|
||||
// resolved-route snapshot, whose own failure (Review round 3, R3-2)
|
||||
// must not undo them or cause a caller to retry-launch an agent that
|
||||
// is already running.
|
||||
func (ss *SchedulerService) handleLaunchSuccess(
|
||||
ctx context.Context, run *models.Run,
|
||||
agent *models.AgentInstance, candidate routing.Candidate,
|
||||
agent *models.AgentInstance, candidate routing.Candidate, sessionID string,
|
||||
) (bool, *routing.BlockReason, error) {
|
||||
service.IncLoopLaunch(agent.WorkspaceID)
|
||||
ss.persistLaunchedSession(ctx, run.ID, agent.WorkspaceID, sessionID)
|
||||
if err := ss.repo.SetRunResolvedRoute(ctx,
|
||||
run.ID, candidate.ExecutionProfileID, string(candidate.ProviderID), candidate.Model); err != nil {
|
||||
return false, nil, err
|
||||
ss.logger.Warn("persist resolved route failed",
|
||||
zap.String("run_id", run.ID), zap.String("provider_id", string(candidate.ProviderID)), zap.Error(err))
|
||||
}
|
||||
ss.markHealthScopes(ctx, agent.WorkspaceID, candidate)
|
||||
return true, nil, nil
|
||||
}
|
||||
|
||||
// persistLaunchedSession stores the session id a successful launch
|
||||
// produced (AC-OFFICE-LOOP-LIVENESS-002.7). An empty id is counted as a
|
||||
// without-session launch (AC-002.8) rather than attempted as a write.
|
||||
// A write failure does not fail the launch — the agent is already
|
||||
// running — and is counted separately from the without-session case
|
||||
// (AC-002.11) so the two causes of an identical "processed, no
|
||||
// session" row stay distinguishable in the counters. A non-empty id
|
||||
// whose write affects zero rows (e.g. the run row vanished between
|
||||
// claim and this write) is the same AC-002.11 case as a hard write
|
||||
// error — a real session id that failed to persist — not the AC-002.8
|
||||
// case, so it counts under the same persist-failed counter.
|
||||
func (ss *SchedulerService) persistLaunchedSession(
|
||||
ctx context.Context, runID, workspaceID, sessionID string,
|
||||
) {
|
||||
if sessionID == "" {
|
||||
service.IncLoopLaunchWithoutSession(workspaceID)
|
||||
return
|
||||
}
|
||||
wrote, err := ss.repo.SetRunSessionID(ctx, runID, sessionID)
|
||||
if err != nil {
|
||||
ss.logger.Warn("persist launched session id failed",
|
||||
zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err))
|
||||
service.IncLoopSessionPersistFailed(workspaceID)
|
||||
return
|
||||
}
|
||||
if !wrote {
|
||||
ss.logger.Warn("persist launched session id matched no row",
|
||||
zap.String("run_id", runID), zap.String("session_id", sessionID))
|
||||
service.IncLoopSessionPersistFailed(workspaceID)
|
||||
}
|
||||
}
|
||||
|
||||
// markHealthScopes marks the provider, tier, and model scopes healthy
|
||||
// for the launched candidate. Best-effort: errors are logged but do
|
||||
// not fail the launch.
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package scheduler_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/scheduler"
|
||||
)
|
||||
|
||||
// newTestRepoSchedRejectingResolvedRouteWrite is newTestRepoSched plus a
|
||||
// trigger that aborts every write to resolved_execution_profile_id, so a
|
||||
// test can force SetRunResolvedRoute to fail without disturbing the
|
||||
// session_id write persistLaunchedSession performs — the two writes
|
||||
// this test needs to tell apart.
|
||||
func newTestRepoSchedRejectingResolvedRouteWrite(t *testing.T) *officesqlite.Repository {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("settings store: %v", err)
|
||||
}
|
||||
repo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
CREATE TRIGGER reject_resolved_route_write
|
||||
BEFORE UPDATE OF resolved_execution_profile_id ON runs
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'resolved route write rejected for test');
|
||||
END;
|
||||
`); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
// AC-002.7 / AC-002.11 / AC-003.9 (Review round 3, R3-2): a launch that
|
||||
// already started a real agent session must not lose that session's
|
||||
// correlation, or go uncounted as a launch, just because an unrelated
|
||||
// bookkeeping write (the resolved provider/model snapshot) fails
|
||||
// afterward. Before the fix, handleLaunchSuccess returned early on a
|
||||
// SetRunResolvedRoute error, before ever reaching persistLaunchedSession
|
||||
// or IncLoopLaunch — silently abandoning a live, now-uncorrelated
|
||||
// session and under-counting a launch that genuinely happened.
|
||||
func TestDispatch_RoutedLaunchSurvivesResolvedRouteWriteFailure(t *testing.T) {
|
||||
repo := newTestRepoSchedRejectingResolvedRouteWrite(t)
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-resolved-route-fail"}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-resolved-route-fail"}`)
|
||||
|
||||
beforeLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
|
||||
|
||||
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
|
||||
if err != nil || !launched || parked {
|
||||
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the resolved-route write failure", launched, parked, err)
|
||||
}
|
||||
if !starter.returnedCall {
|
||||
t.Fatal("expected StartTaskWithRouteReturningSession to be called")
|
||||
}
|
||||
|
||||
updated, err := repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if updated.SessionID != "sess-resolved-route-fail" {
|
||||
t.Fatalf("session_id = %q, want sess-resolved-route-fail (a live launch must not lose its session correlation)", updated.SessionID)
|
||||
}
|
||||
|
||||
afterLaunch := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
|
||||
if afterLaunch != beforeLaunch+1 {
|
||||
t.Fatalf("office_loop_launch_total delta = %d, want 1 (the launch genuinely happened)", afterLaunch-beforeLaunch)
|
||||
}
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
package scheduler_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
settingsstore "github.com/kandev/kandev/internal/agent/settings/store"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/scheduler"
|
||||
)
|
||||
|
||||
// newTestRepoSchedRejectingSessionWrite is newTestRepoSched plus a
|
||||
// trigger that aborts only an UPDATE setting runs.session_id to
|
||||
// wantSessionID — the exact value the launch under test will produce —
|
||||
// so a test can force SetRunSessionID to fail (AC-002.11) without
|
||||
// disturbing any other write persistLaunchedSession's callers depend on
|
||||
// (route-attempt bookkeeping, resolved-route persistence).
|
||||
func newTestRepoSchedRejectingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("settings store: %v", err)
|
||||
}
|
||||
repo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(fmt.Sprintf(`
|
||||
CREATE TRIGGER reject_session_id_write
|
||||
BEFORE UPDATE OF session_id ON runs
|
||||
WHEN NEW.session_id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'session_id write rejected for test');
|
||||
END;
|
||||
`, wantSessionID)); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
// AC-002.11: on a routed launch, a session persist failure is counted
|
||||
// separately from a without-session launch, and does not fail the
|
||||
// launch itself or overwrite the run's session_id — the agent is
|
||||
// already running by the time this write happens.
|
||||
func TestDispatch_RoutedLaunchSessionPersistFailureCountsSeparately(t *testing.T) {
|
||||
repo := newTestRepoSchedRejectingSessionWrite(t, "sess-routed-persist-fail")
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-persist-fail"}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-session-persist-fail"}`)
|
||||
|
||||
beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
|
||||
beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
|
||||
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
|
||||
if err != nil || !launched || parked {
|
||||
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the persist failure", launched, parked, err)
|
||||
}
|
||||
|
||||
updated, err := repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if updated.SessionID != "" {
|
||||
t.Fatalf("session_id = %q, want unchanged/empty after a rejected write", updated.SessionID)
|
||||
}
|
||||
|
||||
afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
|
||||
if afterFailed != beforeFailed+1 {
|
||||
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed)
|
||||
}
|
||||
afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
if afterWithoutSession != beforeWithoutSession {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestRepoSchedSilentlyDroppingSessionWrite is newTestRepoSched plus a
|
||||
// trigger that uses RAISE(IGNORE) rather than RAISE(ABORT): the UPDATE
|
||||
// setting runs.session_id to wantSessionID matches zero rows and returns
|
||||
// no error, exactly the shape a run row that vanished between claim and
|
||||
// launch (an unrelated delete/archive racing the session-persist write)
|
||||
// would produce. This is a different failure mode than
|
||||
// newTestRepoSchedRejectingSessionWrite's hard DB error: SetRunSessionID
|
||||
// reports (wrote=false, err=nil), not (wrote=false, err=non-nil).
|
||||
func newTestRepoSchedSilentlyDroppingSessionWrite(t *testing.T, wantSessionID string) *officesqlite.Repository {
|
||||
t.Helper()
|
||||
db, err := sqlx.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
if _, _, err := settingsstore.Provide(db, db, nil); err != nil {
|
||||
t.Fatalf("settings store: %v", err)
|
||||
}
|
||||
repo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(fmt.Sprintf(`
|
||||
CREATE TRIGGER ignore_session_id_write
|
||||
BEFORE UPDATE OF session_id ON runs
|
||||
WHEN NEW.session_id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(IGNORE);
|
||||
END;
|
||||
`, wantSessionID)); err != nil {
|
||||
t.Fatalf("create trigger: %v", err)
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
// AC-002.11: a launch that yields a real, non-empty session id whose
|
||||
// write affects zero rows (no error) must be counted as a session
|
||||
// persist failure, not miscounted as "launch yielded no session id"
|
||||
// (AC-002.8's counter) — the two causes of an identical "processed, no
|
||||
// session" row must stay distinguishable.
|
||||
func TestDispatch_RoutedLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) {
|
||||
repo := newTestRepoSchedSilentlyDroppingSessionWrite(t, "sess-routed-vanished")
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-vanished"}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-session-vanished"}`)
|
||||
|
||||
beforeFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
|
||||
beforeWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
|
||||
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
|
||||
if err != nil || !launched || parked {
|
||||
t.Fatalf("launched=%v parked=%v err=%v, want launched despite the dropped write", launched, parked, err)
|
||||
}
|
||||
|
||||
afterFailed := expvarMapInt(t, "office_loop_session_persist_failed_total", "workspace="+testWorkspaceID)
|
||||
if afterFailed != beforeFailed+1 {
|
||||
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed)
|
||||
}
|
||||
afterWithoutSession := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
if afterWithoutSession != beforeWithoutSession {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package scheduler_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/routing"
|
||||
"github.com/kandev/kandev/internal/office/scheduler"
|
||||
)
|
||||
|
||||
// fakeTaskStarterWithSession wraps fakeTaskStarter and additionally
|
||||
// implements scheduler.TaskStarterWithSession, so dispatch tests can
|
||||
// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7)
|
||||
// without touching the production orchestrator adapter.
|
||||
type fakeTaskStarterWithSession struct {
|
||||
*fakeTaskStarter
|
||||
sessionID string
|
||||
sessionErr error
|
||||
calledRoute scheduler.RouteOverride
|
||||
returnedCall bool
|
||||
}
|
||||
|
||||
func (f *fakeTaskStarterWithSession) StartTaskWithRouteReturningSession(
|
||||
ctx context.Context, taskID, agentID string,
|
||||
launch scheduler.LaunchContext, route scheduler.RouteOverride,
|
||||
) (string, error) {
|
||||
f.returnedCall = true
|
||||
f.calledRoute = route
|
||||
if err := f.StartTaskWithRoute(ctx, taskID, agentID, launch, route); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if f.sessionErr != nil {
|
||||
return "", f.sessionErr
|
||||
}
|
||||
return f.sessionID, nil
|
||||
}
|
||||
|
||||
// expvarMapInt reads one label key out of a package-global expvar.Map,
|
||||
// returning 0 when the key was never incremented. Used to assert
|
||||
// counter deltas without a reset seam — expvar's registry is global and
|
||||
// process-wide, so this reads exactly what production code would.
|
||||
func expvarMapInt(t *testing.T, mapName, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get(mapName)
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map %q not registered", mapName)
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q is not a Map", mapName)
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q[%q] is not an Int", mapName, key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.7: on a routed launch, the session id the
|
||||
// starter returns is persisted onto the run row.
|
||||
func TestDispatch_RoutedLaunchPersistsSessionID(t *testing.T) {
|
||||
repo := newTestRepoSched(t)
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-routed-1"}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-session-1"}`)
|
||||
|
||||
before := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
|
||||
|
||||
launched, parked, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
|
||||
if err != nil || !launched || parked {
|
||||
t.Fatalf("launched=%v parked=%v err=%v, want launched", launched, parked, err)
|
||||
}
|
||||
if !starter.returnedCall {
|
||||
t.Fatal("expected StartTaskWithRouteReturningSession to be called")
|
||||
}
|
||||
|
||||
updated, err := repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if updated.SessionID != "sess-routed-1" {
|
||||
t.Fatalf("session_id = %q, want sess-routed-1", updated.SessionID)
|
||||
}
|
||||
|
||||
after := expvarMapInt(t, "office_loop_launch_total", "workspace="+testWorkspaceID)
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.8: an empty returned session id is left empty and counted
|
||||
// under launch_without_session, never replaced with a placeholder.
|
||||
func TestDispatch_RoutedLaunchEmptySessionCountsWithoutSession(t *testing.T) {
|
||||
repo := newTestRepoSched(t)
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: ""}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-session-2"}`)
|
||||
|
||||
before := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
|
||||
launched, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{})
|
||||
if err != nil || !launched {
|
||||
t.Fatalf("launched=%v err=%v, want launched", launched, err)
|
||||
}
|
||||
|
||||
updated, err := repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if updated.SessionID != "" {
|
||||
t.Fatalf("session_id = %q, want empty (no placeholder)", updated.SessionID)
|
||||
}
|
||||
after := expvarMapInt(t, "office_loop_launch_without_session_total", "workspace="+testWorkspaceID)
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.10: relaunching the same run with a new session id overwrites
|
||||
// the previous one — last non-empty wins within one claim.
|
||||
func TestDispatch_RoutedRelaunchKeepsLatestSessionID(t *testing.T) {
|
||||
repo := newTestRepoSched(t)
|
||||
seedRoutingConfig(t, repo, []routing.ProviderID{"claude-acp"})
|
||||
starter := &fakeTaskStarterWithSession{fakeTaskStarter: newFakeTaskStarter(), sessionID: "sess-first"}
|
||||
ss := buildScheduler(t, repo, starter)
|
||||
run := seedRun(t, repo, `{"task_id":"t-session-3"}`)
|
||||
|
||||
if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil {
|
||||
t.Fatalf("first dispatch: %v", err)
|
||||
}
|
||||
starter.sessionID = "sess-second"
|
||||
// Re-fetch: DispatchWithRouting mutates run.LogicalProviderOrder in
|
||||
// place, so reuse the same struct to simulate a relaunch of the
|
||||
// same claimed row.
|
||||
if _, _, err := ss.DispatchWithRouting(context.Background(), run, makeAgent(), scheduler.LaunchContext{}); err != nil {
|
||||
t.Fatalf("second dispatch: %v", err)
|
||||
}
|
||||
|
||||
updated, err := repo.GetRunByID(context.Background(), run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if updated.SessionID != "sess-second" {
|
||||
t.Fatalf("session_id = %q, want sess-second (last non-empty wins)", updated.SessionID)
|
||||
}
|
||||
}
|
||||
@@ -33,13 +33,15 @@ func (ss *SchedulerService) ClaimNextRun(ctx context.Context) (*models.Run, erro
|
||||
// happened, so it is not bucketed as processed (docs/specs/
|
||||
// task-delivery-ledger/spec.md, "Office run outcome").
|
||||
func (ss *SchedulerService) FinishRun(ctx context.Context, id string) error {
|
||||
return ss.repo.FinishRun(ctx, id, RunStatusFinished, nil)
|
||||
_, err := ss.repo.FinishRun(ctx, id, RunStatusFinished, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// FailRun marks a claimed run as failed. outcome is written as NULL, same
|
||||
// as the service-layer FailRun: a failed run is bucketed on status alone.
|
||||
func (ss *SchedulerService) FailRun(ctx context.Context, id string) error {
|
||||
return ss.repo.FinishRun(ctx, id, RunStatusFailed, nil)
|
||||
_, err := ss.repo.FinishRun(ctx, id, RunStatusFailed, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// ProcessRunGuard checks if the agent is still eligible to be woken.
|
||||
|
||||
@@ -172,7 +172,18 @@ func (si *SchedulerIntegration) finishPolicyBlock(
|
||||
} else {
|
||||
incBudgetBlockedByLimit(provenance)
|
||||
}
|
||||
_ = si.svc.FinishRun(ctx, run.ID, outcome)
|
||||
wrote, err := si.svc.FinishRun(ctx, run.ID, outcome)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to finish policy-blocked run",
|
||||
zap.String("run_id", run.ID), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
if !wrote {
|
||||
// Another writer already moved the run out of claimed between the
|
||||
// admission decision and this write; it did not actually end via a
|
||||
// budget block, so there is nothing to log (Review round 3, R3-1).
|
||||
return false
|
||||
}
|
||||
|
||||
fields := map[string]string{"degraded": strconv.FormatBool(p.Degraded)}
|
||||
if p.IsDefault {
|
||||
@@ -197,11 +208,21 @@ func (si *SchedulerIntegration) cancelBudgetRun(
|
||||
si.releaseCheckoutIfNeeded(ctx, run)
|
||||
si.svc.clearAgentWorking(ctx, agent.ID, run.ID)
|
||||
|
||||
if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil {
|
||||
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err))
|
||||
} else {
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
return false
|
||||
}
|
||||
if !cancelled {
|
||||
// Another writer already moved the run out of claimed between the
|
||||
// admission decision and this write; it did not actually end via
|
||||
// this cancellation, so there is nothing to classify, publish, or
|
||||
// log.
|
||||
return false
|
||||
}
|
||||
|
||||
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
|
||||
fields := map[string]string{activityFieldCeiling: ceilingNotDetermined}
|
||||
for k, v := range extraFields {
|
||||
@@ -383,11 +404,17 @@ func (si *SchedulerIntegration) cancelUnresolvableAgentRun(ctx context.Context,
|
||||
incBudgetCancelledNoWorkspace(provenance)
|
||||
si.cleanupWorkspaceLookupRun(ctx, run)
|
||||
|
||||
if err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace"); err != nil {
|
||||
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, "no_resolvable_workspace")
|
||||
if err != nil {
|
||||
si.logger.Error("failed to cancel run", zap.String("run_id", run.ID), zap.Error(err))
|
||||
} else {
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
return
|
||||
}
|
||||
if !cancelled {
|
||||
return
|
||||
}
|
||||
|
||||
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
si.svc.LogActivityWithRun(ctx, "", "scheduler", "office-scheduler",
|
||||
"run_budget_workspace_unresolvable", "run", run.ID,
|
||||
mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "")
|
||||
@@ -409,11 +436,19 @@ func (si *SchedulerIntegration) deferWorkspaceLookupFailure(ctx context.Context,
|
||||
|
||||
if run.RetryCount >= MaxRetryCount {
|
||||
incBudgetFailedWorkspaceLookup(provenance)
|
||||
if err := si.svc.FailRun(ctx, run.ID); err != nil {
|
||||
wrote, err := si.svc.FailRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to fail run without escalation",
|
||||
zap.String("run_id", run.ID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if !wrote {
|
||||
// Already terminal via another writer (e.g. a concurrent
|
||||
// cancel) between the lookup failure and this write; it did
|
||||
// not actually end via this failure, so there is nothing to
|
||||
// log (Review round 3, R3-1).
|
||||
return
|
||||
}
|
||||
si.svc.LogActivityWithRun(ctx, "", "system", "scheduler",
|
||||
"run_budget_workspace_lookup_failed", "run", run.ID,
|
||||
mustJSON(map[string]string{activityFieldCeiling: ceilingNotDetermined}), run.ID, "")
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// cancelBudgetRun and cancelUnresolvableAgentRun mirror cancelStaleRun's
|
||||
// release/cancel/classify/publish/log sequence, but their CancelRun call
|
||||
// must still be gated the same way: a run that another writer already
|
||||
// moved out of claimed/queued before this cancel runs must not be
|
||||
// classified, broadcast, or logged as cancelled.
|
||||
func TestCancelBudgetRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-budget-cancel-race", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
|
||||
// Another writer wins the terminal-transition race before gate 1's
|
||||
// cancel below runs: the row is already finished, so CancelRun's
|
||||
// guarded UPDATE (status IN ('queued','claimed')) will match nothing.
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish (simulating the winning writer): %v", err)
|
||||
}
|
||||
|
||||
got := make(chan *bus.Event, 1)
|
||||
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
|
||||
got <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
unresolvedAgent := *agent
|
||||
unresolvedAgent.WorkspaceID = ""
|
||||
service.AdmitRunForTest(svc, ctx, run, &unresolvedAgent)
|
||||
|
||||
select {
|
||||
case e := <-got:
|
||||
t.Fatalf("expected no OfficeRunProcessed publish for a lost budget-cancel race; got %+v", e)
|
||||
default:
|
||||
}
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries))
|
||||
}
|
||||
|
||||
survivor, err := svc.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if survivor.Status != service.RunStatusFinished {
|
||||
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// deferWorkspaceLookupFailure's MaxRetryCount-exhausted branch (CodeRabbit,
|
||||
// PR fixup round 2) has the same lost-race obligation as cancelBudgetRun
|
||||
// above: it discarded FailRun's wrote result and logged
|
||||
// run_budget_workspace_lookup_failed unconditionally whenever err was nil,
|
||||
// even when another writer had already made the run terminal.
|
||||
func TestDeferWorkspaceLookupFailure_LostRaceDoesNotLog(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-workspace-lookup-race", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
run.RetryCount = service.MaxRetryCount
|
||||
|
||||
// Another writer wins the terminal-transition race before this call's
|
||||
// FailRun below runs.
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish (simulating the winning writer): %v", err)
|
||||
}
|
||||
|
||||
got := make(chan *bus.Event, 1)
|
||||
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
|
||||
got <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
service.DeferWorkspaceLookupFailureForTest(svc, ctx, run)
|
||||
|
||||
select {
|
||||
case e := <-got:
|
||||
t.Fatalf("expected no OfficeRunProcessed publish for a lost workspace-lookup-failure race; got %+v", e)
|
||||
default:
|
||||
}
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_lookup_failed", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_budget_workspace_lookup_failed activity entries = %d, want 0 for a lost race", len(entries))
|
||||
}
|
||||
|
||||
survivor, err := svc.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if survivor.Status != service.RunStatusFinished {
|
||||
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// cancelUnresolvableAgentRun (reached when the run's agent was deleted)
|
||||
// has the same lost-race obligation as cancelBudgetRun above.
|
||||
func TestCancelUnresolvableAgentRun_LostRaceDoesNotClassifyPublishOrLog(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-unresolvable-cancel-race", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
if err := svc.RepoForTest().DeleteAgentInstance(ctx, agent.ID); err != nil {
|
||||
t.Fatalf("delete agent: %v", err)
|
||||
}
|
||||
|
||||
// Another writer wins the terminal-transition race before the
|
||||
// missing-agent cancel below runs.
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish (simulating the winning writer): %v", err)
|
||||
}
|
||||
|
||||
got := make(chan *bus.Event, 1)
|
||||
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
|
||||
got <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
service.ProcessRunForTest(svc, ctx, run)
|
||||
|
||||
select {
|
||||
case e := <-got:
|
||||
t.Fatalf("expected no OfficeRunProcessed publish for a lost missing-agent cancel race; got %+v", e)
|
||||
default:
|
||||
}
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "", "run_budget_workspace_unresolvable", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_budget_workspace_unresolvable activity entries = %d, want 0 for a lost cancel race", len(entries))
|
||||
}
|
||||
|
||||
survivor, err := svc.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if survivor.Status != service.RunStatusFinished {
|
||||
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
|
||||
}
|
||||
}
|
||||
@@ -180,7 +180,14 @@ func TestAdmitRun_EvaluatorFault_DefersThenFailsWithoutEscalation(t *testing.T)
|
||||
|
||||
// Drive the run to MaxRetryCount and re-process it directly (bypassing
|
||||
// the real backoff delay, exactly as the existing retry tests do).
|
||||
// FailRun's guarded write only takes effect while the row is still
|
||||
// status='claimed' (Review round 3, R3-1), so the deferred row —
|
||||
// left 'queued' by the first tick above — must be reclaimed before
|
||||
// ProcessRunForTest, mirroring scheduler_run_outcome_test.go.
|
||||
svc.ExecSQL(t, `UPDATE runs SET retry_count = ?, status = 'claimed' WHERE id = ?`,
|
||||
service.MaxRetryCount, run.ID)
|
||||
run.RetryCount = service.MaxRetryCount
|
||||
run.Status = "claimed"
|
||||
service.ProcessRunForTest(svc, ctx, run)
|
||||
|
||||
failed, err := svc.GetRun(ctx, run.ID)
|
||||
|
||||
@@ -421,14 +421,6 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Lifecycle: terminal "complete" event for the run detail Events log.
|
||||
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
|
||||
"task_id": data.TaskID,
|
||||
"session_id": data.SessionID,
|
||||
})
|
||||
s.markRoutingSuccess(ctx, run)
|
||||
s.recordRunOutputSummary(ctx, run, *data)
|
||||
s.warnIfReviewDecisionMissing(ctx, run)
|
||||
// resolveLifecycleRun prefers the immutable run ID from the event and
|
||||
// falls back to task plus agent only for legacy events. Releasing here
|
||||
// (rather than unconditionally inside transitionRunTerminal) is safe —
|
||||
@@ -438,10 +430,35 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er
|
||||
// must stay held so ReapStaleCheckouts (not a same-agent race) is what
|
||||
// eventually reclaims it, instead of releasing a lock for a run that
|
||||
// never actually reached a terminal state.
|
||||
if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil {
|
||||
wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed)
|
||||
if err != nil {
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
return err
|
||||
}
|
||||
if !wrote {
|
||||
// The run reached a terminal state through another writer (e.g. a
|
||||
// concurrent cancel) between resolveLifecycleRun's read and this
|
||||
// write — it never actually held the checkout from this call's
|
||||
// point of view, so releasing it here would steal a live lock the
|
||||
// same way an unconditional release would (Review round 3, R3-1).
|
||||
// The agent may still be stuck "working" from the launch though.
|
||||
// The completion side effects below (timeline event, routing
|
||||
// health, output summary, review-decision warning) must not run
|
||||
// either: this run did not actually complete from this call's
|
||||
// point of view, so persisting them would attribute another
|
||||
// writer's outcome (e.g. a cancel) with this one's evidence
|
||||
// (CodeRabbit, PR fixup round 2).
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
return nil
|
||||
}
|
||||
// Lifecycle: terminal "complete" event for the run detail Events log.
|
||||
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
|
||||
"task_id": data.TaskID,
|
||||
"session_id": data.SessionID,
|
||||
})
|
||||
s.markRoutingSuccess(ctx, run)
|
||||
s.recordRunOutputSummary(ctx, run, *data)
|
||||
s.warnIfReviewDecisionMissing(ctx, run)
|
||||
s.releaseTaskCheckoutForRun(ctx, run)
|
||||
s.stampRunFinished(ctx, run)
|
||||
return nil
|
||||
@@ -507,6 +524,12 @@ func (s *Service) markRoutingSuccess(ctx context.Context, run *models.Run) {
|
||||
rd.MarkRunSuccessHealth(ctx, run, agent)
|
||||
}
|
||||
|
||||
// runEventFieldAgentID is the run-event payload key for an agent id.
|
||||
// Named to avoid a duplicate-literal lint failure — "agent_id" also
|
||||
// appears as a JSON struct tag elsewhere in this file, and those two
|
||||
// uses are otherwise unrelated to each other.
|
||||
const runEventFieldAgentID = "agent_id"
|
||||
|
||||
// handleTasklessAgentCompleted attributes a taskless run completion,
|
||||
// finishes the run, and refreshes the per-agent, per-scope continuation
|
||||
// summary so the next fire has bridge context. The
|
||||
@@ -540,12 +563,6 @@ func (s *Service) handleTasklessAgentCompleted(
|
||||
}
|
||||
return err
|
||||
}
|
||||
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
|
||||
"agent_id": data.AgentID,
|
||||
"session_id": data.SessionID,
|
||||
})
|
||||
s.refreshContinuationSummary(ctx, run, run.AgentProfileID)
|
||||
s.recordRunOutputSummary(ctx, run, *data)
|
||||
// run came from GetClaimedTasklessRunForAgent: it is the run that
|
||||
// actually launched. Taskless runs typically carry no task_id, so this
|
||||
// is a no-op in the common case, but call it for the same reason as
|
||||
@@ -553,9 +570,26 @@ func (s *Service) handleTasklessAgentCompleted(
|
||||
//
|
||||
// Finish before releasing, same as handleAgentCompleted: a failed
|
||||
// FinishRun must not still give up the checkout.
|
||||
if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil {
|
||||
wrote, err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !wrote {
|
||||
// Already terminal via another writer — see handleAgentCompleted's
|
||||
// matching branch (Review round 3, R3-1). The completion side
|
||||
// effects below must not run either: this run did not actually
|
||||
// complete from this call's point of view, so recording them
|
||||
// (timeline event, continuation summary, output summary) would
|
||||
// attribute another writer's outcome with this one's evidence
|
||||
// (CodeRabbit, PR fixup round 2).
|
||||
return nil
|
||||
}
|
||||
s.AppendRunEvent(ctx, run.ID, "complete", "info", map[string]interface{}{
|
||||
runEventFieldAgentID: data.AgentID,
|
||||
"session_id": data.SessionID,
|
||||
})
|
||||
s.refreshContinuationSummary(ctx, run, run.AgentProfileID)
|
||||
s.recordRunOutputSummary(ctx, run, *data)
|
||||
s.releaseTaskCheckoutForRun(ctx, run)
|
||||
s.stampRunFinished(ctx, run)
|
||||
return nil
|
||||
@@ -708,9 +742,16 @@ func (s *Service) handleAgentFailed(ctx context.Context, event *bus.Event) error
|
||||
// rate-limit-retry callers; we deliberately do NOT call into it
|
||||
// here. See docs/specs/office/requirements/runtime.md.
|
||||
errMsg := enrichModelFailureMessage(run, data.ErrorMessage)
|
||||
if err := s.HandleAgentFailure(ctx, run, errMsg); err != nil {
|
||||
wrote, err := s.HandleAgentFailure(ctx, run, errMsg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !wrote {
|
||||
// Already terminal via another writer — waking the CEO agent
|
||||
// over a run that never actually failed would be a false alarm
|
||||
// (Review round 3, R3-1).
|
||||
return nil
|
||||
}
|
||||
s.dispatchAgentErrorTrigger(ctx, run, data.TaskID, data.SessionID, errMsg)
|
||||
return nil
|
||||
}
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects is the
|
||||
// PR fixup round 2 regression test (CodeRabbit). handleAgentCompleted used
|
||||
// to record the "complete" run event and the output summary BEFORE checking
|
||||
// FinishRun's result at all — so a run whose terminal write never actually
|
||||
// landed (a genuine persistence failure here, or a lost race to a
|
||||
// concurrent writer in production) still got its evidence recorded as if
|
||||
// this call had won. Both outcomes reach the same guard now: the
|
||||
// completion side effects only run once FinishRun confirms wrote=true.
|
||||
func TestHandleAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
ensureTaskSessionMessagesTable(t, svc)
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "worker-race")
|
||||
taskID := createOfficeTask(t, svc, "ws-1", "worker-race")
|
||||
|
||||
if err := svc.QueueRun(
|
||||
ctx, "worker-race", service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, "run-output-race",
|
||||
); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim run: %v (run=%v)", err, run)
|
||||
}
|
||||
claimedAt := requireClaimedAt(t, run)
|
||||
|
||||
svc.ExecSQL(t, `
|
||||
INSERT INTO task_session_messages (id, task_session_id, type, author_type, content, created_at)
|
||||
VALUES ('m-race-1', 'sess-race', 'message', 'agent', 'should never be recorded', ?)
|
||||
`, claimedAt.Add(5*time.Second))
|
||||
|
||||
// Force FinishRun's UPDATE to fail with a genuine error, matching
|
||||
// TestSchedulerTick_AgentCompletedKeepsCheckoutWhenFinishRunFails'
|
||||
// fault injection: a targeted fault (drop the column FinishRun sets)
|
||||
// rather than a global read-only pragma.
|
||||
svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at")
|
||||
|
||||
completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{
|
||||
"task_id": taskID,
|
||||
"session_id": "sess-race",
|
||||
"agent_profile_id": "worker-race",
|
||||
})
|
||||
if pErr := eb.Publish(ctx, events.AgentCompleted, completed); pErr != nil {
|
||||
t.Fatalf("publish completed: %v", pErr)
|
||||
}
|
||||
|
||||
got := findRunByID(t, svc, "ws-1", run.ID)
|
||||
if got.OutputSummary != "" {
|
||||
t.Errorf("output_summary = %q, want empty — FinishRun never landed, so recordRunOutputSummary "+
|
||||
"must not have run", got.OutputSummary)
|
||||
}
|
||||
if got.Status != service.RunStatusClaimed {
|
||||
t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status)
|
||||
}
|
||||
|
||||
runEvents, err := svc.ListRunEventsForTest(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list run events: %v", err)
|
||||
}
|
||||
for _, e := range runEvents {
|
||||
if e.EventType == "complete" {
|
||||
t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects
|
||||
// is the taskless-path counterpart: refreshContinuationSummary and
|
||||
// recordRunOutputSummary must not persist evidence for a completion whose
|
||||
// terminal write never landed either.
|
||||
func TestHandleTasklessAgentCompleted_FinishRunFailureSkipsCompletionSideEffects(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
svc.SetSyncHandlers(true)
|
||||
ctx := context.Background()
|
||||
eb := bus.NewMemoryEventBus(logger.Default())
|
||||
if err := svc.RegisterEventSubscribers(eb); err != nil {
|
||||
t.Fatalf("register subscribers: %v", err)
|
||||
}
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "coordinator-taskless-race-effects",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "coordinator-taskless-race-effects",
|
||||
Role: models.AgentRoleCEO,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `
|
||||
INSERT INTO runs (
|
||||
id, agent_profile_id, reason, payload, status, coalesced_count,
|
||||
context_snapshot, requested_at, claimed_at
|
||||
) VALUES (
|
||||
'run-taskless-race-1', ?, 'routine_trigger', '{}',
|
||||
'claimed', 1, '{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
|
||||
)
|
||||
`, agent.ID)
|
||||
|
||||
svc.ExecSQL(t, "ALTER TABLE runs DROP COLUMN finished_at")
|
||||
|
||||
completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{
|
||||
"agent_id": agent.ID,
|
||||
"agent_profile_id": agent.ID,
|
||||
"session_id": "sess-taskless-race",
|
||||
})
|
||||
if err := eb.Publish(ctx, events.AgentCompleted, completed); err != nil {
|
||||
t.Fatalf("publish completed: %v", err)
|
||||
}
|
||||
|
||||
got := findRunByID(t, svc, "ws-1", "run-taskless-race-1")
|
||||
if got.Status != service.RunStatusClaimed {
|
||||
t.Errorf("status = %q, want claimed (FinishRun's write failed)", got.Status)
|
||||
}
|
||||
|
||||
runEvents, err := svc.ListRunEventsForTest(ctx, "run-taskless-race-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list run events: %v", err)
|
||||
}
|
||||
for _, e := range runEvents {
|
||||
if e.EventType == "complete" {
|
||||
t.Fatalf("expected no 'complete' run event when FinishRun's write failed; got %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
summary, err := svc.GetContinuationSummaryForTest(ctx, agent.ID, "")
|
||||
if err == nil && summary != nil {
|
||||
t.Fatalf("expected no continuation summary written when FinishRun's write failed; got %+v", summary)
|
||||
}
|
||||
}
|
||||
@@ -200,7 +200,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) {
|
||||
taskID := uuidish("task-fail-runner", i)
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "runner-pm4")
|
||||
w := queueAndReadRun(t, svc, "runner-pm4", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure (runner) %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
@@ -208,7 +208,7 @@ func TestAutoPostAgentComment_RecordsSuccessForActingAgent(t *testing.T) {
|
||||
taskID := uuidish("task-fail-critic", i)
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "critic4")
|
||||
w := queueAndReadRun(t, svc, "critic4", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure (critic) %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,18 +33,36 @@ const (
|
||||
//
|
||||
// No retry is scheduled — the user resolves via Resume session in the
|
||||
// chat or Mark fixed in the inbox.
|
||||
//
|
||||
// Returns wrote=false when MarkRunFailed's guarded write was a no-op —
|
||||
// the run reached a terminal state through another writer (e.g. a
|
||||
// concurrent cancel) between the caller's read and this call — so
|
||||
// callers know not to treat a cancelled/already-terminal run as a
|
||||
// genuine agent failure (Review round 3, R3-1).
|
||||
func (s *Service) HandleAgentFailure(
|
||||
ctx context.Context,
|
||||
run *models.Run,
|
||||
errorMessage string,
|
||||
) error {
|
||||
if err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage); err != nil {
|
||||
return fmt.Errorf("mark run failed: %w", err)
|
||||
) (bool, error) {
|
||||
wrote, err := s.repo.MarkRunFailed(ctx, run.ID, errorMessage)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("mark run failed: %w", err)
|
||||
}
|
||||
if !wrote {
|
||||
// Nothing to classify, release, or escalate: the row's real
|
||||
// terminal state was written by someone else. Still make sure
|
||||
// the agent isn't left stuck "working" from the launch.
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
return false, nil
|
||||
}
|
||||
// MarkRunFailed bypasses transitionRunTerminal (this is the office v1
|
||||
// failure path, not FailRun), so the checkout release that lives there
|
||||
// has to be duplicated here — otherwise every agent-error terminal
|
||||
// transition leaks the task checkout the same way FinishRun used to.
|
||||
// failure path, not FailRun), so the checkout release and terminal-shape
|
||||
// count that live there have to be duplicated here — otherwise every
|
||||
// agent-error terminal transition leaks the task checkout the same way
|
||||
// FinishRun used to, and every genuine post-launch crash goes uncounted
|
||||
// in office_loop_terminal_total. MarkRunFailed writes status='failed'
|
||||
// with outcome left untouched (NULL for a launched run), matching FailRun.
|
||||
s.recordTerminalShape(ctx, run, RunStatusFailed, nil)
|
||||
s.releaseTaskCheckoutForRun(ctx, run)
|
||||
// Leave "working" before the auto-pause decision below, not after: the
|
||||
// reset is a working → idle CAS, so running it first lets a subsequent
|
||||
@@ -78,7 +96,7 @@ func (s *Service) HandleAgentFailure(
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// RecordAgentSuccess resets the consecutive-failure counter for the
|
||||
|
||||
@@ -5,15 +5,20 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// queueAndReadRun enqueues a run with a unique idempotency key
|
||||
// (so multiple calls in one test don't coalesce) and reads it back.
|
||||
// Mirrors the production path (QueueRun) without going through the
|
||||
// claim step (which lives on the repository).
|
||||
// (so multiple calls in one test don't coalesce), reads it back, and
|
||||
// marks it claimed — the precondition every production caller of
|
||||
// HandleAgentFailure actually has (resolveLifecycleRun resolves via
|
||||
// GetClaimedRunByID before ever reaching it). Review round 3, R3-1
|
||||
// guards MarkRunFailed's write to status = 'claimed', so a test that
|
||||
// skipped this step would see every HandleAgentFailure call in this
|
||||
// file silently no-op.
|
||||
func queueAndReadRun(
|
||||
t *testing.T, svc *service.Service, agentID, taskID string,
|
||||
) *models.Run {
|
||||
@@ -30,6 +35,9 @@ func queueAndReadRun(
|
||||
}
|
||||
for _, w := range rows {
|
||||
if w.AgentProfileID == agentID && taskIDFromPayload(t, w.Payload) == taskID && w.Status != service.RunStatusFailed {
|
||||
svc.ExecSQL(t, `UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
|
||||
time.Now().UTC(), w.ID)
|
||||
w.Status = service.RunStatusClaimed
|
||||
return w
|
||||
}
|
||||
}
|
||||
@@ -94,7 +102,7 @@ func TestHandleAgentFailure_AutoPausesAtThreshold(t *testing.T) {
|
||||
taskID := uuidish("task-pause", i)
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-pause")
|
||||
w := queueAndReadRun(t, svc, "agent-pause", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +136,7 @@ func TestRecordAgentSuccess_ResetsCounter(t *testing.T) {
|
||||
taskID := uuidish("task-succ", i)
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-success")
|
||||
w := queueAndReadRun(t, svc, "agent-success", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
@@ -172,7 +180,7 @@ func TestHandleAgentFailure_RespectsPerAgentThreshold(t *testing.T) {
|
||||
taskID := "task-tight-1"
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-tight")
|
||||
w := queueAndReadRun(t, svc, "agent-tight", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
|
||||
@@ -198,7 +206,7 @@ func autoPauseAgent(
|
||||
taskID := agentID + "-task-" + uuidish("p", i)
|
||||
insertSyntheticTask(t, svc, taskID, wsID, agentID)
|
||||
w := queueAndReadRun(t, svc, agentID, taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
@@ -328,7 +336,7 @@ func TestMarkAgentRunFailedFixed_LeavesInboxWhenRequeueFails(t *testing.T) {
|
||||
taskID := "task-retry-after-queue-error"
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-task-retry")
|
||||
run := queueAndReadRun(t, svc, "agent-task-retry", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
if err := svc.UpdateAgentStatusFields(ctx, "agent-task-retry",
|
||||
@@ -358,7 +366,7 @@ func TestMarkAgentPausedFixed_UsesPauseSnapshot(t *testing.T) {
|
||||
oldTaskID := "old-failure-task"
|
||||
insertSyntheticTask(t, svc, oldTaskID, "ws-1", "agent-snapshot")
|
||||
oldRun := queueAndReadRun(t, svc, "agent-snapshot", oldTaskID)
|
||||
if err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, oldRun, "old failure"); err != nil {
|
||||
t.Fatalf("old failure: %v", err)
|
||||
}
|
||||
svc.RecordAgentSuccess(ctx, "agent-snapshot")
|
||||
@@ -392,7 +400,7 @@ func TestMarkAgentPausedFixed_DiscardsRecoveryForReassignedTask(t *testing.T) {
|
||||
reassignedTaskID := "reassigned-task"
|
||||
insertSyntheticTask(t, svc, reassignedTaskID, "ws-1", "agent-reassigned-from")
|
||||
failedRun := queueAndReadRun(t, svc, "agent-reassigned-from", reassignedTaskID)
|
||||
if err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, failedRun, "boom"); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
// Two more failures (on other tasks) to cross the default threshold
|
||||
@@ -520,6 +528,35 @@ func TestMarkAgentPausedFixed_NonPausedAgentGuardAndRequeueError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// HandleAgentFailure is Office's v1 post-launch failure path
|
||||
// (event_subscribers.go's handleAgentFailed) and bypasses
|
||||
// transitionRunTerminal's own counting, so it must record its own
|
||||
// terminal shape or office_loop_terminal_total silently misses every
|
||||
// genuine agent crash (Review round 1, R1-1).
|
||||
func TestHandleAgentFailure_RecordsTerminalShape(t *testing.T) {
|
||||
svc, _ := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
createTestAgent(t, svc, "ws-1", "agent-crash")
|
||||
taskID := "task-crash"
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "agent-crash")
|
||||
run := queueAndReadRun(t, svc, "agent-crash", taskID)
|
||||
svc.ExecSQL(t, `UPDATE runs SET session_id = ? WHERE id = ?`, "session-crash", run.ID)
|
||||
run.SessionID = "session-crash"
|
||||
|
||||
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeLaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if _, err := svc.HandleAgentFailure(ctx, run, "boom"); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("launched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// Pins that reassigning a task auto-dismisses the per-task inbox
|
||||
// entry for the OLD agent without resetting that agent's counter.
|
||||
func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing.T) {
|
||||
@@ -532,7 +569,7 @@ func TestOnAssigneeChanged_DismissesPriorEntryWithoutResettingCounter(t *testing
|
||||
taskID := "task-reassign-1"
|
||||
insertSyntheticTask(t, svc, taskID, "ws-1", "old-agent")
|
||||
w := queueAndReadRun(t, svc, "old-agent", taskID)
|
||||
if err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
if _, err := svc.HandleAgentFailure(ctx, w, "boom"); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged is the PR fixup
|
||||
// round 2 regression test (CodeRabbit). The idle-skip FinishRun call site
|
||||
// used to discard FinishRun's error (`wrote, _ := ...`), treating a genuine
|
||||
// persistence failure exactly like a benign lost terminal-write race
|
||||
// (wrote=false, err=nil) — silently, with no log line either way. A run
|
||||
// stuck 'claimed' because the terminal write itself failed needs to be
|
||||
// distinguishable in logs from one that lost a race to another writer, so
|
||||
// operators (and ReapStaleCheckouts/RecoverStale, the actual recovery path)
|
||||
// aren't flying blind on which case occurred.
|
||||
func TestSchedulerOutcome_IdleSkipped_FinishRunErrorIsLogged(t *testing.T) {
|
||||
core, logs := observer.New(zapcore.ErrorLevel)
|
||||
log, err := logger.NewFromZap(zap.New(core))
|
||||
if err != nil {
|
||||
t.Fatalf("create observer logger: %v", err)
|
||||
}
|
||||
svc := newTestService(t, service.ServiceOptions{Logger: log})
|
||||
ctx := context.Background()
|
||||
|
||||
// Reject the UPDATE FinishRun issues (status -> 'finished') so the
|
||||
// idle-skip path's FinishRun call returns a genuine persistence error
|
||||
// instead of a lost-race no-op.
|
||||
svc.ExecSQL(t, `
|
||||
CREATE TRIGGER reject_idle_skip_finish
|
||||
BEFORE UPDATE OF status ON runs
|
||||
WHEN NEW.status = 'finished'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'finish run write rejected for test');
|
||||
END;
|
||||
`)
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "idle-outcome-worker-err",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
if logs.FilterMessage("failed to finish idle-skipped run").Len() == 0 {
|
||||
t.Fatal("expected the FinishRun persistence error to be logged, not silently swallowed")
|
||||
}
|
||||
|
||||
run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonHeartbeat)
|
||||
if run.Status != "claimed" {
|
||||
t.Fatalf("status = %q, want claimed — the write was rejected, so the run must stay "+
|
||||
"claimed for the stale-claim reaper to recover, not silently read as finished", run.Status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Loop-liveness counters (REQ-OFFICE-LOOP-LIVENESS-003). Same
|
||||
// "k1=v1;k2=v2" expvar.NewMap label convention as
|
||||
// internal/office/scheduler/metrics_vars.go and
|
||||
// internal/orchestrator/office_stall_metrics.go, so a Prometheus
|
||||
// translation layer can split on the same delimiters everywhere.
|
||||
//
|
||||
// AC-003.9's two increment rules: the six counters naming a persisted
|
||||
// change (trigger_claimed, routine_run, wakeup_created, run_claimed,
|
||||
// launch, terminal) increment at the persisting site after the write
|
||||
// succeeds. The five naming an event that persists nothing (cron_tick,
|
||||
// launch_without_session, session_persist_failed,
|
||||
// last_run_at_write_failed, liveness_degraded) increment at the
|
||||
// observing site regardless of any write outcome.
|
||||
var (
|
||||
loopCronTickTotal = expvar.NewMap("office_loop_cron_tick_total")
|
||||
loopTriggerClaimedTotal = expvar.NewMap("office_loop_trigger_claimed_total")
|
||||
loopRoutineRunTotal = expvar.NewMap("office_loop_routine_run_total")
|
||||
loopWakeupCreatedTotal = expvar.NewMap("office_loop_wakeup_created_total")
|
||||
loopRunClaimedTotal = expvar.NewMap("office_loop_run_claimed_total")
|
||||
loopLaunchTotal = expvar.NewMap("office_loop_launch_total")
|
||||
loopLaunchWithoutSessionTotal = expvar.NewMap("office_loop_launch_without_session_total")
|
||||
loopSessionPersistFailedTotal = expvar.NewMap("office_loop_session_persist_failed_total")
|
||||
loopTerminalTotal = expvar.NewMap("office_loop_terminal_total")
|
||||
loopLastRunAtWriteFailedTotal = expvar.NewMap("office_loop_last_run_at_write_failed_total")
|
||||
loopLivenessDegradedTotal = expvar.NewMap("office_loop_liveness_degraded_total")
|
||||
|
||||
// Instants, not counters — see LoopMetricLabel's callers below.
|
||||
// Both are monotonic (never decrease), unlike the drifting gauges
|
||||
// metrics_vars.go warns against: without them a flat counter is
|
||||
// ambiguous between a restarted process and a stopped loop
|
||||
// (AC-003.6).
|
||||
loopCronTickAt = expvar.NewString("office_loop_cron_tick_at")
|
||||
loopProcessStartedAt = expvar.NewString("office_loop_process_started_at")
|
||||
)
|
||||
|
||||
// LoopUnattributedWorkspace is the explicit label used when an event's
|
||||
// workspace cannot be resolved (AC-003.8) — never dropped, never
|
||||
// guessed, and never merged into a real workspace's totals.
|
||||
const LoopUnattributedWorkspace = "_unattributed"
|
||||
|
||||
// LoopMetricLabel builds a "k1=v1;k2=v2;..." expvar map key. pairs must
|
||||
// be an even-length (key, value) sequence.
|
||||
func LoopMetricLabel(pairs ...string) string {
|
||||
if len(pairs)%2 != 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
parts = append(parts, pairs[i]+"="+pairs[i+1])
|
||||
}
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
|
||||
// IncLoopCronTick counts one cron loop tick evaluation and publishes
|
||||
// the tick instant (AC-003.1, AC-003.6).
|
||||
func IncLoopCronTick(atRFC3339 string) {
|
||||
loopCronTickTotal.Add("", 1)
|
||||
loopCronTickAt.Set(atRFC3339)
|
||||
}
|
||||
|
||||
// RecordLoopProcessStarted publishes the process-start instant once,
|
||||
// at boot (AC-003.6).
|
||||
func RecordLoopProcessStarted(atRFC3339 string) {
|
||||
loopProcessStartedAt.Set(atRFC3339)
|
||||
}
|
||||
|
||||
// IncLoopTriggerClaimed counts one eligible trigger claimed for firing.
|
||||
func IncLoopTriggerClaimed(workspaceID string) {
|
||||
loopTriggerClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopRoutineRun counts one routine run created, by (source,
|
||||
// disposition) so a coalesced, a skipped, and a launched fire are
|
||||
// separately countable (AC-003.2).
|
||||
func IncLoopRoutineRun(workspaceID, source, disposition string) {
|
||||
loopRoutineRunTotal.Add(
|
||||
LoopMetricLabel("workspace", workspaceID, "source", source, "disposition", disposition), 1)
|
||||
}
|
||||
|
||||
// IncLoopWakeupCreated counts one wakeup request created.
|
||||
func IncLoopWakeupCreated(workspaceID, source string) {
|
||||
loopWakeupCreatedTotal.Add(LoopMetricLabel("workspace", workspaceID, "source", source), 1)
|
||||
}
|
||||
|
||||
// IncLoopRunClaimed counts one run claimed off the queue.
|
||||
func IncLoopRunClaimed(workspaceID string) {
|
||||
loopRunClaimedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopLaunch counts one agent launch attempt that reached the
|
||||
// adapter successfully.
|
||||
func IncLoopLaunch(workspaceID string) {
|
||||
loopLaunchTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopLaunchWithoutSession counts a launch that reported success but
|
||||
// yielded no session id (AC-002.8).
|
||||
func IncLoopLaunchWithoutSession(workspaceID string) {
|
||||
loopLaunchWithoutSessionTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopSessionPersistFailed counts a launch that yielded a session id
|
||||
// which then failed to persist (AC-002.11) — distinct from
|
||||
// IncLoopLaunchWithoutSession because the underlying rows are otherwise
|
||||
// byte-identical.
|
||||
func IncLoopSessionPersistFailed(workspaceID string) {
|
||||
loopSessionPersistFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopTerminal counts one terminal run transition by its classified
|
||||
// shape. Counts transitions, not entities: a run that reaches terminal
|
||||
// twice (a requeue after a post-start fallback) contributes two events.
|
||||
func IncLoopTerminal(workspaceID, shape string) {
|
||||
loopTerminalTotal.Add(LoopMetricLabel("workspace", workspaceID, "shape", shape), 1)
|
||||
}
|
||||
|
||||
// IncLoopLastRunAtWriteFailed counts a TouchRoutineLastRun write that
|
||||
// returned an error (AC-001.7). The dispatch still completes — a
|
||||
// routine that fired must not read as having failed to fire because a
|
||||
// bookkeeping write lost a lock.
|
||||
func IncLoopLastRunAtWriteFailed(workspaceID string) {
|
||||
loopLastRunAtWriteFailedTotal.Add(LoopMetricLabel("workspace", workspaceID), 1)
|
||||
}
|
||||
|
||||
// IncLoopLivenessDegraded counts one /loop-health read that failed
|
||||
// loudly, labelled by which input could not be read (AC-004.10,
|
||||
// "Failure" in the design).
|
||||
func IncLoopLivenessDegraded(workspaceID, reason string) {
|
||||
loopLivenessDegradedTotal.Add(LoopMetricLabel("workspace", workspaceID, "reason", reason), 1)
|
||||
}
|
||||
@@ -95,15 +95,23 @@ func isRetryStale(run *models.Run) (bool, string) {
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// cancelRetry cancels a run that is too stale to retry.
|
||||
// cancelRetry cancels a run that is too stale to retry. Terminal-shape
|
||||
// recording and the OfficeRunProcessed broadcast only happen when the
|
||||
// cancel actually applied: a run another writer already finished or
|
||||
// failed must not be reported to subscribers as cancelled.
|
||||
func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason string) error {
|
||||
s.logger.Info("cancelling stale run retry",
|
||||
zap.String("run_id", run.ID),
|
||||
zap.String("reason", reason))
|
||||
if err := s.repo.CancelRun(ctx, run.ID, reason); err != nil {
|
||||
cancelled, err := s.repo.CancelRun(ctx, run.ID, reason)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
if !cancelled {
|
||||
return nil
|
||||
}
|
||||
s.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
|
||||
s.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
return nil
|
||||
}
|
||||
@@ -113,10 +121,17 @@ func (s *Service) cancelRetry(ctx context.Context, run *models.Run, reason strin
|
||||
func (s *Service) escalateFailure(
|
||||
ctx context.Context, run *models.Run, runErr error,
|
||||
) error {
|
||||
if err := s.FailRun(ctx, run.ID); err != nil {
|
||||
wrote, err := s.FailRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail run: %w", err)
|
||||
}
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
if !wrote {
|
||||
// Already terminal via another writer (e.g. a concurrent cancel)
|
||||
// between the retry decision and this write — nothing left to
|
||||
// escalate (Review round 3, R3-1).
|
||||
return nil
|
||||
}
|
||||
|
||||
agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID)
|
||||
if err != nil {
|
||||
@@ -158,10 +173,17 @@ func (s *Service) failRunNoEscalation(
|
||||
ctx context.Context, run *models.Run, agent *models.AgentInstance,
|
||||
cause budgetDeferralCause, policyID string,
|
||||
) error {
|
||||
if err := s.FailRun(ctx, run.ID); err != nil {
|
||||
wrote, err := s.FailRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail run: %w", err)
|
||||
}
|
||||
s.clearAgentWorking(ctx, run.AgentProfileID, run.ID)
|
||||
if !wrote {
|
||||
// Already terminal via another writer (e.g. a concurrent cancel)
|
||||
// between the budget decision and this write — nothing left to
|
||||
// report (Review round 3, R3-1).
|
||||
return nil
|
||||
}
|
||||
|
||||
fields := map[string]string{activityFieldCeiling: ceilingNotDetermined}
|
||||
if policyID != "" {
|
||||
|
||||
@@ -25,5 +25,10 @@ func (s *Service) CancelPendingRetriesForTask(ctx context.Context, taskID string
|
||||
zap.String("task_id", taskID),
|
||||
zap.Int("count", len(ids)))
|
||||
|
||||
return s.repo.BulkCancelRuns(ctx, ids, "task_reassigned")
|
||||
cancelled, err := s.repo.BulkCancelRuns(ctx, ids, "task_reassigned")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// A stale-retry cancel that loses the terminal-transition race (another
|
||||
// writer already finished the run) must not tell subscribers the run was
|
||||
// cancelled: the run's real terminal state is finished, and a status
|
||||
// mismatch between the persisted row and the broadcast event misleads
|
||||
// every WS-driven UI reading it.
|
||||
func TestHandleRunFailure_LostCancelRaceDoesNotPublishFalseCancelled(t *testing.T) {
|
||||
svc, eb := newTestServiceWithBus(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-stale-retry-race", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry-race"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
|
||||
staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour)
|
||||
svc.ExecSQL(t,
|
||||
`UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`,
|
||||
staleRequestedAt.Add(-time.Hour).Format(time.RFC3339))
|
||||
svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID)
|
||||
run.RequestedAt = staleRequestedAt
|
||||
|
||||
// Another writer wins the terminal-transition race before the stale
|
||||
// cancel below runs: the row is already finished, so CancelRun's
|
||||
// guarded UPDATE (status IN ('queued','claimed')) will match it.
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish (simulating the winning writer): %v", err)
|
||||
}
|
||||
|
||||
got := make(chan *bus.Event, 1)
|
||||
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
|
||||
got <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
netErr := errors.New("connection timed out")
|
||||
if err := svc.HandleRunFailure(ctx, run, netErr); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case e := <-got:
|
||||
t.Fatalf("expected no OfficeRunProcessed publish for a lost cancel race; got %+v", e)
|
||||
default:
|
||||
}
|
||||
|
||||
reqs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
if len(reqs) != 1 || reqs[0].Status != service.RunStatusFinished {
|
||||
t.Fatalf("run status = %+v, want the winning writer's finished state to survive", reqs)
|
||||
}
|
||||
}
|
||||
|
||||
// The stale-claim cancel path (evaluateRunStaleness's workflow_step_changed
|
||||
// branch, reached from the scheduler tick) has the same obligation as the
|
||||
// stale-retry path above: a lost cancel race must not publish a false
|
||||
// "cancelled" event or log a false cancel activity entry.
|
||||
func TestSchedulerIntegration_CancelStaleRunLostRaceDoesNotPublishOrLog(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
eb := bus.NewMemoryEventBus(logger.Default())
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock, EventBus: eb})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-moved-step-race", models.AgentRoleWorker)
|
||||
agent.ExecutorPreference = `{"type":"local_pc"}`
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks
|
||||
(id, workspace_id, workflow_step_id, title, created_at, updated_at)
|
||||
VALUES ('task-moved-step-race', 'ws-1', 'step-current', 'Moved task',
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-moved-step-race","workflow_step_id":"step-old"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil || len(runs) != 1 {
|
||||
t.Fatalf("list runs: %v (%d)", err, len(runs))
|
||||
}
|
||||
runID := runs[0].ID
|
||||
|
||||
// Simulate another writer winning the race for this exact row: the
|
||||
// tick's own CancelRun write below will match zero rows and return
|
||||
// no error, the same shape a genuinely lost race produces.
|
||||
svc.ExecSQL(t, fmt.Sprintf(`
|
||||
CREATE TRIGGER ignore_stale_cancel_write
|
||||
BEFORE UPDATE OF status ON runs
|
||||
WHEN NEW.status = 'cancelled' AND OLD.id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(IGNORE);
|
||||
END;
|
||||
`, runID))
|
||||
|
||||
got := make(chan *bus.Event, 1)
|
||||
sub, err := eb.Subscribe(events.OfficeRunProcessed, func(_ context.Context, e *bus.Event) error {
|
||||
got <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
select {
|
||||
case e := <-got:
|
||||
t.Fatalf("expected no OfficeRunProcessed publish for a lost stale-cancel race; got %+v", e)
|
||||
default:
|
||||
}
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_cancelled_stale", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_cancelled_stale activity entries = %d, want 0 for a lost cancel race", len(entries))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// TestCancelPendingRetriesForTask_RecordsTerminalShape is the Testing
|
||||
// round 3 regression test. CancelPendingRetriesForTask calls
|
||||
// repo.BulkCancelRuns directly, which bypassed office_loop_terminal_total
|
||||
// entirely — the same bypass class Review round 1 (R1-1) fixed for
|
||||
// HandleAgentFailure and the single-run CancelRun call sites, just missed
|
||||
// here. A pending retry never launched, so cancelling it must classify
|
||||
// as unlaunched_failed.
|
||||
func TestCancelPendingRetriesForTask_RecordsTerminalShape(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("retry-cancel-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
taskID := "task-retry-cancel"
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES (?, ?, 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
|
||||
taskID, agent.WorkspaceID)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"`+taskID+`"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
runs, err := svc.ListRuns(ctx, agent.WorkspaceID)
|
||||
if err != nil || len(runs) != 1 {
|
||||
t.Fatalf("list runs: runs=%v err=%v", runs, err)
|
||||
}
|
||||
run := runs[0]
|
||||
if err := svc.RepoForTest().ScheduleRetry(ctx, run.ID, run.RequestedAt.Add(time.Hour), 1); err != nil {
|
||||
t.Fatalf("schedule retry: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if err := svc.CancelPendingRetriesForTask(ctx, taskID); err != nil {
|
||||
t.Fatalf("cancel pending retries: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// cancelRetry (the 24h-stale-retry path reached through HandleRunFailure)
|
||||
// bypasses transitionRunTerminal's own counting the same way
|
||||
// HandleAgentFailure and cancelStaleRun do, so it must record its own
|
||||
// terminal shape or office_loop_terminal_total silently misses every
|
||||
// abandoned-retry cancellation (Review round 1, R1-1). This is a distinct
|
||||
// trigger condition from cancelStaleRun's workflow_step_changed path,
|
||||
// covered separately in TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape.
|
||||
func TestHandleRunFailure_CancelsStaleRetryAndRecordsTerminalShape(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-stale-retry", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k-stale-retry"); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
|
||||
// Backdate past retryMaxAge (24h) so isRetryStale routes this failure
|
||||
// to cancelRetry instead of a normal backoff reschedule. Also push the
|
||||
// loop-liveness activation instant back further still, so the
|
||||
// backdated run doesn't fall outside the activation window and
|
||||
// classify as pre_activation instead of the shape under test.
|
||||
staleRequestedAt := time.Now().UTC().Add(-25 * time.Hour)
|
||||
svc.ExecSQL(t,
|
||||
`UPDATE kandev_meta SET value = ? WHERE key = 'telemetry.office_loop_liveness.activated_at'`,
|
||||
staleRequestedAt.Add(-time.Hour).Format(time.RFC3339))
|
||||
svc.ExecSQL(t, `UPDATE runs SET requested_at = ? WHERE id = ?`, staleRequestedAt, run.ID)
|
||||
run.RequestedAt = staleRequestedAt
|
||||
|
||||
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
netErr := errors.New("connection timed out")
|
||||
if err := svc.HandleRunFailure(ctx, run, netErr); err != nil {
|
||||
t.Fatalf("handle failure: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
|
||||
reqs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
if len(reqs) != 1 {
|
||||
t.Fatalf("expected 1 run, got %d", len(reqs))
|
||||
}
|
||||
if reqs[0].Status != service.RunStatusCancelled {
|
||||
t.Errorf("status = %q, want cancelled", reqs[0].Status)
|
||||
}
|
||||
if reqs[0].CancelReason == nil || *reqs[0].CancelReason != "run_too_old" {
|
||||
t.Errorf("cancel_reason = %v, want run_too_old", reqs[0].CancelReason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
func runClaimedExpvarInt(t *testing.T, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_loop_run_claimed_total")
|
||||
if v == nil {
|
||||
t.Fatal("expvar map office_loop_run_claimed_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatal("office_loop_run_claimed_total is not a Map")
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("office_loop_run_claimed_total[%q] is not an Int", key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// REQ-OFFICE-LOOP-LIVENESS-003: ClaimNextRun counts a successfully
|
||||
// claimed run labelled by the claiming agent's workspace.
|
||||
func TestClaimNextRun_IncrementsRunClaimedCounter(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("run-claimed-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID)
|
||||
before := runClaimedExpvarInt(t, key)
|
||||
|
||||
req, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || req == nil {
|
||||
t.Fatalf("claim: req=%v err=%v", req, err)
|
||||
}
|
||||
|
||||
after := runClaimedExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("run_claimed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// ClaimNextRun with no queued work never increments the counter.
|
||||
func TestClaimNextRun_EmptyQueueDoesNotIncrementRunClaimedCounter(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
key := service.LoopMetricLabel("workspace", "ws-1")
|
||||
before := runClaimedExpvarInt(t, key)
|
||||
|
||||
req, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
if req != nil {
|
||||
t.Fatalf("expected nil run, got %+v", req)
|
||||
}
|
||||
|
||||
after := runClaimedExpvarInt(t, key)
|
||||
if after != before {
|
||||
t.Fatalf("run_claimed delta = %d, want 0", after-before)
|
||||
}
|
||||
}
|
||||
@@ -314,7 +314,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) {
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
@@ -375,7 +375,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
@@ -427,7 +427,7 @@ func TestFailRun_PublishesOfficeRunProcessedFailed(t *testing.T) {
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
if err := svc.FailRun(ctx, run.ID); err != nil {
|
||||
if _, err := svc.FailRun(ctx, run.ID); err != nil {
|
||||
t.Fatalf("fail: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -229,7 +229,10 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
|
||||
si.logger.Info("run skipped (agent not active)",
|
||||
zap.String("run_id", runID),
|
||||
zap.String("agent_status", string(agent.Status)))
|
||||
_ = si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive)
|
||||
if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive); err != nil {
|
||||
si.logger.Error("failed to finish agent-inactive run",
|
||||
zap.String("run_id", runID), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -251,6 +254,19 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
|
||||
si.logger.Info("run skipped (no actionable tasks)",
|
||||
zap.String("run_id", runID),
|
||||
zap.String("agent", agent.Name))
|
||||
si.svc.clearAgentWorking(ctx, agent.ID, runID)
|
||||
wrote, err := si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to finish idle-skipped run",
|
||||
zap.String("run_id", runID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if !wrote {
|
||||
// Another writer already moved the run out of claimed; it did
|
||||
// not actually end via an idle skip, so there is nothing to
|
||||
// log.
|
||||
return
|
||||
}
|
||||
si.svc.LogActivityWithRun(ctx, agent.WorkspaceID,
|
||||
"scheduler", "office-scheduler",
|
||||
"run_idle_skipped", "run", runID,
|
||||
@@ -258,8 +274,6 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run)
|
||||
"agent": agent.Name,
|
||||
"agent_id": agent.ID,
|
||||
}), runID, "")
|
||||
si.svc.clearAgentWorking(ctx, agent.ID, runID)
|
||||
_ = si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -547,7 +561,10 @@ func (si *SchedulerIntegration) isTaskTreeGated(ctx context.Context, runID, task
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("hold_id", hold.ID),
|
||||
zap.String("mode", hold.Mode))
|
||||
_ = si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld)
|
||||
if _, err := si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld); err != nil {
|
||||
si.logger.Error("failed to finish tree-held run",
|
||||
zap.String("run_id", runID), zap.String("task_id", taskID), zap.Error(err))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -606,12 +623,19 @@ func (si *SchedulerIntegration) launchAgent(
|
||||
return launched
|
||||
}
|
||||
var err error
|
||||
if starter, ok := si.svc.taskStarter.(TaskStarterWithLaunchContext); ok {
|
||||
var sessionID string
|
||||
switch starter := si.svc.taskStarter.(type) {
|
||||
case TaskStarterWithLaunchContextSession:
|
||||
sessionID, err = starter.StartTaskWithLaunchContextReturningSession(ctx, taskID, launch.ProfileID, launch)
|
||||
case TaskStarterWithLaunchContext:
|
||||
err = starter.StartTaskWithLaunchContext(ctx, taskID, launch.ProfileID, launch)
|
||||
} else if starter, ok := si.svc.taskStarter.(TaskStarterWithEnv); ok {
|
||||
case TaskStarterWithSession:
|
||||
sessionID, err = starter.StartTaskWithEnvReturningSession(ctx, taskID, launch.ProfileID, "", "", "",
|
||||
launch.Prompt, "", false, nil, launch.Env)
|
||||
case TaskStarterWithEnv:
|
||||
err = starter.StartTaskWithEnv(ctx, taskID, launch.ProfileID, "", "", "",
|
||||
launch.Prompt, "", false, nil, launch.Env)
|
||||
} else {
|
||||
default:
|
||||
err = si.svc.taskStarter.StartTask(ctx, taskID, launch.ProfileID, "", "", "",
|
||||
launch.Prompt, "", false, nil)
|
||||
}
|
||||
@@ -626,9 +650,37 @@ func (si *SchedulerIntegration) launchAgent(
|
||||
_ = si.svc.HandleRunFailure(ctx, run, err)
|
||||
return false
|
||||
}
|
||||
IncLoopLaunch(agent.WorkspaceID)
|
||||
si.persistLaunchedSession(ctx, runID, agent.WorkspaceID, sessionID)
|
||||
return true
|
||||
}
|
||||
|
||||
// persistLaunchedSession stores the session id a successful direct
|
||||
// launch produced (AC-OFFICE-LOOP-LIVENESS-002.7) — the scheduler-side
|
||||
// counterpart to scheduler.SchedulerService.persistLaunchedSession for
|
||||
// the routed launch path. See that method's doc comment for the
|
||||
// without-session / persist-failed counter semantics (AC-002.8, .11).
|
||||
func (si *SchedulerIntegration) persistLaunchedSession(
|
||||
ctx context.Context, runID, workspaceID, sessionID string,
|
||||
) {
|
||||
if sessionID == "" {
|
||||
IncLoopLaunchWithoutSession(workspaceID)
|
||||
return
|
||||
}
|
||||
wrote, err := si.svc.repo.SetRunSessionID(ctx, runID, sessionID)
|
||||
if err != nil {
|
||||
si.logger.Warn("persist launched session id failed",
|
||||
zap.String("run_id", runID), zap.String("session_id", sessionID), zap.Error(err))
|
||||
IncLoopSessionPersistFailed(workspaceID)
|
||||
return
|
||||
}
|
||||
if !wrote {
|
||||
si.logger.Warn("persist launched session id matched no row",
|
||||
zap.String("run_id", runID), zap.String("session_id", sessionID))
|
||||
IncLoopSessionPersistFailed(workspaceID)
|
||||
}
|
||||
}
|
||||
|
||||
// failTasklessRun terminally fails a run that launchAgent determined has
|
||||
// no task_id. This is a scheduler capability gap, not an agent failure, so
|
||||
// it must NOT go through HandleAgentFailure's consecutive-failure/
|
||||
@@ -665,16 +717,25 @@ func (si *SchedulerIntegration) launchAgent(
|
||||
func (si *SchedulerIntegration) failTasklessRun(
|
||||
ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string,
|
||||
) {
|
||||
si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{
|
||||
"phase": "scheduler.launch",
|
||||
"error_message": msg,
|
||||
})
|
||||
si.releaseCheckoutIfNeeded(ctx, run)
|
||||
if err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg); err != nil {
|
||||
wrote, err := si.svc.repo.MarkRunFailed(ctx, run.ID, msg)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to mark taskless run as failed",
|
||||
zap.String("run_id", run.ID), zap.Error(err))
|
||||
return // don't publish a terminal event when persistence failed
|
||||
}
|
||||
if !wrote {
|
||||
// Already terminal via another writer (e.g. a concurrent cancel)
|
||||
// between the caller's read and this write — nothing to classify,
|
||||
// publish, or log as a scheduler-launch error on this run's
|
||||
// timeline (Review round 3, R3-1).
|
||||
return
|
||||
}
|
||||
si.svc.AppendRunEvent(ctx, run.ID, "error", "error", map[string]interface{}{
|
||||
"phase": "scheduler.launch",
|
||||
"error_message": msg,
|
||||
})
|
||||
si.svc.recordTerminalShape(ctx, run, RunStatusFailed, nil)
|
||||
run.ErrorMessage = msg
|
||||
|
||||
// Settle the auto-dismiss decision before publishing: the frontend
|
||||
@@ -728,11 +789,17 @@ func (si *SchedulerIntegration) failUnlaunchableRun(
|
||||
"error_message": msg,
|
||||
})
|
||||
si.releaseCheckoutIfNeeded(ctx, run)
|
||||
if err := si.svc.HandleAgentFailure(ctx, run, msg); err != nil {
|
||||
wrote, err := si.svc.HandleAgentFailure(ctx, run, msg)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to handle agent failure for unlaunchable run",
|
||||
zap.String("run_id", run.ID), zap.Error(err))
|
||||
return // don't publish a terminal event when persistence failed
|
||||
}
|
||||
if !wrote {
|
||||
// Already terminal via another writer — nothing to publish
|
||||
// (Review round 3, R3-1).
|
||||
return
|
||||
}
|
||||
run.ErrorMessage = msg
|
||||
si.svc.publishRunProcessedForWorkspace(ctx, run.ID, RunStatusFailed, run, agent.WorkspaceID)
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) {
|
||||
}
|
||||
|
||||
// Finish the run.
|
||||
if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
@@ -94,6 +94,40 @@ func TestSchedulerIntegration_CancelsRunForMovedWorkflowStep(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// cancelStaleRun bypasses transitionRunTerminal's own counting the same
|
||||
// way HandleAgentFailure does, so it must record its own terminal shape
|
||||
// or office_loop_terminal_total silently misses every stale-claim
|
||||
// cancellation (Review round 1, R1-1).
|
||||
func TestSchedulerIntegration_CancelStaleRunRecordsTerminalShape(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("worker-moved-step-shape", models.AgentRoleWorker)
|
||||
agent.ExecutorPreference = `{"type":"local_pc"}`
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks
|
||||
(id, workspace_id, workflow_step_id, title, created_at, updated_at)
|
||||
VALUES ('task-moved-step-shape', 'ws-1', 'step-current', 'Moved task',
|
||||
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-moved-step-shape","workflow_step_id":"step-old"}`, ""); err != nil {
|
||||
t.Fatalf("queue run: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerIntegration_ResolvesExecutorFromTaskProject(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
@@ -236,7 +270,7 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) {
|
||||
}
|
||||
|
||||
// Finish the first run.
|
||||
if err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil {
|
||||
if _, err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
@@ -314,7 +348,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) {
|
||||
t.Errorf("prompt should contain %q, got: %s", tt.contains, prompt)
|
||||
}
|
||||
|
||||
_ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed)
|
||||
_, _ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// ignoreFinishedWrite installs a trigger that makes the given run's
|
||||
// terminal-transition write to status='finished' a no-op, simulating
|
||||
// another writer having already claimed the terminal-transition race.
|
||||
func ignoreFinishedWrite(t *testing.T, svc *service.Service, runID string) {
|
||||
t.Helper()
|
||||
svc.ExecSQL(t, fmt.Sprintf(`
|
||||
CREATE TRIGGER ignore_finished_write
|
||||
BEFORE UPDATE OF status ON runs
|
||||
WHEN NEW.status = 'finished' AND OLD.id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(IGNORE);
|
||||
END;
|
||||
`, runID))
|
||||
}
|
||||
|
||||
// A heartbeat run that loses the idle-skip terminal-transition race (another
|
||||
// writer already moved it out of 'claimed') must not log a false
|
||||
// run_idle_skipped activity entry: the run's real terminal state came from
|
||||
// whichever writer actually won, and a stray activity entry misreports why
|
||||
// it ended.
|
||||
func TestSchedulerIntegration_IdleSkipLostRaceDoesNotLogFalseActivity(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "idle-worker-race",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
// Worker defaults to skip_idle_runs=true, no tasks assigned.
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil || len(runs) != 1 {
|
||||
t.Fatalf("list runs: %v (%d)", err, len(runs))
|
||||
}
|
||||
ignoreFinishedWrite(t, svc, runs[0].ID)
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
if mock.callCount() != 0 {
|
||||
t.Errorf("expected 0 StartTask calls, got %d", mock.callCount())
|
||||
}
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_idle_skipped", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_idle_skipped activity entries = %d, want 0 for a lost terminal-transition race", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// A run that loses the budget-blocked terminal-transition race has the same
|
||||
// obligation: no false run_budget_blocked activity entry.
|
||||
func TestSchedulerIntegration_BudgetBlockedLostRaceDoesNotLogFalseActivity(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "budget-worker-race",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
policy := &models.BudgetPolicy{
|
||||
WorkspaceID: "ws-1",
|
||||
ScopeType: "agent",
|
||||
ScopeID: agent.ID,
|
||||
LimitSubcents: 100,
|
||||
Period: "monthly",
|
||||
AlertThresholdPct: 80,
|
||||
ActionOnExceed: "pause_agent",
|
||||
}
|
||||
if err := svc.CreateBudgetPolicy(ctx, policy); err != nil {
|
||||
t.Fatalf("create policy: %v", err)
|
||||
}
|
||||
insertTestTask(t, svc, "task-budget-race", "ws-1")
|
||||
insertTestCostEvent(t, svc, agent.ID, "task-budget-race", int64(600))
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"task-budget-race"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil || len(runs) != 1 {
|
||||
t.Fatalf("list runs: %v (%d)", err, len(runs))
|
||||
}
|
||||
ignoreFinishedWrite(t, svc, runs[0].ID)
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
entries, err := svc.RepoForTest().ListActivityEntriesByAction(ctx, "ws-1", "run_budget_blocked", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list activity: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("run_budget_blocked activity entries = %d, want 0 for a lost terminal-transition race", len(entries))
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/runs/commentkeys"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
// ClaimNextRun atomically claims the next eligible run from the queue.
|
||||
@@ -28,6 +29,11 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) {
|
||||
zap.String("id", req.ID),
|
||||
zap.String("agent", req.AgentProfileID),
|
||||
zap.String("reason", req.Reason))
|
||||
workspaceID := LoopUnattributedWorkspace
|
||||
if agent, err := s.GetAgentFromConfig(ctx, req.AgentProfileID); err == nil && agent != nil {
|
||||
workspaceID = agent.WorkspaceID
|
||||
}
|
||||
IncLoopRunClaimed(workspaceID)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
@@ -37,7 +43,12 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) {
|
||||
// an OfficeRunProcessed bus event. The run row is fetched first so the
|
||||
// published payload carries enough context (agent, task, comment, reason)
|
||||
// for downstream WS consumers to scope updates.
|
||||
func (s *Service) FinishRun(ctx context.Context, id, outcome string) error {
|
||||
//
|
||||
// Returns wrote=false when the guarded write was a no-op — the run was
|
||||
// no longer claimed (already terminal via a concurrent writer, e.g. a
|
||||
// cancel) by the time this statement ran — so there is nothing to
|
||||
// classify or publish (Review round 3, R3-1).
|
||||
func (s *Service) FinishRun(ctx context.Context, id, outcome string) (bool, error) {
|
||||
return s.transitionRunTerminal(ctx, id, RunStatusFinished, &outcome)
|
||||
}
|
||||
|
||||
@@ -45,17 +56,26 @@ func (s *Service) FinishRun(ctx context.Context, id, outcome string) error {
|
||||
// OfficeRunProcessed event. outcome is written as NULL: a failed run is
|
||||
// bucketed by RunCountsByDayForAgent on status alone and never reaches the
|
||||
// outcome-derived buckets, so no value from the five-value vocabulary is
|
||||
// invented for it. See FinishRun for the lifecycle contract.
|
||||
func (s *Service) FailRun(ctx context.Context, id string) error {
|
||||
// invented for it. See FinishRun for the lifecycle contract and the
|
||||
// wrote=false no-op case.
|
||||
func (s *Service) FailRun(ctx context.Context, id string) (bool, error) {
|
||||
return s.transitionRunTerminal(ctx, id, RunStatusFailed, nil)
|
||||
}
|
||||
|
||||
// transitionRunTerminal updates the run row to the given terminal status
|
||||
// and outcome (nil writes SQL NULL) and emits OfficeRunProcessed.
|
||||
// Pre-fetching the run keeps the published payload self-contained even
|
||||
// when the caller doesn't hold a reference to the model. Publish errors
|
||||
// are logged at debug and swallowed; persistence errors are returned to
|
||||
// the caller.
|
||||
// FinishRun returns the row as it stands right after that write (via
|
||||
// RETURNING), so the published payload and the terminal-shape
|
||||
// classification both read the same statement that persisted the change —
|
||||
// neither depends on a separate read succeeding independently of it.
|
||||
// Publish errors are logged at debug and swallowed; persistence errors are
|
||||
// returned to the caller.
|
||||
//
|
||||
// FinishRun's guarded write (status = 'claimed') returns a nil run when
|
||||
// another writer already moved the row to a different terminal state
|
||||
// between the caller's read and this statement — nothing to classify or
|
||||
// publish in that case, so this returns wrote=false without touching
|
||||
// recordTerminalShape or publishRunProcessed (Review round 3, R3-1).
|
||||
//
|
||||
// Deliberately does NOT release the task checkout: transitionRunTerminal
|
||||
// is reached by every terminal run, including ones that never held the
|
||||
@@ -67,24 +87,68 @@ func (s *Service) FailRun(ctx context.Context, id string) error {
|
||||
// for the SAME agent + task races a first run that is genuinely still
|
||||
// executing and holds the checkout — that second run's release matches
|
||||
// the first run's own checkout_agent_id and steals its own live lock out
|
||||
// from under it (Review round 3, BLOCKING FINDING 1). Callers that KNOW
|
||||
// their run actually held the checkout call releaseTaskCheckoutForRun
|
||||
// explicitly instead: handleAgentCompleted / handleTasklessAgentCompleted
|
||||
// (event_subscribers.go) for the launched-run completion path, and
|
||||
// HandleAgentFailure (failure.go) for the launched-run failure path.
|
||||
func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) error {
|
||||
run, getErr := s.repo.GetRunByID(ctx, id)
|
||||
if getErr != nil && !errors.Is(getErr, sql.ErrNoRows) {
|
||||
s.logger.Debug("get run for terminal transition failed",
|
||||
zap.String("run_id", id),
|
||||
zap.String("status", status),
|
||||
zap.Error(getErr))
|
||||
// from under it. Callers that KNOW their run actually held the checkout
|
||||
// call releaseTaskCheckoutForRun explicitly instead, and only when this
|
||||
// method reports wrote=true: handleAgentCompleted /
|
||||
// handleTasklessAgentCompleted (event_subscribers.go) for the
|
||||
// launched-run completion path, and HandleAgentFailure (failure.go) for
|
||||
// the launched-run failure path.
|
||||
func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) (bool, error) {
|
||||
run, err := s.repo.FinishRun(ctx, id, status, outcome)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.repo.FinishRun(ctx, id, status, outcome); err != nil {
|
||||
return err
|
||||
if run == nil {
|
||||
return false, nil
|
||||
}
|
||||
s.recordTerminalShape(ctx, run, status, outcome)
|
||||
s.publishRunProcessed(ctx, id, status, run)
|
||||
return nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// recordTerminalShape classifies the just-persisted terminal transition
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-005) and increments office_loop_terminal_total
|
||||
// by the resulting shape. run is nil when transitionRunTerminal's
|
||||
// pre-fetch failed; there is nothing to classify in that case. The
|
||||
// workspace label falls back to LoopUnattributedWorkspace when the
|
||||
// owning agent can't be resolved, matching AC-003.8 — never dropped,
|
||||
// never guessed into a real workspace's totals.
|
||||
func (s *Service) recordTerminalShape(ctx context.Context, run *models.Run, status string, outcome *string) {
|
||||
if run == nil {
|
||||
return
|
||||
}
|
||||
activationInstant, activationPublished := s.repo.LoopLivenessActivation()
|
||||
shape := ClassifyTerminalRun(status, outcome, run.SessionID, run.RequestedAt, activationInstant, activationPublished)
|
||||
workspaceID := LoopUnattributedWorkspace
|
||||
if agent, err := s.GetAgentFromConfig(ctx, run.AgentProfileID); err == nil && agent != nil {
|
||||
workspaceID = agent.WorkspaceID
|
||||
}
|
||||
IncLoopTerminal(workspaceID, string(shape))
|
||||
}
|
||||
|
||||
// recordTerminalShapesForCancelledRuns classifies and counts one
|
||||
// cancelled-transition shape per row a bulk cancel (CancelRunsForTasks,
|
||||
// BulkCancelRuns) actually persisted — a bulk write covers rows whose
|
||||
// session_id can differ per row (a claimed run may have already launched),
|
||||
// so each row is classified individually rather than assuming one shape
|
||||
// for the whole batch.
|
||||
func (s *Service) recordTerminalShapesForCancelledRuns(ctx context.Context, cancelled []runssqlite.CancelledRun) {
|
||||
for _, row := range cancelled {
|
||||
s.recordTerminalShape(ctx, &models.Run{
|
||||
AgentProfileID: row.AgentProfileID,
|
||||
SessionID: row.SessionID,
|
||||
RequestedAt: row.RequestedAt,
|
||||
}, RunStatusCancelled, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// RecordCancelledRunTerminalShapes is recordTerminalShapesForCancelledRuns,
|
||||
// exported for the dashboard package's TerminalShapeRecorder seam — a
|
||||
// dashboard-driven cancellation (displaced participant) reaches the same
|
||||
// counter every other cancellation path reaches, without a second,
|
||||
// divergence-prone classification implementation in dashboard.
|
||||
func (s *Service) RecordCancelledRunTerminalShapes(ctx context.Context, cancelled []runssqlite.CancelledRun) {
|
||||
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
|
||||
}
|
||||
|
||||
// releaseTaskCheckoutForRun releases the run's task checkout. Call this
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestFinishAndFailRun(t *testing.T) {
|
||||
t.Fatal("expected claimed run")
|
||||
}
|
||||
|
||||
if err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil {
|
||||
if _, err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) {
|
||||
t.Fatal("expected claimed run")
|
||||
}
|
||||
|
||||
if err := svc.FailRun(ctx, req.ID); err != nil {
|
||||
if _, err := svc.FailRun(ctx, req.ID); err != nil {
|
||||
t.Fatalf("fail: %v", err)
|
||||
}
|
||||
|
||||
@@ -157,14 +157,19 @@ func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome is Review
|
||||
// round 2's lower-priority TEST-002 (spec:2130-2132): two terminal callers
|
||||
// reaching the same run row must never leave status and outcome
|
||||
// disagreeing, because transitionRunTerminal's underlying FinishRun writes
|
||||
// both columns in a single UPDATE statement. This drives both orderings —
|
||||
// FinishRun-then-FailRun and FailRun-then-FinishRun — and asserts the row
|
||||
// always ends up wholly consistent with whichever call went last.
|
||||
func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) {
|
||||
// TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome supersedes
|
||||
// Review round 2's lower-priority TEST-002 (spec:2130-2132) under Review
|
||||
// round 3's R3-1 guard: transitionRunTerminal's underlying FinishRun now
|
||||
// only writes while the row is still status = 'claimed' (the same guard
|
||||
// that stops a stale write from clobbering a concurrent cancel), so two
|
||||
// terminal callers reaching the same row no longer race to "whichever
|
||||
// wrote last" — the first write reaches a real terminal state and every
|
||||
// later one is a no-op that leaves status and outcome exactly as the
|
||||
// first write left them. This drives both orderings — FinishRun-then-
|
||||
// FailRun and FailRun-then-FinishRun — and asserts the row stays
|
||||
// consistent with whichever call went FIRST, and that the second call
|
||||
// reports wrote=false.
|
||||
func TestTransitionRunTerminal_FirstWriterWinsOnStatusAndOutcome(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
queueAndClaim := func(t *testing.T, svc *service.Service) string {
|
||||
@@ -186,36 +191,46 @@ func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) {
|
||||
t.Run("FailRun after FinishRun", func(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
id := queueAndClaim(t, svc)
|
||||
if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
wrote, err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed)
|
||||
if err != nil || !wrote {
|
||||
t.Fatalf("finish: wrote=%v err=%v, want wrote=true", wrote, err)
|
||||
}
|
||||
if err := svc.FailRun(ctx, id); err != nil {
|
||||
t.Fatalf("fail: %v", err)
|
||||
}
|
||||
got, err := svc.GetRun(ctx, id)
|
||||
wrote, err = svc.FailRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if got.Status != models.RunStatusFailed || got.Outcome != nil {
|
||||
t.Fatalf("row = {status: %q, outcome: %v}, want the later FailRun write: {failed, nil}", got.Status, got.Outcome)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("FinishRun after FailRun", func(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
id := queueAndClaim(t, svc)
|
||||
if err := svc.FailRun(ctx, id); err != nil {
|
||||
t.Fatalf("fail: %v", err)
|
||||
}
|
||||
if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
if wrote {
|
||||
t.Error("fail: wrote = true, want false (run was already finished, not claimed)")
|
||||
}
|
||||
got, err := svc.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if got.Status != models.RunStatusFinished || got.Outcome == nil || *got.Outcome != service.RunOutcomeProcessed {
|
||||
t.Fatalf("row = {status: %q, outcome: %v}, want the later FinishRun write: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed)
|
||||
t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("FinishRun after FailRun", func(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
id := queueAndClaim(t, svc)
|
||||
wrote, err := svc.FailRun(ctx, id)
|
||||
if err != nil || !wrote {
|
||||
t.Fatalf("fail: wrote=%v err=%v, want wrote=true", wrote, err)
|
||||
}
|
||||
wrote, err = svc.FinishRun(ctx, id, service.RunOutcomeProcessed)
|
||||
if err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Error("finish: wrote = true, want false (run was already failed, not claimed)")
|
||||
}
|
||||
got, err := svc.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if got.Status != models.RunStatusFailed || got.Outcome != nil {
|
||||
t.Fatalf("row = {status: %q, outcome: %v}, want the FIRST write preserved: {failed, nil}", got.Status, got.Outcome)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -44,7 +44,11 @@ func (si *SchedulerIntegration) evaluateRunStaleness(
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
// cancelStaleRun marks the run cancelled, logs the event, and releases any checkout.
|
||||
// cancelStaleRun marks the run cancelled, logs the event, and releases
|
||||
// any checkout. Terminal-shape recording, the OfficeRunProcessed
|
||||
// broadcast, and the activity log entry only happen when the cancel
|
||||
// actually applied: a run another writer already finished or failed
|
||||
// must not be reported as cancelled.
|
||||
func (si *SchedulerIntegration) cancelStaleRun(
|
||||
ctx context.Context,
|
||||
run *models.Run,
|
||||
@@ -60,12 +64,18 @@ func (si *SchedulerIntegration) cancelStaleRun(
|
||||
si.releaseCheckoutIfNeeded(ctx, run)
|
||||
si.svc.clearAgentWorking(ctx, agent.ID, run.ID)
|
||||
|
||||
if err := si.svc.repo.CancelRun(ctx, run.ID, reason); err != nil {
|
||||
cancelled, err := si.svc.repo.CancelRun(ctx, run.ID, reason)
|
||||
if err != nil {
|
||||
si.logger.Error("failed to cancel stale run",
|
||||
zap.String("run_id", run.ID), zap.Error(err))
|
||||
} else {
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
return
|
||||
}
|
||||
if !cancelled {
|
||||
return
|
||||
}
|
||||
|
||||
si.svc.recordTerminalShape(ctx, run, RunStatusCancelled, nil)
|
||||
si.svc.publishRunProcessed(ctx, run.ID, RunStatusCancelled, run)
|
||||
|
||||
si.svc.LogActivityWithRun(ctx, agent.WorkspaceID,
|
||||
"scheduler", "office-scheduler",
|
||||
|
||||
@@ -74,6 +74,103 @@ func TestSchedulerTick_TasklessRunFailsInsteadOfFinishing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSchedulerTick_TasklessRunRecordsTerminalShape is the Testing round 3
|
||||
// regression test. failTasklessRun calls repo.MarkRunFailed directly
|
||||
// (not through HandleAgentFailure), so it bypassed office_loop_terminal_total
|
||||
// entirely — the same bypass class Review round 1 (R1-1) fixed for the other
|
||||
// three production terminal-writers, just missed here. A taskless run never
|
||||
// launches, so it must classify as unlaunched_failed.
|
||||
func TestSchedulerTick_TasklessRunRecordsTerminalShape(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "coordinator-terminal-shape",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "coordinator-terminal-shape",
|
||||
Role: models.AgentRoleCEO,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", "ws-1", "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent is the PR
|
||||
// fixup round 2 regression test (CodeRabbit). failTasklessRun used to
|
||||
// call AppendRunEvent before checking MarkRunFailed's wrote result, so a
|
||||
// run a concurrent writer (e.g. a cancel) already made terminal still
|
||||
// picked up a "scheduler.launch" error event on its timeline — a spurious
|
||||
// entry on a run whose actual outcome was decided by the other writer.
|
||||
// AppendRunEvent must only fire once wrote=true confirms this call won
|
||||
// the terminal-write race.
|
||||
func TestFailTasklessRun_LostRaceDoesNotAppendSpuriousErrorEvent(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "coordinator-taskless-race",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "coordinator-taskless-race",
|
||||
Role: models.AgentRoleCEO,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonRoutineTrigger, `{}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: %v %v", run, err)
|
||||
}
|
||||
|
||||
// Another writer wins the terminal-transition race before
|
||||
// failTasklessRun below runs: the row is already finished, so
|
||||
// MarkRunFailed's guarded UPDATE (status = 'claimed') matches nothing.
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish (simulating the winning writer): %v", err)
|
||||
}
|
||||
|
||||
service.FailTasklessRunForTest(svc, ctx, run, agent, "scheduler cannot launch a taskless run")
|
||||
|
||||
events, err := svc.ListRunEventsForTest(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("list run events: %v", err)
|
||||
}
|
||||
for _, e := range events {
|
||||
if e.EventType == "error" {
|
||||
t.Fatalf("expected no error event appended for a lost taskless-fail race; got %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
survivor, err := svc.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if survivor.Status != service.RunStatusFinished {
|
||||
t.Fatalf("status = %q, want the winning writer's finished state to survive", survivor.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSchedulerTick_TaskBoundRunStillLaunches is the regression guard
|
||||
// alongside the taskless-failure fix above: an ordinary task-bound run with
|
||||
// a wired task starter must still launch normally and stay `claimed` (not
|
||||
|
||||
@@ -53,6 +53,30 @@ type TaskStarterWithLaunchContext interface {
|
||||
StartTaskWithLaunchContext(ctx context.Context, taskID string, agentProfileID string, launch LaunchContext) error
|
||||
}
|
||||
|
||||
// TaskStarterWithSession optionally returns the id of the agent session
|
||||
// a direct (non-routed) launch created, so the caller can persist it on
|
||||
// the run row (AC-OFFICE-LOOP-LIVENESS-002.7). A starter that does not
|
||||
// implement this leaves the run's session id empty, counted as a
|
||||
// without-session launch.
|
||||
type TaskStarterWithSession interface {
|
||||
StartTaskWithEnvReturningSession(ctx context.Context, taskID string, agentProfileID string, executorID string,
|
||||
executorProfileID string, priority string, prompt string, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment, env map[string]string) (sessionID string, err error)
|
||||
}
|
||||
|
||||
// TaskStarterWithLaunchContextSession combines TaskStarterWithLaunchContext
|
||||
// and TaskStarterWithSession: a starter satisfying this carries the full
|
||||
// launch context (skills included) AND returns the launched session id in
|
||||
// the same call, so neither capability has to be dropped for the other.
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.7 requires the session id unconditionally, on
|
||||
// every direct launch, regardless of whether that launch also carries
|
||||
// per-run skill additions — the production adapter must satisfy this
|
||||
// rather than TaskStarterWithLaunchContext alone.
|
||||
type TaskStarterWithLaunchContextSession interface {
|
||||
StartTaskWithLaunchContextReturningSession(ctx context.Context, taskID string, agentProfileID string,
|
||||
launch LaunchContext) (sessionID string, err error)
|
||||
}
|
||||
|
||||
// LaunchContext mirrors scheduler.LaunchContext so the office.service
|
||||
// package can carry the Office-built launch context (prompt, env,
|
||||
// workflow step, attachments, plan-mode, profile) into the routing
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
// mockTaskStarterWithSession wraps mockTaskStarter and additionally
|
||||
// implements service.TaskStarterWithSession, so direct-launch tests can
|
||||
// drive the session-returning seam (AC-OFFICE-LOOP-LIVENESS-002.7)
|
||||
// without touching the production orchestrator adapter.
|
||||
type mockTaskStarterWithSession struct {
|
||||
*mockTaskStarter
|
||||
sessionID string
|
||||
}
|
||||
|
||||
func (m *mockTaskStarterWithSession) StartTaskWithEnvReturningSession(
|
||||
ctx context.Context, taskID, agentProfileID, executorID,
|
||||
executorProfileID string, priority string, prompt, workflowStepID string,
|
||||
planMode bool, attachments []v1.MessageAttachment, env map[string]string,
|
||||
) (string, error) {
|
||||
if err := m.StartTaskWithEnv(ctx, taskID, agentProfileID, executorID,
|
||||
executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return m.sessionID, nil
|
||||
}
|
||||
|
||||
func directHopExpvarInt(t *testing.T, mapName, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get(mapName)
|
||||
if v == nil {
|
||||
t.Fatalf("expvar map %q not registered", mapName)
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q is not a Map", mapName)
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("expvar %q[%q] is not an Int", mapName, key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.7: on a direct (non-routed) launch, the
|
||||
// session id the starter returns is persisted onto the run row.
|
||||
func TestSchedulerTick_DirectLaunchPersistsSessionID(t *testing.T) {
|
||||
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-1"}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "direct-session-agent",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "direct-session-worker",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-direct-session-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-session-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
before := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1")
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
if mock.callCount() != 1 {
|
||||
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
|
||||
}
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
if len(runs) != 1 || runs[0].SessionID != "sess-direct-1" {
|
||||
t.Fatalf("session_id = %q, want sess-direct-1", runs[0].SessionID)
|
||||
}
|
||||
after := directHopExpvarInt(t, "office_loop_launch_total", "workspace=ws-1")
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_loop_launch_total delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.8: a starter that only implements TaskStarterWithEnv (no
|
||||
// session-returning seam) leaves the run's session id empty and counts
|
||||
// under launch_without_session — the existing production fallback.
|
||||
func TestSchedulerTick_DirectLaunchWithoutSessionSeamCountsWithoutSession(t *testing.T) {
|
||||
mock := &mockTaskStarter{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "direct-nosession-agent",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "direct-nosession-worker",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-direct-nosession-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-nosession-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
before := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
if len(runs) != 1 || runs[0].SessionID != "" {
|
||||
t.Fatalf("session_id = %q, want empty", runs[0].SessionID)
|
||||
}
|
||||
after := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
if after != before+1 {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
// rejectSessionIDWrites installs a trigger that aborts only an UPDATE
|
||||
// setting runs.session_id to wantSessionID — the exact value the
|
||||
// launch under test will produce — so it targets persistLaunchedSession's
|
||||
// post-launch SetRunSessionID write without also catching the
|
||||
// pre-launch runtime-context build, which writes session_id too (see
|
||||
// runtime/context_builder.go) but with the run's pre-launch id.
|
||||
func rejectSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) {
|
||||
t.Helper()
|
||||
svc.ExecSQL(t, fmt.Sprintf(`
|
||||
CREATE TRIGGER reject_session_id_write
|
||||
BEFORE UPDATE OF session_id ON runs
|
||||
WHEN NEW.session_id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'session_id write rejected for test');
|
||||
END;
|
||||
`, wantSessionID))
|
||||
}
|
||||
|
||||
// AC-002.11: on a direct (non-routed) launch, a session persist failure
|
||||
// is counted separately from a without-session launch, and does not
|
||||
// fail the launch itself or overwrite the run's session_id — the agent
|
||||
// is already running by the time this write happens.
|
||||
func TestSchedulerTick_DirectLaunchSessionPersistFailureCountsSeparately(t *testing.T) {
|
||||
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-persist-fail"}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "direct-persist-fail-agent",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "direct-persist-fail-worker",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-direct-persist-fail-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-persist-fail-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
rejectSessionIDWrites(t, svc, "sess-direct-persist-fail")
|
||||
|
||||
beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
|
||||
beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
if mock.callCount() != 1 {
|
||||
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
|
||||
}
|
||||
runs, err := svc.ListRuns(ctx, "ws-1")
|
||||
if err != nil {
|
||||
t.Fatalf("list runs: %v", err)
|
||||
}
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("run count = %d, want 1", len(runs))
|
||||
}
|
||||
if runs[0].SessionID != "" {
|
||||
t.Fatalf("session_id = %q, want empty after a rejected write", runs[0].SessionID)
|
||||
}
|
||||
// The launch itself must still count as claimed/in-flight, not failed —
|
||||
// the agent is already running despite the persist failure.
|
||||
if runs[0].Status != service.RunStatusClaimed {
|
||||
t.Fatalf("run status = %q, want claimed (launch succeeded despite persist failure)", runs[0].Status)
|
||||
}
|
||||
|
||||
afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
|
||||
if afterFailed != beforeFailed+1 {
|
||||
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1", afterFailed-beforeFailed)
|
||||
}
|
||||
afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
if afterWithoutSession != beforeWithoutSession {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0", afterWithoutSession-beforeWithoutSession)
|
||||
}
|
||||
}
|
||||
|
||||
// ignoreSessionIDWrites installs a trigger that uses RAISE(IGNORE)
|
||||
// rather than RAISE(ABORT): the UPDATE setting runs.session_id to
|
||||
// wantSessionID matches zero rows and returns no error, the shape a run
|
||||
// row that vanished between claim and launch would produce — a
|
||||
// different failure mode than rejectSessionIDWrites' hard DB error.
|
||||
func ignoreSessionIDWrites(t *testing.T, svc *service.Service, wantSessionID string) {
|
||||
t.Helper()
|
||||
svc.ExecSQL(t, fmt.Sprintf(`
|
||||
CREATE TRIGGER ignore_session_id_write
|
||||
BEFORE UPDATE OF session_id ON runs
|
||||
WHEN NEW.session_id = '%s'
|
||||
BEGIN
|
||||
SELECT RAISE(IGNORE);
|
||||
END;
|
||||
`, wantSessionID))
|
||||
}
|
||||
|
||||
// AC-002.11: on a direct (non-routed) launch, a real, non-empty session
|
||||
// id whose write affects zero rows (no error) must be counted as a
|
||||
// session persist failure, not miscounted as "launch yielded no session
|
||||
// id" (AC-002.8's counter).
|
||||
func TestSchedulerTick_DirectLaunchZeroRowSessionWriteCountsAsPersistFailure(t *testing.T) {
|
||||
mock := &mockTaskStarterWithSession{mockTaskStarter: &mockTaskStarter{}, sessionID: "sess-direct-vanished"}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskStarter: mock})
|
||||
ctx := context.Background()
|
||||
|
||||
agent := &models.AgentInstance{
|
||||
ID: "direct-vanished-agent",
|
||||
WorkspaceID: "ws-1",
|
||||
Name: "direct-vanished-worker",
|
||||
Role: models.AgentRoleWorker,
|
||||
Status: models.AgentStatusIdle,
|
||||
ExecutorPreference: `{"type":"worktree"}`,
|
||||
}
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, description, created_at, updated_at)
|
||||
VALUES ('task-direct-vanished-1', 'ws-1', 'Build API', 'Implement endpoint', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"task-direct-vanished-1"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
ignoreSessionIDWrites(t, svc, "sess-direct-vanished")
|
||||
|
||||
beforeFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
|
||||
beforeWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
|
||||
service.RunSchedulerTick(svc, ctx)
|
||||
|
||||
if mock.callCount() != 1 {
|
||||
t.Fatalf("expected 1 launch call, got %d", mock.callCount())
|
||||
}
|
||||
|
||||
afterFailed := directHopExpvarInt(t, "office_loop_session_persist_failed_total", "workspace=ws-1")
|
||||
if afterFailed != beforeFailed+1 {
|
||||
t.Fatalf("office_loop_session_persist_failed_total delta = %d, want 1 (AC-002.11: a real session id that failed to persist)", afterFailed-beforeFailed)
|
||||
}
|
||||
afterWithoutSession := directHopExpvarInt(t, "office_loop_launch_without_session_total", "workspace=ws-1")
|
||||
if afterWithoutSession != beforeWithoutSession {
|
||||
t.Fatalf("office_loop_launch_without_session_total delta = %d, want 0 (this counter is reserved for AC-002.8's empty-session-id case)", afterWithoutSession-beforeWithoutSession)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
func terminalShapeExpvarInt(t *testing.T, key string) int64 {
|
||||
t.Helper()
|
||||
v := expvar.Get("office_loop_terminal_total")
|
||||
if v == nil {
|
||||
t.Fatal("expvar map office_loop_terminal_total not registered")
|
||||
}
|
||||
m, ok := v.(*expvar.Map)
|
||||
if !ok {
|
||||
t.Fatal("office_loop_terminal_total is not a Map")
|
||||
}
|
||||
sub := m.Get(key)
|
||||
if sub == nil {
|
||||
return 0
|
||||
}
|
||||
iv, ok := sub.(*expvar.Int)
|
||||
if !ok {
|
||||
t.Fatalf("office_loop_terminal_total[%q] is not an Int", key)
|
||||
}
|
||||
return iv.Value()
|
||||
}
|
||||
|
||||
// REQ-OFFICE-LOOP-LIVENESS-005: FinishRun classifies the just-finished
|
||||
// run and increments office_loop_terminal_total by shape. A run
|
||||
// finished with outcome=processed and no session id classifies as
|
||||
// silent_success (activation is published at repo construction in
|
||||
// these in-memory test DBs, so the run's requested_at is after it).
|
||||
func TestFinishRun_RecordsSilentSuccessTerminalShape(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("terminal-shape-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: run=%v err=%v", run, err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeSilentSuccess))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if _, err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil {
|
||||
t.Fatalf("finish: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("silent_success delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed run with no session id classifies as unlaunched_failed.
|
||||
func TestFailRun_RecordsUnlaunchedFailedTerminalShape(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
agent := makeAgent("terminal-shape-fail-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
run, err := svc.ClaimNextRun(ctx)
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("claim: run=%v err=%v", run, err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if _, err := svc.FailRun(ctx, run.ID); err != nil {
|
||||
t.Fatalf("fail: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package service
|
||||
|
||||
import "time"
|
||||
|
||||
// TerminalShape is the derived classification of a terminal run
|
||||
// (REQ-OFFICE-LOOP-LIVENESS-005). It never widens runs.status or
|
||||
// runs.outcome — see docs/specs/task-delivery-ledger/spec.md, which
|
||||
// owns runs.outcome.
|
||||
type TerminalShape string
|
||||
|
||||
const (
|
||||
// ShapePreActivation is a run predating the point this capability's
|
||||
// columns were confirmed present, or read before that point was ever
|
||||
// published — never a defect (AC-005.6).
|
||||
ShapePreActivation TerminalShape = "pre_activation"
|
||||
// ShapeLaunchedCompleted is a run that reported success and named a
|
||||
// session.
|
||||
ShapeLaunchedCompleted TerminalShape = "launched_completed"
|
||||
// ShapeLaunchedFailed is a run that reached a terminal failure and
|
||||
// named a session.
|
||||
ShapeLaunchedFailed TerminalShape = "launched_failed"
|
||||
// ShapeSilentSuccess is a run whose outcome asserts work happened but
|
||||
// which never named a session — NFR-1 made observable (AC-005.4).
|
||||
ShapeSilentSuccess TerminalShape = "silent_success"
|
||||
// ShapeUnlaunchedSkipped is a legitimate no-launch skip, kept apart
|
||||
// from a launch failure (AC-005.5).
|
||||
ShapeUnlaunchedSkipped TerminalShape = "unlaunched_skipped"
|
||||
// ShapeUnlaunchedFailed is a terminal failure that never named a
|
||||
// session.
|
||||
ShapeUnlaunchedFailed TerminalShape = "unlaunched_failed"
|
||||
// ShapeUnclassified is every combination none of the above predicates
|
||||
// claim, including a null outcome, an outcome this document does not
|
||||
// enumerate (such as the historical no_agent_launched), and an
|
||||
// unknown status — the row that makes the classification total
|
||||
// (AC-005.2).
|
||||
ShapeUnclassified TerminalShape = "unclassified"
|
||||
)
|
||||
|
||||
var terminalFailedStatuses = map[string]bool{
|
||||
RunStatusFailed: true,
|
||||
"timed_out": true,
|
||||
RunStatusCancelled: true,
|
||||
}
|
||||
|
||||
var skipOutcomes = map[string]bool{
|
||||
RunOutcomeIdleSkipped: true,
|
||||
RunOutcomeBudgetBlocked: true,
|
||||
RunOutcomeBudgetUnmeasurable: true,
|
||||
RunOutcomeAgentInactive: true,
|
||||
RunOutcomeTaskTreeHeld: true,
|
||||
}
|
||||
|
||||
// ClassifyTerminalRun classifies one terminal run (status not in
|
||||
// {queued, claimed} — AC-005.9 excludes those from every terminal
|
||||
// total; they are the stuck-run detector's input instead) into exactly
|
||||
// one TerminalShape. Reads only the four persisted values named in its
|
||||
// signature, never a free-text message (AC-005.7), and evaluates
|
||||
// top-to-bottom with first match winning — the ordering is load-bearing:
|
||||
// pre_activation first so the activation guard cannot be bypassed by a
|
||||
// later row matching, silent_success below launched_completed so the
|
||||
// session test is what separates them, and above unclassified so a null
|
||||
// outcome never absorbs it.
|
||||
//
|
||||
// activationInstant/activationPublished carry the boot-time probe result
|
||||
// (persisted separately in kandev_meta); a run requested before
|
||||
// activation, or read when activation was never published, is always
|
||||
// pre_activation regardless of what it would otherwise classify as.
|
||||
func ClassifyTerminalRun(
|
||||
status string,
|
||||
outcome *string,
|
||||
sessionID string,
|
||||
requestedAt time.Time,
|
||||
activationInstant time.Time,
|
||||
activationPublished bool,
|
||||
) TerminalShape {
|
||||
if !activationPublished || requestedAt.Before(activationInstant) {
|
||||
return ShapePreActivation
|
||||
}
|
||||
|
||||
outcomeVal := ""
|
||||
if outcome != nil {
|
||||
outcomeVal = *outcome
|
||||
}
|
||||
return classifyByOutcome(status, outcomeVal, sessionID != "")
|
||||
}
|
||||
|
||||
// classifyByOutcome applies the first-match-wins predicate table once
|
||||
// activation is confirmed, split out from ClassifyTerminalRun to keep
|
||||
// each function's branching within the complexity limit.
|
||||
func classifyByOutcome(status, outcomeVal string, launched bool) TerminalShape {
|
||||
processed := outcomeVal == RunOutcomeProcessed
|
||||
failedTerminal := terminalFailedStatuses[status]
|
||||
|
||||
switch {
|
||||
case status == RunStatusFinished && processed && launched:
|
||||
return ShapeLaunchedCompleted
|
||||
case failedTerminal && launched:
|
||||
return ShapeLaunchedFailed
|
||||
case status == RunStatusFinished && processed && !launched:
|
||||
return ShapeSilentSuccess
|
||||
case status == RunStatusFinished && !launched && skipOutcomes[outcomeVal]:
|
||||
return ShapeUnlaunchedSkipped
|
||||
case failedTerminal && !launched:
|
||||
return ShapeUnlaunchedFailed
|
||||
default:
|
||||
return ShapeUnclassified
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/service"
|
||||
)
|
||||
|
||||
func strp(s string) *string { return &s }
|
||||
|
||||
// crossProductCase pins one (status, outcome, session presence) cell of
|
||||
// the AC-005.2 cross-product to the exact shape the spec's predicate
|
||||
// table assigns it — a literal expectation, not a second computation of
|
||||
// ClassifyTerminalRun's own branching, so a misclassification (not just
|
||||
// an out-of-set value) fails the test.
|
||||
type crossProductCase struct {
|
||||
status string
|
||||
outcome *string
|
||||
sessionID string
|
||||
want service.TerminalShape
|
||||
}
|
||||
|
||||
// crossProductCases enumerates all 5 statuses x 8 outcomes x 2 session
|
||||
// states (80 cells) this codebase's runs.status/outcome/session_id can
|
||||
// hold, each mapped to its expected shape by hand from the spec table.
|
||||
func crossProductCases() []crossProductCase {
|
||||
outcomeCases := []*string{
|
||||
nil,
|
||||
strp(service.RunOutcomeProcessed),
|
||||
strp(service.RunOutcomeIdleSkipped),
|
||||
strp(service.RunOutcomeBudgetBlocked),
|
||||
strp(service.RunOutcomeBudgetUnmeasurable),
|
||||
strp(service.RunOutcomeAgentInactive),
|
||||
strp(service.RunOutcomeTaskTreeHeld),
|
||||
strp("no_agent_launched"), // legacy value; not in current code
|
||||
}
|
||||
|
||||
cases := []crossProductCase{
|
||||
// status=finished: processed+session decides launched_completed
|
||||
// vs. silent_success; a skip outcome decides unlaunched_skipped
|
||||
// only without a session; everything else is unclassified.
|
||||
{"finished", nil, "", service.ShapeUnclassified},
|
||||
{"finished", nil, "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess},
|
||||
{"finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted},
|
||||
{"finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped},
|
||||
{"finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped},
|
||||
{"finished", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped},
|
||||
{"finished", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped},
|
||||
{"finished", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped},
|
||||
{"finished", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified},
|
||||
{"finished", strp("no_agent_launched"), "", service.ShapeUnclassified},
|
||||
{"finished", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified},
|
||||
|
||||
// status=some_unknown_status: never finished, never a recognized
|
||||
// failure status, so always unclassified regardless of outcome
|
||||
// or session.
|
||||
{"some_unknown_status", nil, "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", nil, "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeProcessed), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeBudgetBlocked), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeBudgetUnmeasurable), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeAgentInactive), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp(service.RunOutcomeTaskTreeHeld), "sess-1", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp("no_agent_launched"), "", service.ShapeUnclassified},
|
||||
{"some_unknown_status", strp("no_agent_launched"), "sess-1", service.ShapeUnclassified},
|
||||
}
|
||||
|
||||
// status IN (failed, cancelled, timed_out): the spec's predicate
|
||||
// table names all three under one row and never reads outcome for
|
||||
// them, so every outcome yields the same pair of shapes for each —
|
||||
// unlaunched_failed without a session, launched_failed with one.
|
||||
for _, status := range []string{"failed", "cancelled", "timed_out"} {
|
||||
for _, outcome := range outcomeCases {
|
||||
cases = append(cases,
|
||||
crossProductCase{status, outcome, "", service.ShapeUnlaunchedFailed},
|
||||
crossProductCase{status, outcome, "sess-1", service.ShapeLaunchedFailed},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return cases
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-005.1/.2: classification is total over the
|
||||
// cross-product of every persisted status this codebase writes
|
||||
// (finished, failed, cancelled, timed_out, and an unknown status —
|
||||
// runs.status is not the closed RunStatus enum) against every observed
|
||||
// outcome (including the legacy no_agent_launched value and NULL),
|
||||
// crossed with session_id present/absent — pinned to the exact shape
|
||||
// each of the 80 cells must produce, not just closed-set membership.
|
||||
func TestClassifyTerminalRun_CrossProduct(t *testing.T) {
|
||||
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
requestedAt := activation.Add(time.Hour)
|
||||
|
||||
cases := crossProductCases()
|
||||
if len(cases) != 5*8*2 {
|
||||
t.Fatalf("cross-product case count = %d, want %d (5 statuses x 8 outcomes x 2 session states)",
|
||||
len(cases), 5*8*2)
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := service.ClassifyTerminalRun(
|
||||
tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true,
|
||||
)
|
||||
if got != tc.want {
|
||||
t.Errorf("ClassifyTerminalRun(status=%q, outcome=%v, session=%q) = %q, want %q",
|
||||
tc.status, tc.outcome, tc.sessionID, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyTerminalRun_PreActivation(t *testing.T) {
|
||||
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Before the activation instant: pre_activation regardless of
|
||||
// otherwise-silent-success shape (AC-005.6).
|
||||
requestedBefore := activation.Add(-time.Minute)
|
||||
got := service.ClassifyTerminalRun(
|
||||
"finished", strp(service.RunOutcomeProcessed), "", requestedBefore, activation, true,
|
||||
)
|
||||
if got != service.ShapePreActivation {
|
||||
t.Fatalf("got %q, want pre_activation", got)
|
||||
}
|
||||
|
||||
// Activation instant never published: always pre_activation.
|
||||
got = service.ClassifyTerminalRun(
|
||||
"finished", strp(service.RunOutcomeProcessed), "", requestedBefore.Add(10*time.Hour), activation, false,
|
||||
)
|
||||
if got != service.ShapePreActivation {
|
||||
t.Fatalf("got %q, want pre_activation when unpublished", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyTerminalRun_NamedShapes(t *testing.T) {
|
||||
activation := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
requestedAt := activation.Add(time.Hour)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
status string
|
||||
outcome *string
|
||||
sessionID string
|
||||
want service.TerminalShape
|
||||
}{
|
||||
{"launched_completed", "finished", strp(service.RunOutcomeProcessed), "sess-1", service.ShapeLaunchedCompleted},
|
||||
{"launched_failed/failed", "failed", nil, "sess-1", service.ShapeLaunchedFailed},
|
||||
{"launched_failed/timed_out", "timed_out", nil, "sess-1", service.ShapeLaunchedFailed},
|
||||
{"launched_failed/cancelled", "cancelled", nil, "sess-1", service.ShapeLaunchedFailed},
|
||||
{"silent_success", "finished", strp(service.RunOutcomeProcessed), "", service.ShapeSilentSuccess},
|
||||
{"unlaunched_skipped/idle", "finished", strp(service.RunOutcomeIdleSkipped), "", service.ShapeUnlaunchedSkipped},
|
||||
{"unlaunched_skipped/budget", "finished", strp(service.RunOutcomeBudgetBlocked), "", service.ShapeUnlaunchedSkipped},
|
||||
{"unlaunched_skipped/budget_unmeasurable", "finished", strp(service.RunOutcomeBudgetUnmeasurable), "", service.ShapeUnlaunchedSkipped},
|
||||
{"unlaunched_skipped/inactive", "finished", strp(service.RunOutcomeAgentInactive), "", service.ShapeUnlaunchedSkipped},
|
||||
{"unlaunched_skipped/tree_held", "finished", strp(service.RunOutcomeTaskTreeHeld), "", service.ShapeUnlaunchedSkipped},
|
||||
{"unlaunched_failed", "failed", nil, "", service.ShapeUnlaunchedFailed},
|
||||
{"unclassified/null_outcome_finished", "finished", nil, "", service.ShapeUnclassified},
|
||||
{"unclassified/legacy_no_agent_launched", "finished", strp("no_agent_launched"), "", service.ShapeUnclassified},
|
||||
// A skip outcome carrying a session id is not a real production
|
||||
// shape (skip outcomes are pre-launch guards) but the total
|
||||
// classification must still place it somewhere, and it must not
|
||||
// be reported as "unlaunched" while a session is on record
|
||||
// (AC-005.3) — it falls to unclassified.
|
||||
{"unclassified/skip_with_session", "finished", strp(service.RunOutcomeIdleSkipped), "sess-1", service.ShapeUnclassified},
|
||||
{"unclassified/unknown_status", "some_unknown_status", nil, "", service.ShapeUnclassified},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := service.ClassifyTerminalRun(tc.status, tc.outcome, tc.sessionID, requestedAt, activation, true)
|
||||
if got != tc.want {
|
||||
t.Fatalf("ClassifyTerminalRun(%q, %v, %q) = %q, want %q",
|
||||
tc.status, tc.outcome, tc.sessionID, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -205,6 +205,25 @@ func AdmitRunForTest(svc *Service, ctx context.Context, run *models.Run, agent *
|
||||
return si.admitRun(ctx, run, agent)
|
||||
}
|
||||
|
||||
// DeferWorkspaceLookupFailureForTest exposes deferWorkspaceLookupFailure
|
||||
// directly for external test packages, so its MaxRetryCount-exhausted
|
||||
// lost-race path (another writer already made the run terminal before
|
||||
// this call's FailRun runs) can be driven without reproducing a transient
|
||||
// GetAgentFromConfig lookup error through the full scheduler pipeline.
|
||||
func DeferWorkspaceLookupFailureForTest(svc *Service, ctx context.Context, run *models.Run) {
|
||||
si := &SchedulerIntegration{svc: svc, logger: svc.logger}
|
||||
si.deferWorkspaceLookupFailure(ctx, run)
|
||||
}
|
||||
|
||||
// FailTasklessRunForTest exposes failTasklessRun directly for external test
|
||||
// packages, so the lost-race path (another writer already made the run
|
||||
// terminal before this call runs) can be driven without the scheduler's
|
||||
// atomic claim+launch pipeline standing in the way.
|
||||
func FailTasklessRunForTest(svc *Service, ctx context.Context, run *models.Run, agent *models.AgentInstance, msg string) {
|
||||
si := &SchedulerIntegration{svc: svc, logger: svc.logger}
|
||||
si.failTasklessRun(ctx, run, agent, msg)
|
||||
}
|
||||
|
||||
// LogPolicyObservabilityForTest exposes logPolicyObservability for external
|
||||
// test packages, so the skip/degraded-admitted entries and their
|
||||
// per-policy-per-day dedup (AC-OFFICE-BUDGET-002.13/-004.8) can be tested
|
||||
|
||||
@@ -55,9 +55,11 @@ func (s *Service) PauseTaskTree(ctx context.Context, rootTaskID string) (*models
|
||||
return nil, fmt.Errorf("create pause members: %w", err)
|
||||
}
|
||||
s.cancelTaskExecutions(ctx, members, "tree_paused")
|
||||
if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused"); err != nil {
|
||||
cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_paused")
|
||||
if err != nil {
|
||||
s.logger.Warn("cancel runs for paused tree failed", zap.Error(err))
|
||||
}
|
||||
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
|
||||
s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold)
|
||||
return hold, nil
|
||||
}
|
||||
@@ -99,9 +101,11 @@ func (s *Service) CancelTaskTree(ctx context.Context, rootTaskID, cancelledBy st
|
||||
if err := s.repo.BulkReleaseTaskCheckout(ctx, memberTaskIDs(members)); err != nil {
|
||||
return nil, fmt.Errorf("release task checkouts: %w", err)
|
||||
}
|
||||
if _, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled"); err != nil {
|
||||
cancelled, err := s.repo.CancelRunsForTasks(ctx, memberTaskIDs(members), "tree_cancelled")
|
||||
if err != nil {
|
||||
s.logger.Warn("cancel runs for cancelled tree failed", zap.Error(err))
|
||||
}
|
||||
s.recordTerminalShapesForCancelledRuns(ctx, cancelled)
|
||||
_ = cancelledBy
|
||||
s.publishTreeHoldEvent(ctx, events.OfficeTaskTreeHoldCreated, hold)
|
||||
return hold, nil
|
||||
|
||||
@@ -61,6 +61,40 @@ func TestPauseTaskTree(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns is the Testing
|
||||
// round 3 regression test. PauseTaskTree cancels every member task's runs
|
||||
// via repo.CancelRunsForTasks, which bypassed office_loop_terminal_total
|
||||
// entirely — the same bypass class Review round 1 fixed for the
|
||||
// single-run CancelRun call sites. A queued run never launched, so
|
||||
// pausing its tree must classify the cancellation as unlaunched_failed.
|
||||
func TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) {
|
||||
canceller := &mockTaskCanceller{}
|
||||
svc := newTestService(t, service.ServiceOptions{TaskCanceller: canceller})
|
||||
ctx := context.Background()
|
||||
insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS")
|
||||
|
||||
agent := makeAgent("tree-pause-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"root"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if _, err := svc.PauseTaskTree(ctx, "root"); err != nil {
|
||||
t.Fatalf("PauseTaskTree: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelAndRestoreTaskTree(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
@@ -106,3 +140,33 @@ func TestCancelAndRestoreTaskTree(t *testing.T) {
|
||||
t.Fatalf("active hold after restore = %+v, want nil; cancel hold was %s", preview.ActiveHold, hold.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns mirrors
|
||||
// TestPauseTaskTree_RecordsTerminalShapeForCancelledRuns for
|
||||
// CancelTaskTree's own repo.CancelRunsForTasks call site.
|
||||
func TestCancelTaskTree_RecordsTerminalShapeForCancelledRuns(t *testing.T) {
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
insertTreeControlTask(t, svc, "root", "", "IN_PROGRESS")
|
||||
|
||||
agent := makeAgent("tree-cancel-worker", models.AgentRoleWorker)
|
||||
if err := svc.CreateAgentInstance(ctx, agent); err != nil {
|
||||
t.Fatalf("create agent: %v", err)
|
||||
}
|
||||
if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned,
|
||||
`{"task_id":"root"}`, ""); err != nil {
|
||||
t.Fatalf("queue: %v", err)
|
||||
}
|
||||
|
||||
key := service.LoopMetricLabel("workspace", agent.WorkspaceID, "shape", string(service.ShapeUnlaunchedFailed))
|
||||
before := terminalShapeExpvarInt(t, key)
|
||||
|
||||
if _, err := svc.CancelTaskTree(ctx, "root", "user:test"); err != nil {
|
||||
t.Fatalf("CancelTaskTree: %v", err)
|
||||
}
|
||||
|
||||
after := terminalShapeExpvarInt(t, key)
|
||||
if after != before+1 {
|
||||
t.Fatalf("unlaunched_failed delta = %d, want 1", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,7 +189,7 @@ func seedAgentFailureHandler(
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil {
|
||||
if _, err := repo.MarkRunFailed(ctx, run.ID, req.ErrorMessage); err != nil {
|
||||
log.Error("test harness: mark run failed", zap.Error(err))
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package wakeup_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
officemodels "github.com/kandev/kandev/internal/office/models"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/office/shared"
|
||||
"github.com/kandev/kandev/internal/office/wakeup"
|
||||
)
|
||||
|
||||
func (h *testHarness) seedWakeupWithCausation(id, source, payload, causationID string) {
|
||||
h.t.Helper()
|
||||
if err := h.repo.CreateWakeupRequest(context.Background(), &officesqlite.WakeupRequest{
|
||||
ID: id, AgentProfileID: h.agentID, Source: source, Payload: payload, CausationID: causationID,
|
||||
}); err != nil {
|
||||
h.t.Fatalf("seed wakeup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *testHarness) seedRunWithReasonAndCausation(id, status, reason, causationID string) {
|
||||
h.t.Helper()
|
||||
run := &officemodels.Run{
|
||||
ID: id,
|
||||
AgentProfileID: h.agentID,
|
||||
Reason: reason,
|
||||
Payload: "{}",
|
||||
Status: officemodels.RunStatus(status),
|
||||
CoalescedCount: 1,
|
||||
ContextSnapshot: `{"prior":"snapshot"}`,
|
||||
CausationID: causationID,
|
||||
}
|
||||
if err := h.repo.CreateRun(context.Background(), run); err != nil {
|
||||
h.t.Fatalf("seed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.3: createFreshRun copies the causation id
|
||||
// from the wakeup request onto the fresh run.
|
||||
func TestDispatch_FreshRunCopiesCausationIDFromWakeup(t *testing.T) {
|
||||
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
|
||||
h.seedWakeupWithCausation("w-1", wakeup.SourceSelf, `{"reason":"test"}`, "cause-fresh-1")
|
||||
|
||||
if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
got, err := h.repo.GetWakeupRequest(context.Background(), "w-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get wakeup request: %v", err)
|
||||
}
|
||||
run, err := h.repo.GetRunByID(context.Background(), got.RunID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if run.CausationID != "cause-fresh-1" {
|
||||
t.Fatalf("run.CausationID = %q, want cause-fresh-1", run.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.4: coalescing touches only the wakeup
|
||||
// request, never the run. The in-flight run keeps the id of the wake
|
||||
// that created it; the coalesced request keeps its own — the two stay
|
||||
// joinable through agent_wakeup_requests.run_id.
|
||||
func TestDispatch_CoalesceLeavesRunCausationIDUntouched(t *testing.T) {
|
||||
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
|
||||
h.seedRunWithReasonAndCausation("run-pre", "queued", shared.RunReasonHeartbeat, "cause-run-original")
|
||||
h.seedWakeupWithCausation("w-1", wakeup.SourceComment, `{"task_id":"t-1"}`, "cause-wakeup-coalesced")
|
||||
|
||||
if err := h.dispatcher.Dispatch(context.Background(), "w-1"); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
|
||||
gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get wakeup request: %v", err)
|
||||
}
|
||||
if gotReq.CausationID != "cause-wakeup-coalesced" {
|
||||
t.Fatalf("coalesced request CausationID = %q, want unchanged cause-wakeup-coalesced", gotReq.CausationID)
|
||||
}
|
||||
if gotReq.RunID != "run-pre" {
|
||||
t.Fatalf("coalesced request run_id = %q, want run-pre", gotReq.RunID)
|
||||
}
|
||||
|
||||
gotRun, err := h.repo.GetRunByID(context.Background(), "run-pre")
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if gotRun.CausationID != "cause-run-original" {
|
||||
t.Fatalf("run.CausationID = %q, want unchanged cause-run-original (coalesce must not touch it)", gotRun.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.4 skip half: MarkWakeupRequestSkipped touches only the
|
||||
// request; it has no run to touch, and the request's own causation id
|
||||
// survives the transition.
|
||||
func TestDispatch_SkipLeavesWakeupCausationIDUntouched(t *testing.T) {
|
||||
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
|
||||
// skip_if_active requires the routine lookup to resolve that policy;
|
||||
// simplest deterministic path here is a non-routine source with an
|
||||
// in-flight run under the default coalesce policy is not skip — so
|
||||
// drive skip via the routine lookup fallback used elsewhere in this
|
||||
// package (skip_if_active policy routine).
|
||||
h.dispatcher.SetRoutineLookup(fakeRoutineLookupSkip{})
|
||||
h.seedRunWithReasonAndCausation("run-inflight", "claimed", shared.RunReasonRoutineDispatchCron, "cause-inflight")
|
||||
h.seedWakeupWithCausation("w-skip", wakeup.SourceRoutine, `{"routine_id":"r-skip"}`, "cause-skip-1")
|
||||
|
||||
if err := h.dispatcher.Dispatch(context.Background(), "w-skip"); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
got, err := h.repo.GetWakeupRequest(context.Background(), "w-skip")
|
||||
if err != nil {
|
||||
t.Fatalf("get wakeup request: %v", err)
|
||||
}
|
||||
if got.Status != officesqlite.WakeupStatusSkipped {
|
||||
t.Fatalf("status = %q, want skipped", got.Status)
|
||||
}
|
||||
if got.CausationID != "cause-skip-1" {
|
||||
t.Fatalf("skipped request CausationID = %q, want unchanged cause-skip-1", got.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.4/AC-002.5: PromoteRunAndCoalesceWakeupIfQueued promotes the
|
||||
// run's reason but must never touch causation_id — a promotion is not a
|
||||
// new cause.
|
||||
func TestDispatch_PromotionDoesNotChangeRunCausationID(t *testing.T) {
|
||||
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
|
||||
h.seedRunWithReasonAndCausation("run-cron", "queued", shared.RunReasonRoutineDispatchCron, "cause-cron-original")
|
||||
h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-1")
|
||||
// Give the event wakeup an event-classified reason so promotion fires.
|
||||
if _, err := h.repo.ExecRaw(context.Background(),
|
||||
`UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`,
|
||||
shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil {
|
||||
t.Fatalf("seed reason: %v", err)
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
run, err := h.repo.GetRunByID(context.Background(), "run-cron")
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if run.Reason != shared.RunReasonRoutineDispatchEvent {
|
||||
t.Fatalf("run.Reason = %q, want promoted to %q", run.Reason, shared.RunReasonRoutineDispatchEvent)
|
||||
}
|
||||
if run.CausationID != "cause-cron-original" {
|
||||
t.Fatalf("run.CausationID = %q, want unchanged cause-cron-original (promotion is not a new cause)", run.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-002.3/AC-002.4: the lost-CAS fresh-run path (event coalescing into
|
||||
// an already-claimed cron run) creates its own run, and that run
|
||||
// carries the *requesting* wake's causation id, not the claimed run's.
|
||||
func TestDispatch_LostCASFreshRunCarriesRequestingWakeCausationID(t *testing.T) {
|
||||
h := newHarness(t, wakeup.PolicyCoalesceIfActive)
|
||||
h.seedRunWithReasonAndCausation("run-cron", "claimed", shared.RunReasonRoutineDispatchCron, "cause-claimed-cron")
|
||||
h.seedWakeupWithCausation("w-event", wakeup.SourceRoutine, `{"routine_id":"r-1"}`, "cause-event-fresh")
|
||||
if _, err := h.repo.ExecRaw(context.Background(),
|
||||
`UPDATE agent_wakeup_requests SET reason = ? WHERE id = ?`,
|
||||
shared.RunReasonRoutineDispatchEvent, "w-event"); err != nil {
|
||||
t.Fatalf("seed reason: %v", err)
|
||||
}
|
||||
|
||||
if err := h.dispatcher.Dispatch(context.Background(), "w-event"); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
gotReq, err := h.repo.GetWakeupRequest(context.Background(), "w-event")
|
||||
if err != nil {
|
||||
t.Fatalf("get wakeup request: %v", err)
|
||||
}
|
||||
if gotReq.RunID == "" || gotReq.RunID == "run-cron" {
|
||||
t.Fatalf("expected a fresh run distinct from the claimed one, got %q", gotReq.RunID)
|
||||
}
|
||||
freshRun, err := h.repo.GetRunByID(context.Background(), gotReq.RunID)
|
||||
if err != nil {
|
||||
t.Fatalf("get fresh run: %v", err)
|
||||
}
|
||||
if freshRun.CausationID != "cause-event-fresh" {
|
||||
t.Fatalf("fresh run.CausationID = %q, want cause-event-fresh (the requesting wake's id)", freshRun.CausationID)
|
||||
}
|
||||
|
||||
claimedRun, err := h.repo.GetRunByID(context.Background(), "run-cron")
|
||||
if err != nil {
|
||||
t.Fatalf("get claimed run: %v", err)
|
||||
}
|
||||
if claimedRun.CausationID != "cause-claimed-cron" {
|
||||
t.Fatalf("claimed run.CausationID = %q, want unchanged cause-claimed-cron", claimedRun.CausationID)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRoutineLookupSkip resolves every routine to skip_if_active so the
|
||||
// skip transition can be driven deterministically without a real
|
||||
// office_routines row.
|
||||
type fakeRoutineLookupSkip struct{}
|
||||
|
||||
func (fakeRoutineLookupSkip) GetRoutine(_ context.Context, id string) (*officemodels.Routine, error) {
|
||||
return &officemodels.Routine{ID: id, ConcurrencyPolicy: officemodels.RoutineConcurrencyPolicy(wakeup.PolicySkipIfActive)}, nil
|
||||
}
|
||||
@@ -290,6 +290,10 @@ func (d *Dispatcher) createFreshRun(
|
||||
CoalescedCount: 1,
|
||||
ContextSnapshot: payload,
|
||||
RequestedAt: time.Now().UTC(),
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.3: copied from the requesting
|
||||
// wake, including on the lost-CAS fresh-run path — that run
|
||||
// still carries the requesting wake's id, not a new one.
|
||||
CausationID: req.CausationID,
|
||||
}
|
||||
if err := d.repo.CreateRun(ctx, run); err != nil {
|
||||
return fmt.Errorf("create run for wakeup %s: %w", req.ID, err)
|
||||
|
||||
@@ -1102,8 +1102,8 @@ type startTaskOptions struct {
|
||||
func (s *Service) StartTaskWithRoute(
|
||||
ctx context.Context, taskID, agentProfileID string,
|
||||
launch executor.LaunchContext, route executor.RouteOverride,
|
||||
) error {
|
||||
_, err := s.startTask(ctx, taskID, agentProfileID,
|
||||
) (*executor.TaskExecution, error) {
|
||||
return s.startTask(ctx, taskID, agentProfileID,
|
||||
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority,
|
||||
launch.Prompt, launch.WorkflowStepID, launch.PlanMode, false,
|
||||
launch.Attachments, startTaskOptions{
|
||||
@@ -1111,7 +1111,6 @@ func (s *Service) StartTaskWithRoute(
|
||||
AdditionalSkillSlugs: append([]string(nil), launch.AdditionalSkillSlugs...),
|
||||
Route: &route,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) prepareExplicitWorkflowStartRoute(
|
||||
|
||||
@@ -225,7 +225,7 @@ func TestReadPathsUseTheReadOnlyHandle(t *testing.T) {
|
||||
}
|
||||
|
||||
// Writes must not fall back to the reader.
|
||||
if err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil {
|
||||
if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil {
|
||||
t.Errorf("FinishRun succeeded with a closed writer; the write path is not using r.db")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,54 +16,81 @@ import (
|
||||
// and have its real finished_at overwritten.
|
||||
const cancellableRunStatuses = `status IN ('queued', 'claimed')`
|
||||
|
||||
// CancelledRun identifies one row CancelRunsWhere actually cancelled,
|
||||
// carrying the fields a caller needs to classify the transition's
|
||||
// terminal shape (office_loop_terminal_total) without a second read.
|
||||
type CancelledRun struct {
|
||||
ID string
|
||||
AgentProfileID string
|
||||
SessionID string
|
||||
RequestedAt time.Time
|
||||
}
|
||||
|
||||
// CancelRunsWhere is the single writer of the terminal cancel state on the
|
||||
// runs table. CancelRun, BulkCancelRuns and the office repository's
|
||||
// CancelRunsForTasks are all thin selectors over it, so the guard above
|
||||
// cannot be bypassed by picking a different entry point.
|
||||
// CancelRunsForTasks and CancelDisplacedParticipantRun are all thin
|
||||
// selectors over it, so the guard above cannot be bypassed by picking a
|
||||
// different entry point.
|
||||
//
|
||||
// selector is an extra WHERE fragment ANDed onto the guard; it must be a
|
||||
// SQL literal owned by the calling repository method, using ? placeholders
|
||||
// (rebound here) for every value. Never interpolate caller-supplied data
|
||||
// into it. selectorArgs bind those placeholders, in order.
|
||||
//
|
||||
// Returns the number of rows actually cancelled: 0 means every run the
|
||||
// selector matched had already reached a terminal state.
|
||||
// Returns the rows actually cancelled: an empty slice means every run the
|
||||
// selector matched had already reached a terminal state. The guard and the
|
||||
// write are the same UPDATE statement (RETURNING reads the classification
|
||||
// columns off the rows it just changed), so there is no separate read a
|
||||
// row's status could change behind: a run that reaches a terminal state on
|
||||
// another connection is either fully visible to this statement or not, and
|
||||
// either way it cannot be cancelled by it.
|
||||
func (r *Repository) CancelRunsWhere(
|
||||
ctx context.Context,
|
||||
cancelReason string,
|
||||
selector string,
|
||||
selectorArgs ...interface{},
|
||||
) (int64, error) {
|
||||
args := make([]interface{}, 0, len(selectorArgs)+2)
|
||||
args = append(args, cancelReason, time.Now().UTC())
|
||||
args = append(args, selectorArgs...)
|
||||
|
||||
) ([]CancelledRun, error) {
|
||||
query := fmt.Sprintf(`
|
||||
UPDATE runs
|
||||
SET status = 'cancelled', cancel_reason = ?, finished_at = ?
|
||||
WHERE %s
|
||||
AND (%s)
|
||||
RETURNING id, agent_profile_id, session_id, requested_at
|
||||
`, cancellableRunStatuses, selector)
|
||||
|
||||
res, err := r.db.ExecContext(ctx, r.db.Rebind(query), args...)
|
||||
args := append([]interface{}{cancelReason, time.Now().UTC()}, selectorArgs...)
|
||||
rows, err := r.db.QueryxContext(ctx, r.db.Rebind(query), args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return nil, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
defer rows.Close() //nolint:errcheck
|
||||
|
||||
var cancelled []CancelledRun
|
||||
for rows.Next() {
|
||||
var row CancelledRun
|
||||
if err := rows.Scan(&row.ID, &row.AgentProfileID, &row.SessionID, &row.RequestedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cancelled = append(cancelled, row)
|
||||
}
|
||||
return cancelled, rows.Err()
|
||||
}
|
||||
|
||||
// CancelRun marks a run as cancelled with an optional cancel reason.
|
||||
// A run that already reached a terminal state is left untouched.
|
||||
func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) error {
|
||||
_, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id)
|
||||
return err
|
||||
// A run that already reached a terminal state is left untouched. Returns
|
||||
// whether this call actually cancelled the row, so a caller counting the
|
||||
// transition (office_loop_terminal_total) can tell a persisted change
|
||||
// from a no-op on a run that was already terminal.
|
||||
func (r *Repository) CancelRun(ctx context.Context, id, cancelReason string) (bool, error) {
|
||||
rows, err := r.CancelRunsWhere(ctx, cancelReason, `id = ?`, id)
|
||||
return len(rows) > 0, err
|
||||
}
|
||||
|
||||
// BulkCancelRuns cancels multiple runs by ID with the given reason.
|
||||
// Runs in the set that already reached a terminal state are left untouched.
|
||||
func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) error {
|
||||
// BulkCancelRuns cancels multiple runs by ID with the given reason,
|
||||
// returning the rows actually cancelled (see CancelRunsWhere). Runs in
|
||||
// the set that already reached a terminal state are left untouched.
|
||||
func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelReason string) ([]CancelledRun, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
placeholders := make([]string, len(ids))
|
||||
args := make([]interface{}, len(ids))
|
||||
@@ -72,6 +99,5 @@ func (r *Repository) BulkCancelRuns(ctx context.Context, ids []string, cancelRea
|
||||
args[i] = id
|
||||
}
|
||||
selector := fmt.Sprintf(`id IN (%s)`, strings.Join(placeholders, ","))
|
||||
_, err := r.CancelRunsWhere(ctx, cancelReason, selector, args...)
|
||||
return err
|
||||
return r.CancelRunsWhere(ctx, cancelReason, selector, args...)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
// TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun is
|
||||
// the two-connection regression for a TOCTOU CancelRunsWhere used to have:
|
||||
// its cancellable-status guard was applied only to the candidate read, not
|
||||
// to the write, so a run that genuinely finished on another connection
|
||||
// between the two could be silently overwritten to cancelled. Real
|
||||
// Postgres row locking forces the interleaving this needs: one connection
|
||||
// holds an uncommitted finish on the row, CancelRunsWhere's write blocks on
|
||||
// that row lock, and only proceeds once the finish commits — at which
|
||||
// point it must see the fresh 'finished' status and cancel nothing. Skips
|
||||
// unless KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresCancelRunsWhere_DoesNotOverwriteAConcurrentlyFinishedRun(t *testing.T) {
|
||||
dsn := testutil.PostgresDSNFromEnv(t)
|
||||
// db is the repo's own pool: OpenIsolatedPostgres pins it to exactly
|
||||
// one physical connection and sets that connection's search_path to a
|
||||
// fresh schema, so every statement through it (including the schema
|
||||
// init below and repo.CancelRunsWhere's own transaction) lands there.
|
||||
db := testutil.OpenIsolatedPostgres(t, dsn)
|
||||
|
||||
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo: %v", err)
|
||||
}
|
||||
officeRepo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init office repo: %v", err)
|
||||
}
|
||||
repo := officeRepo.RunsRepository()
|
||||
ctx := context.Background()
|
||||
|
||||
// finisherDB is a second, genuinely separate physical connection to
|
||||
// the SAME schema. A pool-wide MaxOpenConns bump on a single *sqlx.DB
|
||||
// would not do this safely: search_path is a per-connection session
|
||||
// setting, so a new connection the pool opens under load would not
|
||||
// inherit it and could silently operate against the wrong schema.
|
||||
var schema string
|
||||
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
|
||||
t.Fatalf("read isolated schema name: %v", err)
|
||||
}
|
||||
finisherDB, err := sqlx.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open finisher connection: %v", err)
|
||||
}
|
||||
finisherDB.SetMaxOpenConns(1)
|
||||
finisherDB.SetMaxIdleConns(1)
|
||||
t.Cleanup(func() { _ = finisherDB.Close() })
|
||||
if _, err := finisherDB.Exec("SET search_path TO " + schema); err != nil {
|
||||
t.Fatalf("set finisher search_path: %v", err)
|
||||
}
|
||||
|
||||
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
|
||||
), now, run.ID); err != nil {
|
||||
t.Fatalf("seed claimed: %v", err)
|
||||
}
|
||||
|
||||
tx, err := finisherDB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin finisher tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
if _, err := tx.ExecContext(ctx, finisherDB.Rebind(
|
||||
`UPDATE runs SET status = 'finished', outcome = 'processed', finished_at = ? WHERE id = ?`,
|
||||
), now, run.ID); err != nil {
|
||||
t.Fatalf("finisher update (holds the row lock, uncommitted): %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
cancelled []runssqlite.CancelledRun
|
||||
cancelErr error
|
||||
cancelDone = make(chan struct{})
|
||||
commitErr error
|
||||
)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer close(cancelDone)
|
||||
cancelled, cancelErr = repo.CancelRunsWhere(ctx, "too late", `id = ?`, run.ID)
|
||||
}()
|
||||
|
||||
// Give CancelRunsWhere's write time to reach Postgres and block on the
|
||||
// row lock the finisher's uncommitted transaction holds, before that
|
||||
// transaction commits. Without this window the two goroutines could
|
||||
// run in either order and the test would not exercise the lock-wait
|
||||
// path this bug depends on.
|
||||
select {
|
||||
case <-cancelDone:
|
||||
t.Fatal("CancelRunsWhere returned before the finisher committed; it never blocked on the row lock")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
commitErr = tx.Commit()
|
||||
wg.Wait()
|
||||
if commitErr != nil {
|
||||
t.Fatalf("finisher commit: %v", commitErr)
|
||||
}
|
||||
if cancelErr != nil {
|
||||
t.Fatalf("cancel where: %v", cancelErr)
|
||||
}
|
||||
|
||||
if len(cancelled) != 0 {
|
||||
t.Fatalf("cancelled = %d, want 0: a run that committed finished must not be reported cancelled", len(cancelled))
|
||||
}
|
||||
if got := mustGetRunStatus(t, db, run.ID); got != "finished" {
|
||||
t.Fatalf("run status = %q, want finished (the concurrent cancel must not have overwritten it)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustGetRunStatus(t *testing.T, db *sqlx.DB, id string) string {
|
||||
t.Helper()
|
||||
var status string
|
||||
if err := db.Get(&status, db.Rebind(`SELECT status FROM runs WHERE id = ?`), id); err != nil {
|
||||
t.Fatalf("get run status: %v", err)
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
runssqlite "github.com/kandev/kandev/internal/runs/repository/sqlite"
|
||||
)
|
||||
|
||||
// TestCancelRun_CancelsQueuedAndClaimedRuns pins every column the
|
||||
@@ -21,9 +22,13 @@ func TestCancelRun_CancelsQueuedAndClaimedRuns(t *testing.T) {
|
||||
setStatus(t, repo, claimed.ID, "claimed", timePtr(now), nil)
|
||||
|
||||
for _, id := range []string{queued.ID, claimed.ID} {
|
||||
if err := repo.CancelRun(ctx, id, "user stopped the task"); err != nil {
|
||||
cancelled, err := repo.CancelRun(ctx, id, "user stopped the task")
|
||||
if err != nil {
|
||||
t.Fatalf("cancel %q: %v", id, err)
|
||||
}
|
||||
if !cancelled {
|
||||
t.Errorf("%s: cancelled = false, want true", id)
|
||||
}
|
||||
got := mustGetRun(t, repo, id)
|
||||
checkString(t, id+" status", string(got.Status), "cancelled")
|
||||
checkStringPtr(t, id+" cancel_reason", got.CancelReason, strPtr("user stopped the task"))
|
||||
@@ -47,9 +52,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) {
|
||||
run := queueRunAt(t, repo, "run-"+status, "a1", finishedAt.Add(-time.Hour))
|
||||
setStatus(t, repo, run.ID, status, nil, timePtr(finishedAt))
|
||||
|
||||
if err := repo.CancelRun(ctx, run.ID, "too late"); err != nil {
|
||||
cancelled, err := repo.CancelRun(ctx, run.ID, "too late")
|
||||
if err != nil {
|
||||
t.Fatalf("cancel %q: %v", run.ID, err)
|
||||
}
|
||||
if cancelled {
|
||||
t.Errorf("%s: cancelled = true, want false (already terminal)", run.ID)
|
||||
}
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
checkString(t, run.ID+" status", string(got.Status), status)
|
||||
if got.CancelReason != nil {
|
||||
@@ -64,9 +73,13 @@ func TestCancelRun_TerminalRunIsLeftAlone(t *testing.T) {
|
||||
// TestCancelRun_UnknownRunIsANoOp pins that a missing id is not an error.
|
||||
func TestCancelRun_UnknownRunIsANoOp(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
if err := repo.CancelRun(context.Background(), "nope", "reason"); err != nil {
|
||||
cancelled, err := repo.CancelRun(context.Background(), "nope", "reason")
|
||||
if err != nil {
|
||||
t.Fatalf("cancel unknown run: %v", err)
|
||||
}
|
||||
if cancelled {
|
||||
t.Error("cancelled = true, want false (unknown run)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCancelRunsWhere_ReportsRowsActuallyCancelled pins the returned
|
||||
@@ -87,8 +100,8 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("cancel where: %v", err)
|
||||
}
|
||||
if cancelled != 2 {
|
||||
t.Errorf("cancelled = %d, want 2", cancelled)
|
||||
if len(cancelled) != 2 {
|
||||
t.Errorf("cancelled = %d, want 2", len(cancelled))
|
||||
}
|
||||
checkString(t, "q1", string(mustGetRun(t, repo, "q1").Status), "cancelled")
|
||||
checkString(t, "c1", string(mustGetRun(t, repo, "c1").Status), "cancelled")
|
||||
@@ -101,8 +114,71 @@ func TestCancelRunsWhere_ReportsRowsActuallyCancelled(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("cancel where (second pass): %v", err)
|
||||
}
|
||||
if cancelled != 0 {
|
||||
t.Errorf("second pass cancelled = %d, want 0", cancelled)
|
||||
if len(cancelled) != 0 {
|
||||
t.Errorf("second pass cancelled = %d, want 0", len(cancelled))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCancelRunsWhere_ReturnedRowsCarryClassificationFields pins the
|
||||
// fields a caller needs to classify each cancelled run's loop-liveness
|
||||
// terminal shape (office_loop_terminal_total) without a second read: the
|
||||
// launched run's session_id must survive onto the returned row, distinct
|
||||
// from the never-launched run's empty one.
|
||||
func TestCancelRunsWhere_ReturnedRowsCarryClassificationFields(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
ctx := context.Background()
|
||||
requestedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
||||
|
||||
launched := mustCreateRun(t, repo, &models.Run{
|
||||
ID: "launched", AgentProfileID: "a1", Reason: "task_assigned",
|
||||
Payload: `{}`, Status: "claimed",
|
||||
})
|
||||
if err := repo.SetRunRequestedAtForTest(ctx, launched.ID, requestedAt); err != nil {
|
||||
t.Fatalf("backdate requested_at: %v", err)
|
||||
}
|
||||
if _, err := repo.SetRunSessionID(ctx, launched.ID, "session-in-flight"); err != nil {
|
||||
t.Fatalf("set session id: %v", err)
|
||||
}
|
||||
neverLaunched := mustCreateRun(t, repo, &models.Run{
|
||||
ID: "never-launched", AgentProfileID: "a2", Reason: "task_assigned",
|
||||
Payload: `{}`, Status: "queued",
|
||||
})
|
||||
if err := repo.SetRunRequestedAtForTest(ctx, neverLaunched.ID, requestedAt); err != nil {
|
||||
t.Fatalf("backdate requested_at: %v", err)
|
||||
}
|
||||
|
||||
cancelled, err := repo.CancelRunsWhere(ctx, "reason",
|
||||
`id IN (?, ?)`, launched.ID, neverLaunched.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("cancel where: %v", err)
|
||||
}
|
||||
if len(cancelled) != 2 {
|
||||
t.Fatalf("cancelled = %d, want 2", len(cancelled))
|
||||
}
|
||||
|
||||
byID := map[string]runssqlite.CancelledRun{}
|
||||
for _, row := range cancelled {
|
||||
byID[row.ID] = row
|
||||
}
|
||||
got, ok := byID[launched.ID]
|
||||
if !ok {
|
||||
t.Fatalf("launched run missing from cancelled rows: %v", cancelled)
|
||||
}
|
||||
if got.AgentProfileID != "a1" {
|
||||
t.Errorf("launched agent_profile_id = %q, want a1", got.AgentProfileID)
|
||||
}
|
||||
if got.SessionID != "session-in-flight" {
|
||||
t.Errorf("launched session_id = %q, want session-in-flight", got.SessionID)
|
||||
}
|
||||
if !got.RequestedAt.Equal(requestedAt) {
|
||||
t.Errorf("launched requested_at = %v, want %v", got.RequestedAt, requestedAt)
|
||||
}
|
||||
got, ok = byID[neverLaunched.ID]
|
||||
if !ok {
|
||||
t.Fatalf("never-launched run missing from cancelled rows: %v", cancelled)
|
||||
}
|
||||
if got.SessionID != "" {
|
||||
t.Errorf("never-launched session_id = %q, want empty", got.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,8 +202,8 @@ func TestCancelRunsWhere_MultiArgSelectorBindsInOrder(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("cancel where: %v", err)
|
||||
}
|
||||
if cancelled != 1 {
|
||||
t.Errorf("cancelled = %d, want 1", cancelled)
|
||||
if len(cancelled) != 1 {
|
||||
t.Errorf("cancelled = %d, want 1", len(cancelled))
|
||||
}
|
||||
checkString(t, "match", string(mustGetRun(t, repo, "match").Status), "cancelled")
|
||||
checkString(t, "wrong-reason", string(mustGetRun(t, repo, "wrong-reason").Status), "queued")
|
||||
@@ -148,12 +224,15 @@ func TestBulkCancelRuns_CancelsOnlyTheListedEligibleRuns(t *testing.T) {
|
||||
setStatus(t, repo, finished.ID, "finished", nil, timePtr(now))
|
||||
queueRunAt(t, repo, "out-of-set", "a1", now)
|
||||
|
||||
err := repo.BulkCancelRuns(ctx,
|
||||
cancelled, err := repo.BulkCancelRuns(ctx,
|
||||
[]string{"in-set-queued", "in-set-claimed", "in-set-finished", "missing-id"},
|
||||
"workspace deleted")
|
||||
if err != nil {
|
||||
t.Fatalf("bulk cancel: %v", err)
|
||||
}
|
||||
if len(cancelled) != 2 {
|
||||
t.Errorf("cancelled = %d, want 2 (queued + claimed, not the finished run)", len(cancelled))
|
||||
}
|
||||
|
||||
checkString(t, "in-set-queued", string(mustGetRun(t, repo, "in-set-queued").Status), "cancelled")
|
||||
checkString(t, "in-set-claimed", string(mustGetRun(t, repo, "in-set-claimed").Status), "cancelled")
|
||||
@@ -174,9 +253,13 @@ func TestBulkCancelRuns_EmptyIDListIsANoOp(t *testing.T) {
|
||||
queueRunAt(t, repo, "survivor", "a1", time.Now().UTC())
|
||||
|
||||
for _, ids := range [][]string{nil, {}} {
|
||||
if err := repo.BulkCancelRuns(ctx, ids, "nothing"); err != nil {
|
||||
cancelled, err := repo.BulkCancelRuns(ctx, ids, "nothing")
|
||||
if err != nil {
|
||||
t.Fatalf("bulk cancel %v: %v", ids, err)
|
||||
}
|
||||
if len(cancelled) != 0 {
|
||||
t.Errorf("bulk cancel %v: cancelled = %d, want 0", ids, len(cancelled))
|
||||
}
|
||||
}
|
||||
got := mustGetRun(t, repo, "survivor")
|
||||
checkString(t, "status", string(got.Status), "queued")
|
||||
|
||||
@@ -116,7 +116,8 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) {
|
||||
return repo.UpdateRunOutputSummary(ctx, run.ID, "o", "f")
|
||||
},
|
||||
"FinishRun": func() error {
|
||||
return repo.FinishRun(ctx, run.ID, "finished", nil)
|
||||
_, err := repo.FinishRun(ctx, run.ID, "finished", nil)
|
||||
return err
|
||||
},
|
||||
"ClaimRun": func() error {
|
||||
_, err := repo.ClaimRun(ctx, "a1")
|
||||
@@ -146,10 +147,12 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) {
|
||||
return err
|
||||
},
|
||||
"CancelRun": func() error {
|
||||
return repo.CancelRun(ctx, run.ID, "reason")
|
||||
_, err := repo.CancelRun(ctx, run.ID, "reason")
|
||||
return err
|
||||
},
|
||||
"BulkCancelRuns": func() error {
|
||||
return repo.BulkCancelRuns(ctx, []string{run.ID}, "reason")
|
||||
_, err := repo.BulkCancelRuns(ctx, []string{run.ID}, "reason")
|
||||
return err
|
||||
},
|
||||
"AppendRunEvent": func() error {
|
||||
_, err := repo.AppendRunEvent(ctx, run.ID, "run.progress", "info", "{}")
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
|
||||
taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
// TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun is the
|
||||
// two-connection regression for Review round 3, R3-1: FinishRun's write
|
||||
// used to carry no status guard at all, so a run cancelled on another
|
||||
// connection between the caller's read and this write would have its
|
||||
// 'cancelled' status and real finished_at silently overwritten to
|
||||
// 'finished'/'failed'. Real Postgres row locking forces the interleaving
|
||||
// this needs: one connection holds an uncommitted cancel on the row,
|
||||
// FinishRun's write blocks on that row lock, and only proceeds once the
|
||||
// cancel commits — at which point the guard (status = 'claimed') must see
|
||||
// the fresh 'cancelled' status and write nothing. Skips unless
|
||||
// KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresFinishRun_DoesNotOverwriteAConcurrentlyCancelledRun(t *testing.T) {
|
||||
dsn := testutil.PostgresDSNFromEnv(t)
|
||||
db := testutil.OpenIsolatedPostgres(t, dsn)
|
||||
|
||||
if _, err := taskrepo.NewWithDB(db, db, nil); err != nil {
|
||||
t.Fatalf("init task repo: %v", err)
|
||||
}
|
||||
officeRepo, err := officesqlite.NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init office repo: %v", err)
|
||||
}
|
||||
repo := officeRepo.RunsRepository()
|
||||
ctx := context.Background()
|
||||
|
||||
var schema string
|
||||
if err := db.Get(&schema, "SELECT current_schema()"); err != nil {
|
||||
t.Fatalf("read isolated schema name: %v", err)
|
||||
}
|
||||
cancellerDB, err := sqlx.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open canceller connection: %v", err)
|
||||
}
|
||||
cancellerDB.SetMaxOpenConns(1)
|
||||
cancellerDB.SetMaxIdleConns(1)
|
||||
t.Cleanup(func() { _ = cancellerDB.Close() })
|
||||
if _, err := cancellerDB.Exec("SET search_path TO " + schema); err != nil {
|
||||
t.Fatalf("set canceller search_path: %v", err)
|
||||
}
|
||||
|
||||
run := &models.Run{AgentProfileID: "a1", Reason: "task_assigned", Payload: "{}", Status: "queued"}
|
||||
if err := repo.CreateRun(ctx, run); err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if _, err := db.ExecContext(ctx, db.Rebind(
|
||||
`UPDATE runs SET status = 'claimed', claimed_at = ? WHERE id = ?`,
|
||||
), now, run.ID); err != nil {
|
||||
t.Fatalf("seed claimed: %v", err)
|
||||
}
|
||||
|
||||
tx, err := cancellerDB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin canceller tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
if _, err := tx.ExecContext(ctx, cancellerDB.Rebind(
|
||||
`UPDATE runs SET status = 'cancelled', cancel_reason = ?, finished_at = ? WHERE id = ?`,
|
||||
), "too late", now, run.ID); err != nil {
|
||||
t.Fatalf("canceller update (holds the row lock, uncommitted): %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
finishedRun *models.Run
|
||||
finishErr error
|
||||
finishDone = make(chan struct{})
|
||||
commitErr error
|
||||
)
|
||||
outcome := "processed"
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer close(finishDone)
|
||||
finishedRun, finishErr = repo.FinishRun(ctx, run.ID, "finished", &outcome)
|
||||
}()
|
||||
|
||||
// Give FinishRun's write time to reach Postgres and block on the row
|
||||
// lock the canceller's uncommitted transaction holds, before that
|
||||
// transaction commits. Without this window the two goroutines could
|
||||
// run in either order and the test would not exercise the lock-wait
|
||||
// path this bug depends on.
|
||||
select {
|
||||
case <-finishDone:
|
||||
t.Fatal("FinishRun returned before the canceller committed; it never blocked on the row lock")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
commitErr = tx.Commit()
|
||||
wg.Wait()
|
||||
if commitErr != nil {
|
||||
t.Fatalf("canceller commit: %v", commitErr)
|
||||
}
|
||||
if finishErr != nil {
|
||||
t.Fatalf("finish run: %v", finishErr)
|
||||
}
|
||||
|
||||
if finishedRun != nil {
|
||||
t.Fatalf("finished run = %+v, want nil: a run that committed cancelled must not be reported finished", finishedRun)
|
||||
}
|
||||
if got := mustGetRunStatus(t, db, run.ID); got != "cancelled" {
|
||||
t.Fatalf("run status = %q, want cancelled (the concurrent finish must not have overwritten it)", got)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -41,13 +42,13 @@ func (r *Repository) CreateRunTx(ctx context.Context, tx *sqlx.Tx, req *models.R
|
||||
id, agent_profile_id, reason, payload, status, coalesced_count,
|
||||
idempotency_key, context_snapshot, capabilities, input_snapshot,
|
||||
output_summary, failure_reason, session_id, retry_count, scheduled_retry_at,
|
||||
requested_at, error_message, cancel_reason, continuation_scope
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
requested_at, error_message, cancel_reason, continuation_scope, causation_id
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`), req.ID, req.AgentProfileID, req.Reason, req.Payload, req.Status,
|
||||
req.CoalescedCount, req.IdempotencyKey, req.ContextSnapshot,
|
||||
req.Capabilities, req.InputSnapshot, req.OutputSummary, req.FailureReason,
|
||||
req.SessionID, req.RetryCount, req.ScheduledRetryAt, req.RequestedAt,
|
||||
req.ErrorMessage, req.CancelReason, req.ContinuationScope)
|
||||
req.ErrorMessage, req.CancelReason, req.ContinuationScope, req.CausationID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -103,6 +104,35 @@ func (r *Repository) UpdateRunRuntimeSnapshot(
|
||||
return err
|
||||
}
|
||||
|
||||
// SetRunSessionID persists the session id a launch produced. A no-op
|
||||
// when sessionID is empty — the caller counts that as a without-session
|
||||
// launch rather than clobbering whatever the column already held.
|
||||
// Guarded to status = 'claimed' so a launch that loses a race against a
|
||||
// concurrent cancel or terminal write cannot mutate an already-terminal
|
||||
// row's session id after the fact: without the guard, a terminal-shape
|
||||
// classification already recorded off the empty session id would drift
|
||||
// from what the row shows on a later read. Returns whether the row was
|
||||
// still claimed so the caller can distinguish a real write from a
|
||||
// stale run id or a lost race.
|
||||
func (r *Repository) SetRunSessionID(
|
||||
ctx context.Context, runID, sessionID string,
|
||||
) (bool, error) {
|
||||
if sessionID == "" {
|
||||
return false, nil
|
||||
}
|
||||
res, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE runs SET session_id = ? WHERE id = ? AND status = 'claimed'
|
||||
`), sessionID, runID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// UpdateRunPromptArtifacts persists the assembled prompt the agent
|
||||
// received and the continuation-summary content prepended at dispatch.
|
||||
// Called from the scheduler-integration after BuildAgentPrompt completes
|
||||
@@ -189,12 +219,32 @@ func (r *Repository) ClaimRun(ctx context.Context, agentInstanceID string) (*mod
|
||||
// outcome from a different transition. outcome is nil for the failed path
|
||||
// and for callers with no established semantic label (docs/specs/
|
||||
// task-delivery-ledger/spec.md, "Office run outcome").
|
||||
func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) error {
|
||||
// FinishRun writes the terminal status/outcome and returns the row as it
|
||||
// stands immediately after that write, via the same statement (RETURNING),
|
||||
// so a caller classifying the transition (office_loop_terminal_total) never
|
||||
// depends on a separate read succeeding independently of the write that
|
||||
// persisted it. Guarded to status = 'claimed', the same status every caller
|
||||
// reaches this from (ClaimNextEligibleRun then processRun/an event
|
||||
// subscriber): without the guard, a cancel that commits between the
|
||||
// caller's read and this write would have its 'cancelled' status and
|
||||
// finished_at overwritten by this transition. Returns (nil, nil) for an
|
||||
// unknown id or a run no longer claimed (already terminal by another
|
||||
// writer): zero rows changed, so there is nothing to classify.
|
||||
func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) (*models.Run, error) {
|
||||
now := time.Now().UTC()
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE runs SET status = ?, outcome = ?, finished_at = ? WHERE id = ?
|
||||
`), status, outcome, now, id)
|
||||
return err
|
||||
var run models.Run
|
||||
err := r.db.QueryRowxContext(ctx, r.db.Rebind(`
|
||||
UPDATE runs SET status = ?, outcome = ?, finished_at = ?
|
||||
WHERE id = ? AND status = 'claimed'
|
||||
RETURNING *
|
||||
`), status, outcome, now, id).StructScan(&run)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &run, nil
|
||||
}
|
||||
|
||||
// GetRunByID returns the run row for a given ID. Returns sql.ErrNoRows when unknown.
|
||||
|
||||
@@ -275,7 +275,7 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) {
|
||||
before := time.Now().UTC().Add(-time.Second)
|
||||
run := mustCreateRun(t, repo, fullRun())
|
||||
|
||||
if err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil {
|
||||
if _, err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil {
|
||||
t.Fatalf("finish run: %v", err)
|
||||
}
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
@@ -288,6 +288,35 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinishRun_ReturnsFullRowWithTheReaderClosed pins the property a
|
||||
// terminal-shape classifier depends on: FinishRun's own RETURNING
|
||||
// clause carries every field needed to classify the transition (here,
|
||||
// SessionID), so a caller never needs a separate read through r.ro to
|
||||
// get it.
|
||||
func TestFinishRun_ReturnsFullRowWithTheReaderClosed(t *testing.T) {
|
||||
repo, _, reader := newTestRepoWithHandles(t)
|
||||
ctx := context.Background()
|
||||
run := mustCreateRun(t, repo, fullRun())
|
||||
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Fatalf("close reader: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.FinishRun(ctx, run.ID, "finished", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("finish run with the reader closed: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("finish run returned a nil row")
|
||||
}
|
||||
checkString(t, "status", string(got.Status), "finished")
|
||||
checkString(t, "session_id", got.SessionID, "sess-7")
|
||||
checkString(t, "agent_profile_id", got.AgentProfileID, "agent-42")
|
||||
if got.FinishedAt == nil {
|
||||
t.Fatalf("finished_at = nil, want a stamp")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSetRunStatusForTest_WritesStatusAndBothTimestamps covers the
|
||||
// harness affordance used by the E2E seeder, including its NULL case.
|
||||
func TestSetRunStatusForTest_WritesStatusAndBothTimestamps(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package sqlite_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/office/models"
|
||||
)
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.7/.8/.10: SetRunSessionID persists the
|
||||
// session id a launch produced, is a no-op on an empty id (so a caller
|
||||
// can distinguish "wrote nothing" from a real write), and — given the
|
||||
// single-launch-in-flight invariant — an unconditional non-empty write
|
||||
// is "last non-empty wins".
|
||||
func TestSetRunSessionID_PersistsNonEmptyID(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
run := &models.Run{ID: "run-session-1", AgentProfileID: "agent-1", Status: "claimed"}
|
||||
mustCreateRun(t, repo, run)
|
||||
|
||||
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-abc")
|
||||
if err != nil {
|
||||
t.Fatalf("SetRunSessionID: %v", err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("expected wrote=true for existing run with non-empty session id")
|
||||
}
|
||||
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
if got.SessionID != "sess-abc" {
|
||||
t.Fatalf("session_id = %q, want sess-abc", got.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRunSessionID_EmptyIDIsNoop(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
run := &models.Run{ID: "run-session-2", AgentProfileID: "agent-1", Status: "claimed", SessionID: "sess-existing"}
|
||||
mustCreateRun(t, repo, run)
|
||||
|
||||
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("SetRunSessionID: %v", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Fatal("expected wrote=false for empty session id")
|
||||
}
|
||||
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
if got.SessionID != "sess-existing" {
|
||||
t.Fatalf("session_id = %q, want unchanged sess-existing", got.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRunSessionID_UnknownRunIsNoop(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
|
||||
wrote, err := repo.SetRunSessionID(context.Background(), "run-does-not-exist", "sess-x")
|
||||
if err != nil {
|
||||
t.Fatalf("SetRunSessionID: %v", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Fatal("expected wrote=false for unknown run id")
|
||||
}
|
||||
}
|
||||
|
||||
// A cancel that lands between claim and this write already recorded a
|
||||
// terminal shape off the run's session id at that instant; letting this
|
||||
// write land anyway would make the persisted row disagree with the
|
||||
// classification that was already counted for it.
|
||||
func TestSetRunSessionID_LostRaceAgainstConcurrentCancelIsNoop(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
run := &models.Run{ID: "run-session-4", AgentProfileID: "agent-1", Status: "claimed"}
|
||||
mustCreateRun(t, repo, run)
|
||||
|
||||
cancelled, err := repo.CancelRun(context.Background(), run.ID, "lost_race")
|
||||
if err != nil {
|
||||
t.Fatalf("CancelRun: %v", err)
|
||||
}
|
||||
if !cancelled {
|
||||
t.Fatal("expected CancelRun to cancel the still-claimed run")
|
||||
}
|
||||
|
||||
wrote, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-late")
|
||||
if err != nil {
|
||||
t.Fatalf("SetRunSessionID: %v", err)
|
||||
}
|
||||
if wrote {
|
||||
t.Fatal("expected wrote=false once the run is no longer claimed")
|
||||
}
|
||||
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
if got.SessionID != "" {
|
||||
t.Fatalf("session_id = %q, want unchanged empty string on the cancelled row", got.SessionID)
|
||||
}
|
||||
if got.Status != "cancelled" {
|
||||
t.Fatalf("status = %q, want cancelled", got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// AC-OFFICE-LOOP-LIVENESS-002.10: a later, non-empty write overwrites an
|
||||
// earlier one — "last non-empty wins" for the relaunch-keeps-latest case.
|
||||
func TestSetRunSessionID_LaterWriteOverwritesEarlier(t *testing.T) {
|
||||
repo := newTestRepo(t)
|
||||
run := &models.Run{ID: "run-session-3", AgentProfileID: "agent-1", Status: "claimed"}
|
||||
mustCreateRun(t, repo, run)
|
||||
|
||||
if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-first"); err != nil {
|
||||
t.Fatalf("first SetRunSessionID: %v", err)
|
||||
}
|
||||
if _, err := repo.SetRunSessionID(context.Background(), run.ID, "sess-second"); err != nil {
|
||||
t.Fatalf("second SetRunSessionID: %v", err)
|
||||
}
|
||||
|
||||
got := mustGetRun(t, repo, run.ID)
|
||||
if got.SessionID != "sess-second" {
|
||||
t.Fatalf("session_id = %q, want sess-second", got.SessionID)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user