From f8fc3bf506acf258953e548595545232e76a438b Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 16 Sep 2026 03:22:05 +0800 Subject: [PATCH] feat(orchestrator): add instance-wide agent session concurrency ceiling (#3687) Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio --- apps/backend/internal/backendapp/helpers.go | 3 + apps/backend/internal/backendapp/main.go | 13 + .../backend/internal/common/config/catalog.go | 1 + .../internal/common/config/catalog_test.go | 1 + .../backend/internal/mcp/handlers/handlers.go | 82 +- .../office/scheduler/dispatch_routing.go | 39 + .../office/service/ceiling_credentials.go | 39 + .../office/service/scheduler_integration.go | 46 + .../internal/orchestrator/ceiling_defer.go | 153 +++ .../orchestrator/ceiling_defer_test.go | 174 ++++ .../ceiling_dynamic_launch_release_test.go | 91 ++ .../orchestrator/ceiling_manual_override.go | 96 ++ .../ceiling_manual_override_test.go | 115 +++ .../internal/orchestrator/ceiling_release.go | 88 ++ .../orchestrator/ceiling_release_test.go | 125 +++ .../internal/orchestrator/ceiling_replay.go | 550 ++++++++++ .../orchestrator/ceiling_replay_test.go | 332 ++++++ .../ceiling_seam1_integration_test.go | 121 +++ .../internal/orchestrator/ceiling_seam2.go | 58 ++ .../ceiling_seam2_integration_test.go | 130 +++ .../orchestrator/ceiling_seam2_test.go | 160 +++ .../internal/orchestrator/ceiling_seam3.go | 231 +++++ .../ceiling_seam3_integration_test.go | 312 ++++++ .../orchestrator/ceiling_seam3_test.go | 285 ++++++ .../internal/orchestrator/ceiling_seam4.go | 35 + .../ceiling_seam4_integration_test.go | 173 ++++ .../orchestrator/ceiling_seam4_test.go | 144 +++ .../internal/orchestrator/ceiling_seam5.go | 63 ++ .../ceiling_seam5_integration_test.go | 198 ++++ .../orchestrator/ceiling_seam5_test.go | 160 +++ .../internal/orchestrator/ceiling_seams.go | 212 ++++ .../orchestrator/ceiling_seams_test.go | 223 ++++ .../ceiling_session_keyed_reservation.go | 93 ++ .../ceiling_streaming_release_test.go | 72 ++ .../internal/orchestrator/ceiling_surface.go | 139 +++ .../orchestrator/ceiling_surface_test.go | 155 +++ .../internal/orchestrator/ceiling_sweep.go | 159 +++ .../orchestrator/ceiling_sweep_test.go | 117 +++ .../orchestrator/ci_automation_dispatch.go | 2 +- .../clarification_turn_authority_test.go | 1 + .../orchestrator/deferred_launch_consume.go | 40 +- .../deferred_launch_consume_ceiling_test.go | 149 +++ ...unch_orchestrator_write_protection_test.go | 82 ++ .../internal/orchestrator/dynamic_launch.go | 170 +++- .../dynamic_launch_identity_test.go | 2 +- .../orchestrator/dynamic_launch_test.go | 2 + .../orchestrator/dynamic_policy_recovery.go | 2 +- ...ensure_session_running_cold_resume_test.go | 36 +- .../orchestrator/event_handlers_agent.go | 4 +- .../orchestrator/event_handlers_automation.go | 2 +- .../event_handlers_children_completed.go | 2 +- .../event_handlers_clarification.go | 3 +- .../orchestrator/event_handlers_github.go | 13 + .../event_handlers_queue_round64_test.go | 31 + .../orchestrator/event_handlers_streaming.go | 23 + .../orchestrator/event_handlers_transient.go | 25 +- .../orchestrator/event_handlers_workflow.go | 125 ++- .../event_handlers_workflow_ceiling_test.go | 122 +++ ...lers_workflow_deferred_launch_race_test.go | 113 +++ .../executor/ceiling_bypass_audit.go | 48 + .../executor/ceiling_bypass_audit_test.go | 163 +++ .../executor/ceiling_release_callback_test.go | 59 ++ .../orchestrator/executor/executor.go | 33 + .../orchestrator/executor/executor_execute.go | 6 + .../executor/executor_mocks_test.go | 3 + .../orchestrator/executor/executor_resume.go | 19 + .../orchestrator/executor/executor_state.go | 7 + .../orchestrator/foreground_claim_test.go | 2 +- .../prompt_dispatch_identity_test.go | 2 +- .../orchestrator/prompt_idle_session_test.go | 12 +- .../internal/orchestrator/queue_send_now.go | 2 +- apps/backend/internal/orchestrator/service.go | 42 + .../internal/orchestrator/session_ceiling.go | 466 +++++++++ .../orchestrator/session_ceiling_config.go | 92 ++ .../session_ceiling_config_test.go | 128 +++ .../session_ceiling_lifecycle_test.go | 226 +++++ .../session_ceiling_repo_binding_test.go | 21 + .../orchestrator/session_ceiling_test.go | 344 +++++++ .../session_ceiling_wiring_test.go | 57 ++ .../internal/orchestrator/session_ensure.go | 38 +- .../internal/orchestrator/session_launch.go | 10 + .../internal/orchestrator/task_operations.go | 956 +++++++++++++----- ...ask_operations_resume_cancellation_test.go | 2 + .../orchestrator/task_operations_test.go | 8 +- .../internal/orchestrator/watcher_dispatch.go | 6 + .../internal/orchestrator/workflow_store.go | 4 +- .../internal/plugins/host_data_mappers.go | 2 +- .../plugins/host_data_task_fields_test.go | 27 + .../task/handlers/process_handlers_test.go | 11 + .../task_http_handlers_external_id_test.go | 4 + .../handlers/task_port_forwarding_test.go | 2 +- .../internal/task/models/canonical_json.go | 44 + .../task/models/canonical_json_test.go | 75 ++ .../task/models/ceiling_equivalence.go | 98 ++ .../task/models/ceiling_equivalence_test.go | 124 +++ .../internal/task/models/ceiling_launch.go | 87 ++ .../task/models/ceiling_launch_test.go | 117 +++ .../internal/task/models/ceiling_record.go | 198 ++++ .../task/models/ceiling_record_test.go | 208 ++++ apps/backend/internal/task/models/models.go | 2 +- .../internal/task/models/public_metadata.go | 7 + .../task/models/session_state_sets.go | 27 + .../task/models/session_state_sets_test.go | 66 +- .../internal/task/repository/interface.go | 23 + .../repository/sqlite/deferred_launch_cas.go | 292 ++++++ .../sqlite/deferred_launch_cas_test.go | 344 +++++++ .../sqlite/deferred_launch_wip_test.go | 205 ++++ .../task/repository/sqlite/session_ceiling.go | 48 + .../repository/sqlite/session_ceiling_test.go | 274 +++++ .../step_transition_writers_pin_test.go | 2 +- .../internal/task/repository/sqlite/task.go | 267 +++-- ...deferred_launch_protected_postgres_test.go | 69 ++ .../task_deferred_launch_protected_test.go | 219 ++++ .../internal/task/service/handoff_cascade.go | 12 + .../internal/task/service/handoff_service.go | 48 +- .../task/service/handoff_workspace_test.go | 14 + apps/backend/internal/task/service/service.go | 16 + .../task/service/service_deferred_launch.go | 101 +- .../service/service_deferred_launch_test.go | 96 ++ .../service/service_task_priority_test.go | 5 + .../internal/task/service/service_tasks.go | 15 +- .../internal/task/service/service_workflow.go | 16 +- ...vice_workflow_deferred_launch_race_test.go | 136 +++ .../service_workflow_event_source_test.go | 11 +- .../plans/session-concurrency-ceiling/plan.md | 49 + .../task-01-session-concurrency-ceiling.md | 80 ++ .../session-concurrency-ceiling.md | 65 ++ .../session-concurrency-ceiling.md | 98 ++ 128 files changed, 11934 insertions(+), 563 deletions(-) create mode 100644 apps/backend/internal/office/service/ceiling_credentials.go create mode 100644 apps/backend/internal/orchestrator/ceiling_defer.go create mode 100644 apps/backend/internal/orchestrator/ceiling_defer_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_dynamic_launch_release_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_manual_override.go create mode 100644 apps/backend/internal/orchestrator/ceiling_manual_override_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_release.go create mode 100644 apps/backend/internal/orchestrator/ceiling_release_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_replay.go create mode 100644 apps/backend/internal/orchestrator/ceiling_replay_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam1_integration_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam2.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam2_integration_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam2_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam3.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam3_integration_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam3_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam4.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam4_integration_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam4_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam5.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam5_integration_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seam5_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seams.go create mode 100644 apps/backend/internal/orchestrator/ceiling_seams_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_session_keyed_reservation.go create mode 100644 apps/backend/internal/orchestrator/ceiling_streaming_release_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_surface.go create mode 100644 apps/backend/internal/orchestrator/ceiling_surface_test.go create mode 100644 apps/backend/internal/orchestrator/ceiling_sweep.go create mode 100644 apps/backend/internal/orchestrator/ceiling_sweep_test.go create mode 100644 apps/backend/internal/orchestrator/deferred_launch_consume_ceiling_test.go create mode 100644 apps/backend/internal/orchestrator/deferred_launch_orchestrator_write_protection_test.go create mode 100644 apps/backend/internal/orchestrator/event_handlers_workflow_ceiling_test.go create mode 100644 apps/backend/internal/orchestrator/event_handlers_workflow_deferred_launch_race_test.go create mode 100644 apps/backend/internal/orchestrator/executor/ceiling_bypass_audit.go create mode 100644 apps/backend/internal/orchestrator/executor/ceiling_bypass_audit_test.go create mode 100644 apps/backend/internal/orchestrator/executor/ceiling_release_callback_test.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_config.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_config_test.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_lifecycle_test.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_repo_binding_test.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_test.go create mode 100644 apps/backend/internal/orchestrator/session_ceiling_wiring_test.go create mode 100644 apps/backend/internal/task/models/canonical_json.go create mode 100644 apps/backend/internal/task/models/canonical_json_test.go create mode 100644 apps/backend/internal/task/models/ceiling_equivalence.go create mode 100644 apps/backend/internal/task/models/ceiling_equivalence_test.go create mode 100644 apps/backend/internal/task/models/ceiling_launch.go create mode 100644 apps/backend/internal/task/models/ceiling_launch_test.go create mode 100644 apps/backend/internal/task/models/ceiling_record.go create mode 100644 apps/backend/internal/task/models/ceiling_record_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/deferred_launch_cas.go create mode 100644 apps/backend/internal/task/repository/sqlite/deferred_launch_cas_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/deferred_launch_wip_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/session_ceiling.go create mode 100644 apps/backend/internal/task/repository/sqlite/session_ceiling_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_postgres_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_test.go create mode 100644 apps/backend/internal/task/service/service_workflow_deferred_launch_race_test.go create mode 100644 docs/plans/session-concurrency-ceiling/plan.md create mode 100644 docs/plans/session-concurrency-ceiling/task-01-session-concurrency-ceiling.md create mode 100644 docs/specs/agents/requirements/session-concurrency-ceiling.md create mode 100644 docs/specs/agents/system-design/session-concurrency-ceiling.md diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 482e45c935..c5532e0a0a 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -752,6 +752,7 @@ func registerRoutes(p routeParams) { // clear that session's parked-projection tracking (spec: // docs/specs/disambiguate-waiting). p.taskSvc.SetParkedProjectionCanceller(p.orchestratorSvc) + p.taskSvc.SetSessionCeilingReleaser(p.orchestratorSvc) // Single resolver instance shared by the REST clarification routes and the // external answer_question_kandev/list_pending_questions_kandev MCP tools // (R3: both entry points must race through the same claim). @@ -796,6 +797,7 @@ func registerRoutes(p routeParams) { } handoffSvc.SetRunCanceller(p.orchestratorSvc) handoffSvc.SetGitArchiveCapture(p.orchestratorSvc) + handoffSvc.SetSessionCeilingReleaser(p.orchestratorSvc) // Cascade archive/delete must re-publish task.updated / task.deleted // events; HandoffService walks the repo directly and bypasses the // Service wrappers that normally publish these. Without this wiring @@ -1997,6 +1999,7 @@ func registerMCPAndDebugRoutes( )) } mcpHandlers.SetClarificationInputPauser(p.orchestratorSvc) + mcpHandlers.SetSessionCeilingReleaser(p.orchestratorSvc) mcpHandlers.SetPromptReferenceResolver(p.services.Prompts) mcpHandlers.SetPromptReader(p.services.Prompts) mcpHandlers.SetTaskStopper(p.orchestratorSvc) diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 07f79a6570..2e7a4bc8f6 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -1691,6 +1691,9 @@ func (a *schedulerTaskStarterAdapter) StartTaskWithRouteReturningSession( route officescheduler.RouteOverride, ) (string, error) { execution, err := a.startTaskWithRoute(ctx, taskID, agentProfileID, launch, route) + if errors.Is(err, orchestrator.ErrCeilingLaunchDeferred) { + return "", officeservice.ErrLaunchDeferredByCapacity + } if err != nil || execution == nil { return "", err } @@ -1742,6 +1745,10 @@ func startSchedulingRuntime( orchScheduler := officeservice.NewSchedulerIntegration( runProcessorSvc, tickInterval, ) + // A ceiling-deferred Office launch is replayed by the orchestrator's own + // sweep, independent of this scheduler; it needs a fresh runtime JWT + // rather than the one captured (and redacted) at defer time. + orchestratorSvc.SetCeilingLaunchCredentialReminter(orchScheduler) // Office task-handoffs prompt enrichment. The HandoffService is // constructed alongside the HTTP routes (helpers.go); we stash the // scheduler reference on the Services struct so registerRoutes can @@ -2243,6 +2250,9 @@ func (a *officeOrchestratorTaskStarter) StartTaskWithEnvReturningSession( ) (string, error) { execution, err := a.startTaskWithEnvAndSkills(ctx, taskID, agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, attachments, env, nil) + if errors.Is(err, orchestrator.ErrCeilingLaunchDeferred) { + return "", officeservice.ErrLaunchDeferredByCapacity + } if err != nil || execution == nil { return "", err } @@ -2269,6 +2279,9 @@ func (a *officeOrchestratorTaskStarter) StartTaskWithLaunchContextReturningSessi launch.ExecutorID, launch.ExecutorProfileID, launch.Priority, launch.Prompt, launch.WorkflowStepID, launch.PlanMode, launch.Attachments, launch.Env, launch.AdditionalSkillSlugs) + if errors.Is(err, orchestrator.ErrCeilingLaunchDeferred) { + return "", officeservice.ErrLaunchDeferredByCapacity + } if err != nil || execution == nil { return "", err } diff --git a/apps/backend/internal/common/config/catalog.go b/apps/backend/internal/common/config/catalog.go index a54a146a70..fb32b7edef 100644 --- a/apps/backend/internal/common/config/catalog.go +++ b/apps/backend/internal/common/config/catalog.go @@ -146,6 +146,7 @@ var startupExclusions = []CatalogExclusion{ {EnvVar: "KANDEV_MCP_LOG_FILE", Class: "debug", Reason: "agentctl MCP debug logging"}, {EnvVar: "KANDEV_DEBUG_LOG_DIR", Class: "debug", Reason: "ACP debug logging directory"}, {EnvVar: "AGENTCTL_AUTO_APPROVE_PERMISSIONS", Class: "test", Reason: "profile-selected E2E behavior"}, + {EnvVar: "KANDEV_MAX_CONCURRENT_SESSIONS", Class: "startup only", Reason: "instance session ceiling, environment-only, resolved once at startup with no YAML setting by contract"}, } // ConfigurationCatalog returns a defensive copy of the stable startup diff --git a/apps/backend/internal/common/config/catalog_test.go b/apps/backend/internal/common/config/catalog_test.go index e9b7aaaec5..f9087f859c 100644 --- a/apps/backend/internal/common/config/catalog_test.go +++ b/apps/backend/internal/common/config/catalog_test.go @@ -166,6 +166,7 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment { {envVar: "KANDEV_MCP_LOG_FILE", class: "exclusion"}, {envVar: "KANDEV_DEBUG_LOG_DIR", class: "exclusion"}, {envVar: "AGENTCTL_AUTO_APPROVE_PERMISSIONS", class: "exclusion"}, + {envVar: "KANDEV_MAX_CONCURRENT_SESSIONS", class: "exclusion"}, } } diff --git a/apps/backend/internal/mcp/handlers/handlers.go b/apps/backend/internal/mcp/handlers/handlers.go index 6d2635436d..dc7ce8e63c 100644 --- a/apps/backend/internal/mcp/handlers/handlers.go +++ b/apps/backend/internal/mcp/handlers/handlers.go @@ -83,6 +83,14 @@ type ClarificationInputPauser interface { PauseForClarificationInput(ctx context.Context, sessionID string) (int, error) } +// SessionCeilingReleaser releases the orchestrator's session-ceiling +// reservation for a session that this package just moved out of the counted +// population (STARTING/RUNNING) directly against the repository, bypassing +// the orchestrator's own persistence funnels. +type SessionCeilingReleaser interface { + ReleaseCeilingReservation(sessionID string) +} + type clarificationInputPauserWithOptions interface { PauseForClarificationInputWithOptions( ctx context.Context, @@ -254,28 +262,29 @@ type UserSettingsProvider interface { // Handlers provides MCP WebSocket handlers. type Handlers struct { - taskSvc *service.Service - workflowCtrl *workflowctrl.Controller - clarificationSvc ClarificationService - sessionCanceller SessionCanceller - inputPauser ClarificationInputPauser - messageCreator MessageCreator - sessionRepo SessionRepository - taskRepo TaskRepository - eventBus EventBus - planService *service.PlanService - walkthroughService *service.WalkthroughService - sessionLauncher SessionLauncher - taskStopper TaskStopper - titleBranchRenamer TaskTitleBranchRenamer - stopTaskGetter func(context.Context, string) (*models.Task, error) - messageQueue MessageQueuer - promptResolver PromptReferenceResolver - promptReader PromptReader - userSettingsProvider UserSettingsProvider - settingsRegistry *settingscatalog.Registry - settingsOperations SettingsOperations - logger *logger.Logger + taskSvc *service.Service + workflowCtrl *workflowctrl.Controller + clarificationSvc ClarificationService + sessionCanceller SessionCanceller + inputPauser ClarificationInputPauser + sessionCeilingReleaser SessionCeilingReleaser + messageCreator MessageCreator + sessionRepo SessionRepository + taskRepo TaskRepository + eventBus EventBus + planService *service.PlanService + walkthroughService *service.WalkthroughService + sessionLauncher SessionLauncher + taskStopper TaskStopper + titleBranchRenamer TaskTitleBranchRenamer + stopTaskGetter func(context.Context, string) (*models.Task, error) + messageQueue MessageQueuer + promptResolver PromptReferenceResolver + promptReader PromptReader + userSettingsProvider UserSettingsProvider + settingsRegistry *settingscatalog.Registry + settingsOperations SettingsOperations + logger *logger.Logger // Config-mode dependencies (optional, set via SetConfigDeps) workflowSvc *workflowsvc.Service @@ -384,6 +393,13 @@ func (h *Handlers) SetClarificationInputPauser(pauser ClarificationInputPauser) h.inputPauser = pauser } +// SetSessionCeilingReleaser wires the orchestrator-owned session-ceiling +// release used when this package's own clarification write moves a session +// out of the counted population. +func (h *Handlers) SetSessionCeilingReleaser(releaser SessionCeilingReleaser) { + h.sessionCeilingReleaser = releaser +} + func (h *Handlers) SetPromptReferenceResolver(resolver PromptReferenceResolver) { h.promptResolver = resolver } @@ -4258,14 +4274,34 @@ func (h *Handlers) updateClarificationSessionState( expected, state models.TaskSessionState, ) (bool, time.Time, error) { if updater, ok := h.sessionRepo.(conditionalSessionStateUpdater); ok { - return updater.UpdateTaskSessionStateIfCurrent(ctx, sessionID, expected, state, "") + changed, updatedAt, err := updater.UpdateTaskSessionStateIfCurrent(ctx, sessionID, expected, state, "") + if err == nil && changed { + h.releaseSessionCeilingIfLeftPopulation(sessionID, expected, state) + } + return changed, updatedAt, err } if err := h.sessionRepo.UpdateTaskSessionState(ctx, sessionID, state, ""); err != nil { return false, time.Time{}, err } + h.releaseSessionCeilingIfLeftPopulation(sessionID, expected, state) return true, time.Time{}, nil } +// releaseSessionCeilingIfLeftPopulation is this package's half of AC-51a's +// "mcp/handlers/handlers.go, various, SHALL release when leaving an AC-1 +// state" row: both the CAS-preferred branch and the raw fallback above count. +func (h *Handlers) releaseSessionCeilingIfLeftPopulation(sessionID string, priorState, nextState models.TaskSessionState) { + if h.sessionCeilingReleaser == nil { + return + } + leftPopulation := priorState == models.TaskSessionStateStarting || priorState == models.TaskSessionStateRunning + enteredPopulation := nextState == models.TaskSessionStateStarting || nextState == models.TaskSessionStateRunning + if !leftPopulation || enteredPopulation { + return + } + h.sessionCeilingReleaser.ReleaseCeilingReservation(sessionID) +} + func (h *Handlers) sessionUpdatedAtForStateEvent(ctx context.Context, sessionID string) (string, bool) { if session, err := h.sessionRepo.GetTaskSession(ctx, sessionID); err == nil && session != nil && !session.UpdatedAt.IsZero() { return session.UpdatedAt.UTC().Format(time.RFC3339Nano), true diff --git a/apps/backend/internal/office/scheduler/dispatch_routing.go b/apps/backend/internal/office/scheduler/dispatch_routing.go index 13fbcd459a..e0ec2aec42 100644 --- a/apps/backend/internal/office/scheduler/dispatch_routing.go +++ b/apps/backend/internal/office/scheduler/dispatch_routing.go @@ -42,6 +42,7 @@ const ( RouteAttemptOutcomeSkippedUserAction = "skipped_user_action" RouteAttemptOutcomeSkippedMissingMapping = "skipped_missing_mapping" RouteAttemptOutcomeMaxAttempts = "skipped_max_attempts" + RouteAttemptOutcomeDeferredCapacity = "deferred_capacity" ) // MaxAttemptsPerRun caps the number of route_attempt rows a single run @@ -273,6 +274,9 @@ func (ss *SchedulerService) tryCandidates( return false, nil, err } sessionID, launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, candidateLaunch) + if errors.Is(launchErr, service.ErrLaunchDeferredByCapacity) { + return ss.handleLaunchDeferred(ctx, run, agent.WorkspaceID, seq) + } if launchErr == nil { if prev != nil { // We walked past at least one prior candidate — that's @@ -472,6 +476,41 @@ func (ss *SchedulerService) handleLaunchSuccess( return true, nil, nil } +// handleLaunchDeferred is tryCandidates' disposition when the task starter +// reports ErrLaunchDeferredByCapacity: the orchestrator's own session +// ceiling already persisted a replay record for this exact launch and owns +// retrying it, so this attempt must not be counted as launched (no session +// was created) and this run must not be retried by the scheduler's own +// routing/park wake-up loop — a second automatic attempt here would race +// the ceiling's own replay into a double launch. Parking under +// blocked_provider_action_required (the same status parkRunMaxAttempts +// uses) keeps LiftParkedRuns from ever picking the run back up on its own; +// an operator notices via "Retry now" once capacity is known to be free. +func (ss *SchedulerService) handleLaunchDeferred( + ctx context.Context, run *models.Run, workspaceID string, seq int, +) (bool, *routing.BlockReason, error) { + now := time.Now().UTC() + attempt := models.RouteAttempt{ + RunID: run.ID, + Seq: seq, + Outcome: RouteAttemptOutcomeDeferredCapacity, + FinishedAt: &now, + } + if err := ss.repo.UpdateRouteAttemptOutcome(ctx, &attempt); err != nil { + return false, nil, err + } + hydrated := ss.hydrateAttempt(ctx, run.ID, seq, attempt) + ss.publishRouteAttemptAppended(ctx, run.ID, hydrated) + if err := ss.repo.ParkRunForProviderCapacity(ctx, + run.ID, routing.StatusBlockedActionRequired, time.Time{}); err != nil { + return false, nil, err + } + ss.logger.Info("run parked: launch deferred by session ceiling", + zap.String("run_id", run.ID)) + ss.recordRouteParked(workspaceID, run.ID, routing.StatusBlockedActionRequired) + return false, &routing.BlockReason{Status: routing.StatusBlockedActionRequired}, nil +} + // persistLaunchedSession stores the session id a successful launch // produced (AC-OFFICE-LOOP-LIVENESS-002.7). An empty id is counted as a // without-session launch (AC-002.8) rather than attempted as a write. diff --git a/apps/backend/internal/office/service/ceiling_credentials.go b/apps/backend/internal/office/service/ceiling_credentials.go new file mode 100644 index 0000000000..1d2287fc2f --- /dev/null +++ b/apps/backend/internal/office/service/ceiling_credentials.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "fmt" +) + +// RemintCeilingLaunchCredentials implements orchestrator.CeilingLaunchCredentialReminter, +// re-minting KANDEV_API_KEY / KANDEV_RUN_TOKEN for a ceiling-deferred launch +// replay from the durable identity fields (agent/workspace/run id) that +// remain in env. The original env is returned unchanged when it carries no +// Office run identity (a non-Office launch, or an Office env this build's +// buildEnvVars shape has since dropped a field from) — there is nothing to +// re-mint, and startTask should proceed rather than fail the replay outright. +func (si *SchedulerIntegration) RemintCeilingLaunchCredentials( + ctx context.Context, taskID string, env map[string]string, +) (map[string]string, error) { + runID := env["KANDEV_RUN_ID"] + agentID := env["KANDEV_AGENT_ID"] + workspaceID := env["KANDEV_WORKSPACE_ID"] + if runID == "" || agentID == "" || workspaceID == "" || si.svc.agentTokenMinter == nil { + return env, nil + } + run, err := si.svc.repo.GetRun(ctx, runID) + if err != nil { + return nil, fmt.Errorf("ceiling credential re-mint: load run %s: %w", runID, err) + } + jwt, err := si.svc.agentTokenMinter.MintRuntimeJWT(agentID, taskID, workspaceID, runID, run.SessionID, run.Capabilities) + if err != nil { + return nil, fmt.Errorf("ceiling credential re-mint: mint runtime jwt: %w", err) + } + refreshed := make(map[string]string, len(env)) + for k, v := range env { + refreshed[k] = v + } + refreshed["KANDEV_API_KEY"] = jwt + refreshed["KANDEV_RUN_TOKEN"] = jwt + return refreshed, nil +} diff --git a/apps/backend/internal/office/service/scheduler_integration.go b/apps/backend/internal/office/service/scheduler_integration.go index c7d913955a..ab8d1d2913 100644 --- a/apps/backend/internal/office/service/scheduler_integration.go +++ b/apps/backend/internal/office/service/scheduler_integration.go @@ -12,12 +12,26 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/models" "github.com/kandev/kandev/internal/office/pause" + "github.com/kandev/kandev/internal/office/routing" officeruntime "github.com/kandev/kandev/internal/office/runtime" "github.com/kandev/kandev/internal/office/shared" "github.com/kandev/kandev/internal/office/wakeup" v1 "github.com/kandev/kandev/pkg/api/v1" ) +// ErrLaunchDeferredByCapacity is the TaskStarter-side sentinel a +// TaskStarter*ReturningSession implementation returns when the underlying +// launch was deferred rather than started or failed — the orchestrator's +// session ceiling admitted no reservation but persisted a replay record of +// its own, so the launch will complete later without this caller's +// involvement. A caller that only distinguishes "session created" from +// "error" cannot tell a deferred launch apart from an ordinary success +// that forgot to report its session id; recognizing this sentinel lets it +// stop short of recording the launch as started (counters, health, +// persisted session id) while also not treating it as a failure worth +// retrying through provider fallback or restoring a one-shot claim over. +var ErrLaunchDeferredByCapacity = errors.New("office: launch deferred by capacity, not started") + // DefaultTickInterval is the default run processing interval. const DefaultTickInterval = 5 * time.Second @@ -697,6 +711,10 @@ func (si *SchedulerIntegration) launchAgent( err = si.svc.taskStarter.StartTask(ctx, taskID, launch.ProfileID, "", "", "", launch.Prompt, "", false, nil) } + if errors.Is(err, ErrLaunchDeferredByCapacity) { + si.handleLaunchDeferred(ctx, run) + return false + } if err != nil { si.logger.Error("agent launch failed", zap.String("run_id", runID), zap.Error(err)) @@ -713,6 +731,34 @@ func (si *SchedulerIntegration) launchAgent( return true } +// handleLaunchDeferred is launchAgent's disposition when the task starter +// reports ErrLaunchDeferredByCapacity: the orchestrator's own session +// ceiling already persisted a replay record for this exact launch and owns +// retrying it, so this run must not be counted as launched (no session was +// created) and must not go through HandleRunFailure's backoff-retry path — +// a second automatic attempt from there would race the ceiling's own +// replay into a double launch. Parking under blocked_provider_action_required +// (the same status the routed dispatch path uses for the identical +// disposition, see scheduler.SchedulerService.handleLaunchDeferred) keeps +// the scheduler's own wake-up loop from ever picking the run back up on +// its own; an operator notices via "Retry now" once capacity is known to +// be free. +func (si *SchedulerIntegration) handleLaunchDeferred(ctx context.Context, run *models.Run) { + si.releaseCheckoutIfNeeded(ctx, run) + si.svc.AppendRunEvent(ctx, run.ID, "adapter.invoke", "info", map[string]interface{}{ + "phase": "deferred", + "reason": "session_ceiling", + }) + if err := si.svc.repo.ParkRunForProviderCapacity(ctx, + run.ID, routing.StatusBlockedActionRequired, time.Time{}); err != nil { + si.logger.Error("failed to park run deferred by session ceiling", + zap.String("run_id", run.ID), zap.Error(err)) + return + } + si.logger.Info("run parked: launch deferred by session ceiling", + zap.String("run_id", run.ID)) +} + // persistLaunchedSession stores the session id a successful direct // launch produced (AC-OFFICE-LOOP-LIVENESS-002.7) — the scheduler-side // counterpart to scheduler.SchedulerService.persistLaunchedSession for diff --git a/apps/backend/internal/orchestrator/ceiling_defer.go b/apps/backend/internal/orchestrator/ceiling_defer.go new file mode 100644 index 0000000000..51b2de01e6 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_defer.go @@ -0,0 +1,153 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" +) + +// deferredLaunchCASRetryBudget bounds the read-compare-write retry AC-40a +// requires when a concurrent writer wins the compare-and-set race. +const deferredLaunchCASRetryBudget = 3 + +// ErrCeilingLaunchConflict reports an automatic launch refusal that could not +// replace a different launch already queued for the same task. The first +// record remains durable, while the caller retains ownership of the second +// launch because no replay record was created for it. +var ErrCeilingLaunchConflict = errors.New("a different ceiling launch is already deferred for this task") + +// publishTaskUpdatedByID reloads taskID and publishes it. The ceiling +// deferral CAS helpers (GetTaskDeferredLaunch/SetTaskDeferredLaunchIfUnchanged) +// write tasks.metadata directly and do not publish on their own, so a caller +// that lands a ceiling_deferred change uses this to keep the WS-driven UI +// (queued/cleared state) in sync with the row it just wrote. +func (s *Service) publishTaskUpdatedByID(ctx context.Context, taskID string) { + task, err := s.repo.GetTask(ctx, taskID) + if err != nil { + s.logger.Zap().Warn("could not reload task after a ceiling deferral change; task.updated not published", + zap.String("task_id", taskID), zap.Error(err)) + return + } + s.publishTaskUpdated(ctx, task) +} + +// deferCeilingRefusal persists a ceiling_deferred record for an automatic launch +// the admission controller refused, merging it into whatever the task's shared +// deferred_launch value already holds (AC-46/AC-46a) under a row-locked +// compare-and-set (AC-12e/AC-40). The stored origin is always automatic: AC-13c +// states a manual launch is never deferred, so there is nothing else to record. +// +// kind and payload describe the launch to replay; reasonCode is carried onto the +// record's bookkeeping (AC-22a) and is never nested with the payload. sessionID, +// population, populationKnown and ceiling describe this refusal for the AC-49 +// card carrier; sessionID is empty for seam 1, which AC-49 excludes entirely. +func (s *Service) deferCeilingRefusal( + ctx context.Context, taskID, sessionID string, kind models.CeilingLaunchKind, payload map[string]interface{}, + reasonCode string, population int, populationKnown bool, ceiling int, +) error { + for attempt := 0; attempt < deferredLaunchCASRetryBudget; attempt++ { + existingRaw, prior, err := s.repo.GetTaskDeferredLaunch(ctx, taskID) + if err != nil { + return fmt.Errorf("reading deferred launch for task %s: %w", taskID, err) + } + + deferral := models.CeilingDeferral{ + Kind: kind, + Payload: payload, + Origin: string(launchOriginAutomatic), + ReasonCode: reasonCode, + QueuedAt: time.Now(), + Population: population, + PopulationKnown: populationKnown, + Ceiling: ceiling, + } + + if existingCeiling, readErr := models.ReadCeilingDeferral(existingRaw); readErr == nil { + // A ceiling_deferred record already exists for this task (AC-12d). + equivalent, cmpErr := models.CeilingDeferralsEquivalent(existingCeiling, deferral) + if cmpErr != nil { + return fmt.Errorf("comparing deferred launch payloads for task %s: %w", taskID, cmpErr) + } + if !equivalent { + // A different launch: retain the one already stored and report + // the collision rather than losing either payload silently. + s.logger.Zap().Warn("ceiling refusal superseded by an earlier pending deferral for the same task", + zap.String("task_id", taskID), + zap.String("stored_kind", string(existingCeiling.Kind)), + zap.String("superseded_kind", string(kind)), + zap.String(ceilingFieldReasonCode, ceilingReasonSuperseded)) + return fmt.Errorf("%w: task %s already holds %s", ErrCeilingLaunchConflict, taskID, existingCeiling.Kind) + } + if existingCeiling.ReasonCode == reasonCode { + // AC-12a/AC-49e: the same launch, refused for the same reason, + // already recorded. Leave it exactly as stored, including its + // queued_at (AC-32) and its card-surface bookkeeping. + return nil + } + // AC-49f: the same pending launch, but the reason changed (for + // example a transient population read failure resolved into an + // ordinary refusal). Update the stored reason and this refusal's + // population/ceiling reading, preserve queued_at, and reset the + // surface stamps so the new reason gets its own note (AC-49g1). + updated := existingCeiling + updated.ReasonCode = reasonCode + updated.Population = population + updated.PopulationKnown = populationKnown + updated.Ceiling = ceiling + if s.writeCeilingDeferralUpdate(ctx, taskID, existingRaw, prior, updated) { + s.attemptCeilingSurfaceWrite(ctx, taskID) + s.publishTaskUpdatedByID(ctx, taskID) + } + return nil + } + + record, discarded, replaced := models.MergeCeilingRecord(existingRaw, deferral) + if replaced { + s.logger.Zap().Warn("deferred_launch held a non-object value; the ceiling refusal replaced it", + zap.String("task_id", taskID), zap.Any("discarded_value", discarded)) + } + + stored, lostCompare, err := s.repo.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, prior, record) + if err != nil { + return fmt.Errorf("writing deferred launch for task %s: %w", taskID, err) + } + if stored { + s.attemptCeilingSurfaceWrite(ctx, taskID) + s.publishTaskUpdatedByID(ctx, taskID) + return nil + } + if lostCompare { + continue + } + return fmt.Errorf("writing deferred launch for task %s: repository reported neither stored nor a lost compare", taskID) + } + return fmt.Errorf("%s: could not persist deferred launch for task %s after %d attempts", + ceilingReasonDeferWriteFailed, taskID, deferredLaunchCASRetryBudget) +} + +// writeCeilingDeferralUpdate writes an updated deferral over the record that +// produced it, clearing the AC-49 surface stamps explicitly: CeilingRecordKeys +// does not emit them at all, so without this a stale ceiling_surface_written_at +// from before the reason changed would otherwise survive the merge untouched +// and permanently suppress the new note. Reports whether the write landed; a +// lost compare is treated as "someone else already moved this record on" and +// is not retried here, since the next admission attempt will see fresh state. +func (s *Service) writeCeilingDeferralUpdate( + ctx context.Context, taskID string, existingRaw interface{}, prior interface{}, updated models.CeilingDeferral, +) bool { + record, _, _ := models.MergeCeilingRecord(existingRaw, updated) + delete(record, models.CeilingSurfaceWrittenAtKey) + delete(record, models.CeilingSurfaceAttemptCountKey) + stored, _, err := s.repo.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, prior, record) + if err != nil { + s.logger.Zap().Warn("could not update a ceiling deferral's reason code", + zap.String("task_id", taskID), zap.Error(err)) + return false + } + return stored +} diff --git a/apps/backend/internal/orchestrator/ceiling_defer_test.go b/apps/backend/internal/orchestrator/ceiling_defer_test.go new file mode 100644 index 0000000000..1ac134cd8c --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_defer_test.go @@ -0,0 +1,174 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestDeferCeilingRefusalCreatesRecordFromAbsent covers the first refusal for a +// task with no deferred_launch value at all: AC-11's create case. +func TestDeferCeilingRefusalCreatesRecordFromAbsent(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "defer-absent", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + payload := map[string]interface{}{"prompt": "hello"} + if err := svc.deferCeilingRefusal(ctx, "defer-absent", "", models.CeilingLaunchStart, payload, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal: %v", err) + } + + record := deferredLaunchOf(t, svc, "defer-absent") + if record == nil { + t.Fatal("no deferred_launch record was written") + } + if record[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling_deferred not set: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchStart) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchStart) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "hello" { + t.Fatalf("ceiling_launch_payload did not carry the payload: %+v", record) + } + if record[models.CeilingQueuedAtKey] == nil || record[models.CeilingQueuedAtKey] == "" { + t.Fatalf("ceiling_queued_at was not stamped: %+v", record) + } +} + +// TestDeferCeilingRefusalMergesOntoExistingWIPIntent pins AC-46a/AC-12b: a +// ceiling refusal must not clobber a pre-existing start_when_unblocked intent. +func TestDeferCeilingRefusalMergesOntoExistingWIPIntent(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "defer-merge", Title: "T", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + "user_id": "u1", + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + if err := svc.deferCeilingRefusal(ctx, "defer-merge", "", models.CeilingLaunchStart, map[string]interface{}{"prompt": "p"}, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal: %v", err) + } + + record := deferredLaunchOf(t, svc, "defer-merge") + if record[models.DeferredLaunchStartWhenUnblockedKey] != true { + t.Fatalf("start_when_unblocked was clobbered: %+v", record) + } + if record["user_id"] != "u1" { + t.Fatalf("user_id was clobbered: %+v", record) + } + if record[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling_deferred was not added: %+v", record) + } +} + +// TestDeferCeilingRefusalIsANoOpOnByteIdenticalDuplicate pins AC-12a/AC-32: a +// second refusal that is the same launch must not disturb the stored record's +// queued_at. +func TestDeferCeilingRefusalIsANoOpOnByteIdenticalDuplicate(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "defer-dup", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + payload := map[string]interface{}{"prompt": "same"} + if err := svc.deferCeilingRefusal(ctx, "defer-dup", "", models.CeilingLaunchStart, payload, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal (1): %v", err) + } + first := deferredLaunchOf(t, svc, "defer-dup") + firstQueuedAt := first[models.CeilingQueuedAtKey] + + if err := svc.deferCeilingRefusal(ctx, "defer-dup", "", models.CeilingLaunchStart, map[string]interface{}{"prompt": "same"}, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal (2): %v", err) + } + second := deferredLaunchOf(t, svc, "defer-dup") + if second[models.CeilingQueuedAtKey] != firstQueuedAt { + t.Fatalf("queued_at changed on a byte-identical duplicate: %v -> %v", firstQueuedAt, second[models.CeilingQueuedAtKey]) + } +} + +// TestDeferCeilingRefusalRetainsEarlierRecordOnDifferingDuplicate pins AC-12d: +// a second, differing automatic refusal for the same task must not overwrite the +// first launch's payload. +func TestDeferCeilingRefusalRetainsEarlierRecordOnDifferingDuplicate(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "defer-collide", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := svc.deferCeilingRefusal(ctx, "defer-collide", "", models.CeilingLaunchStart, map[string]interface{}{"prompt": "first"}, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal (1): %v", err) + } + if err := svc.deferCeilingRefusal(ctx, "defer-collide", "", models.CeilingLaunchStart, map[string]interface{}{"prompt": "second"}, ceilingReasonRefused, 0, false, 0); !errors.Is(err, ErrCeilingLaunchConflict) { + t.Fatalf("deferCeilingRefusal (2) error = %v, want ErrCeilingLaunchConflict", err) + } + + record := deferredLaunchOf(t, svc, "defer-collide") + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "first" { + t.Fatalf("the earlier record was not retained: %+v", record) + } +} + +// TestDeferCeilingRefusalReplacesNonObjectValue pins AC-46: a non-object +// deferred_launch value is replaced wholesale by the ceiling record. +func TestDeferCeilingRefusalReplacesNonObjectValue(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "defer-nonobject", Title: "T", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: "garbage"}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + if err := svc.deferCeilingRefusal(ctx, "defer-nonobject", "", models.CeilingLaunchStart, map[string]interface{}{"prompt": "p"}, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal: %v", err) + } + + record := deferredLaunchOf(t, svc, "defer-nonobject") + if record[models.CeilingDeferredKey] != true { + t.Fatalf("non-object value was not replaced with a ceiling record: %+v", record) + } +} + +// TestDeferCeilingRefusalPublishesTaskUpdated pins the fix for a ceiling +// deferral write not publishing task.updated: without it, the WS-driven UI +// never learns a launch was queued behind the session ceiling. +func TestDeferCeilingRefusalPublishesTaskUpdated(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "defer-publish", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + events := &capturingTaskEvents{} + svc.SetTaskEventPublisher(events) + + if err := svc.deferCeilingRefusal(ctx, "defer-publish", "s1", models.CeilingLaunchStart, + map[string]interface{}{"prompt": "hello"}, ceilingReasonRefused, 0, false, 0); err != nil { + t.Fatalf("deferCeilingRefusal: %v", err) + } + + published := events.last() + if published == nil { + t.Fatal("deferCeilingRefusal did not publish a task.updated event") + } + if published.ID != "defer-publish" { + t.Fatalf("published task id = %q, want %q", published.ID, "defer-publish") + } + deferred, ok := published.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok || deferred[models.CeilingDeferredKey] != true { + t.Fatalf("published task's deferred_launch is missing the new ceiling record: %#v", + published.Metadata[models.MetaKeyDeferredLaunch]) + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_dynamic_launch_release_test.go b/apps/backend/internal/orchestrator/ceiling_dynamic_launch_release_test.go new file mode 100644 index 0000000000..62bd3812c7 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_dynamic_launch_release_test.go @@ -0,0 +1,91 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + "github.com/stretchr/testify/require" +) + +// TestHandleAgentProcessStarted_ConfirmsCeilingReservationUnconditionally pins +// AC-52/AC-56a: the acceptance edge must confirm before profileExecutionResolver's +// own nil/sessionID guard, so an instance without dynamic routing configured still +// confirms a reservation on every ordinary launch. +func TestHandleAgentProcessStarted_ConfirmsCeilingReservationUnconditionally(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + require.Nil(t, svc.profileExecutionResolver) + + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "t", sessionID: "session-started", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + + svc.handleAgentProcessStarted(context.Background(), "t", "session-started", "exec-1") + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["session-started"] + svc.sessionCeiling.mu.Unlock() + require.False(t, held, "the acceptance edge must confirm (release) the reservation") + + select { + case <-svc.ceilingSweeper.signal: + t.Fatal("confirming on acceptance must not signal the retry sweep") + default: + } +} + +// TestHandleAgentProcessStartFailed_ReleasesCeilingReservationUnconditionally +// pins AC-52's failure edge, mirroring the acceptance-edge test above. +func TestHandleAgentProcessStartFailed_ReleasesCeilingReservationUnconditionally(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + require.Nil(t, svc.profileExecutionResolver) + + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "t", sessionID: "session-failed", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + + svc.handleAgentProcessStartFailed(context.Background(), "t", "session-failed", "exec-1", context.DeadlineExceeded) + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["session-failed"] + svc.sessionCeiling.mu.Unlock() + require.False(t, held, "the failure edge must release the reservation") + + select { + case <-svc.ceilingSweeper.signal: + default: + t.Fatal("releasing on a start failure must signal the retry sweep") + } +} + +func TestHandleAgentProcessStartFailed_IgnoresStaleExecutionReservationRelease(t *testing.T) { + ctx := context.Background() + svc, repo := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + seedTaskAndSession(t, repo, "task-stale-start", "session-stale-start", models.TaskSessionStateStarting) + seedExecutorRunning(t, repo, "session-stale-start", "task-stale-start", "exec-successor") + + decision := svc.sessionCeiling.admit(ctx, admissionRequest{ + taskID: "task-stale-start", sessionID: "session-stale-start", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + + svc.handleAgentProcessStartFailed(ctx, "task-stale-start", "session-stale-start", "exec-predecessor", context.DeadlineExceeded) + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["session-stale-start"] + svc.sessionCeiling.mu.Unlock() + require.True(t, held, "a stale predecessor callback must not release the successor reservation") + select { + case <-svc.ceilingSweeper.signal: + t.Fatal("a stale callback must not wake the ceiling sweep") + default: + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_manual_override.go b/apps/backend/internal/orchestrator/ceiling_manual_override.go new file mode 100644 index 0000000000..7dd9538438 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_manual_override.go @@ -0,0 +1,96 @@ +package orchestrator + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// ceilingManualOverrideMetadataKey is AC-53's session metadata key: an +// audit record of a manual launch admitted while the instance was at or +// above its session ceiling. +const ceilingManualOverrideMetadataKey = "ceiling_manual_override" + +// recordManualOverrideIfAdmitted implements AC-14/AC-53: once a manually +// admitted reservation is settled onto a real session, it stamps a +// once-per-session audit record on the session and writes the AC-14 card +// warning. It is a no-op unless this launch was an actual override (an +// ordinary admission never reaches here with manualOverride set) and a +// session exists to attach to — called from the point each seam's +// reservation is bound to that session id, independent of whether the +// launch or resume attempt that follows later succeeds or fails, so a +// failure between admission and agent start can never erase the only +// evidence an override happened. AC-14b's "the launch failed before a +// session existed" case (seam 1's pre-session-creation window) still never +// reaches this at all. +func (s *Service) recordManualOverrideIfAdmitted( + ctx context.Context, taskID, sessionID string, manualOverride bool, population int, populationKnown bool, ceiling int, +) { + if !manualOverride || sessionID == "" { + return + } + + reasonCodes := []string{ceilingReasonManualOverride} + if !populationKnown { + // AC-33a: "unknown" is an absent key, not a rendered word — population + // is omitted entirely below, and the reason why is added alongside + // the override code rather than replacing it. + reasonCodes = append(reasonCodes, ceilingReasonUnknownPopulation) + } + record := map[string]interface{}{ + "reason_codes": reasonCodes, + ceilingFieldCeiling: ceiling, + "recorded_at": time.Now().UTC().Format(time.RFC3339), + } + if populationKnown { + record["population"] = population + } + + stored, err := s.repo.SetSessionMetadataKeyIfAbsent(ctx, sessionID, ceilingManualOverrideMetadataKey, record) + if err != nil { + // AC-53b: the launch is already admitted and proceeding; losing the + // audit record is a reporting gap, not a lost launch. Never fail, + // delay, retry or roll back for this. + s.logger.Zap().Warn("could not record the ceiling manual-override audit metadata", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) + } + if err == nil && !stored { + // Idempotent: a prior call for this session already recorded (and, + // per this same rule, already warned) the override that created it. + return + } + + s.writeCeilingManualOverrideWarning(ctx, taskID, sessionID, population, populationKnown, ceiling) +} + +// writeCeilingManualOverrideWarning is AC-14's card warning, naming the +// population and ceiling the override was admitted against. +func (s *Service) writeCeilingManualOverrideWarning( + ctx context.Context, taskID, sessionID string, population int, populationKnown bool, ceiling int, +) { + metadata := map[string]interface{}{ + metaKeyVariant: metaVariantCeiling, + ceilingFieldReasonCode: ceilingReasonManualOverride, + ceilingFieldCeiling: ceiling, + metaKeyTaskID: taskID, + } + content := fmt.Sprintf("This launch was started manually while the instance was at its session ceiling of %d.", ceiling) + if populationKnown { + metadata["population"] = population + content = fmt.Sprintf("This launch was started manually while the instance was at its session ceiling (%d of %d).", + population, ceiling) + } + if s.messageCreator == nil { + return + } + if err := s.messageCreator.CreateSessionMessage( + ctx, taskID, content, sessionID, string(v1.MessageTypeStatus), "", metadata, false, + ); err != nil { + s.logger.Zap().Warn("could not write the ceiling manual-override card warning", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_manual_override_test.go b/apps/backend/internal/orchestrator/ceiling_manual_override_test.go new file mode 100644 index 0000000000..c96f24e0c8 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_manual_override_test.go @@ -0,0 +1,115 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestRecordManualOverrideIfAdmitted_WritesAuditRecordAndWarning pins +// AC-14/AC-53: an admitted manual override, once it reaches a real session, +// stamps a session metadata audit record and writes the AC-14 card warning. +func TestRecordManualOverrideIfAdmitted_WritesAuditRecordAndWarning(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "mo-task", "mo-session", models.TaskSessionStateStarting) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + svc.recordManualOverrideIfAdmitted(ctx, "mo-task", "mo-session", true, 3, true, 2) + + session, err := repo.GetTaskSession(ctx, "mo-session") + require.NoError(t, err) + record, ok := session.Metadata[ceilingManualOverrideMetadataKey].(map[string]interface{}) + require.True(t, ok, "audit record was not written: %+v", session.Metadata) + require.Equal(t, []interface{}{ceilingReasonManualOverride}, toInterfaceSlice(record["reason_codes"])) + require.EqualValues(t, 2, record[ceilingFieldCeiling]) + require.EqualValues(t, 3, record["population"]) + require.NotEmpty(t, record["recorded_at"]) + + require.Len(t, messages.sessionMessages, 1) + note := messages.sessionMessages[0] + require.Equal(t, "mo-session", note.sessionID) + require.Equal(t, metaVariantCeiling, note.metadata[metaKeyVariant]) + require.Equal(t, ceilingReasonManualOverride, note.metadata[ceilingFieldReasonCode]) +} + +// TestRecordManualOverrideIfAdmitted_PopulationUnknownOmitsPopulationKey pins +// AC-33a: when the population was unknown at admission, the audit record +// omits the population key entirely rather than recording a sentinel, and +// carries the unknown-population reason code alongside the override code. +func TestRecordManualOverrideIfAdmitted_PopulationUnknownOmitsPopulationKey(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "mo-unk-task", "mo-unk-session", models.TaskSessionStateStarting) + svc.messageCreator = &mockMessageCreator{} + + svc.recordManualOverrideIfAdmitted(ctx, "mo-unk-task", "mo-unk-session", true, 0, false, 2) + + session, err := repo.GetTaskSession(ctx, "mo-unk-session") + require.NoError(t, err) + record := session.Metadata[ceilingManualOverrideMetadataKey].(map[string]interface{}) + _, hasPopulation := record["population"] + require.False(t, hasPopulation, "population must be omitted, not recorded as a sentinel: %+v", record) + require.Equal(t, + []interface{}{ceilingReasonManualOverride, ceilingReasonUnknownPopulation}, + toInterfaceSlice(record["reason_codes"])) +} + +// TestRecordManualOverrideIfAdmitted_IdempotentSecondCallDoesNotRewriteOrRewarn +// pins AC-53a: the audit record is written once per session, and a second +// admitted override for the same session neither overwrites the record nor +// sends a second warning. +func TestRecordManualOverrideIfAdmitted_IdempotentSecondCallDoesNotRewriteOrRewarn(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "mo-dup-task", "mo-dup-session", models.TaskSessionStateStarting) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + svc.recordManualOverrideIfAdmitted(ctx, "mo-dup-task", "mo-dup-session", true, 3, true, 2) + svc.recordManualOverrideIfAdmitted(ctx, "mo-dup-task", "mo-dup-session", true, 9, true, 2) + + session, err := repo.GetTaskSession(ctx, "mo-dup-session") + require.NoError(t, err) + record := session.Metadata[ceilingManualOverrideMetadataKey].(map[string]interface{}) + require.EqualValues(t, 3, record["population"], "the first admission's reading must be retained") + require.Len(t, messages.sessionMessages, 1, "a repeat admission must not warn twice") +} + +// TestRecordManualOverrideIfAdmitted_NotOverrideOrNoSessionIsNoOp pins that +// this is a no-op for an ordinary admission (manualOverride=false) and for +// AC-14b's window before a session exists (sessionID empty). +func TestRecordManualOverrideIfAdmitted_NotOverrideOrNoSessionIsNoOp(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "mo-noop-task", "mo-noop-session", models.TaskSessionStateStarting) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + svc.recordManualOverrideIfAdmitted(ctx, "mo-noop-task", "mo-noop-session", false, 3, true, 2) + svc.recordManualOverrideIfAdmitted(ctx, "mo-noop-task", "", true, 3, true, 2) + + session, err := repo.GetTaskSession(ctx, "mo-noop-session") + require.NoError(t, err) + require.Nil(t, session.Metadata[ceilingManualOverrideMetadataKey]) + require.Empty(t, messages.sessionMessages) +} + +func toInterfaceSlice(v interface{}) []interface{} { + switch typed := v.(type) { + case []interface{}: + return typed + case []string: + out := make([]interface{}, len(typed)) + for i, s := range typed { + out[i] = s + } + return out + default: + return nil + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_release.go b/apps/backend/internal/orchestrator/ceiling_release.go new file mode 100644 index 0000000000..2bc1803409 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_release.go @@ -0,0 +1,88 @@ +package orchestrator + +import ( + "context" + "errors" + + "github.com/kandev/kandev/internal/task/models" +) + +// ceilingCallbackOwnsSession rejects a late process callback when the session +// already points at a different execution. The callback can arrive after a +// failed launch has been replaced; releasing the reservation by session ID in +// that case would free the successor's slot. Focused adapters that do not +// expose a session row retain the legacy callback behavior, while repository +// read failures fail closed so a transient read cannot release another launch. +func (s *Service) ceilingCallbackOwnsSession(ctx context.Context, sessionID, agentExecutionID string) bool { + if sessionID == "" || s.repo == nil { + return true + } + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return errors.Is(err, models.ErrTaskSessionNotFound) + } + if session == nil { + return false + } + return session.AgentExecutionID == "" || agentExecutionID == "" || session.AgentExecutionID == agentExecutionID +} + +// isAC1SessionState reports whether state counts toward the session ceiling's +// population: task_sessions in STARTING or RUNNING (AC-1). +func isAC1SessionState(state models.TaskSessionState) bool { + return state == models.TaskSessionStateStarting || state == models.TaskSessionStateRunning +} + +// confirmCeilingReservation is AC-52's acceptance edge: the launch reached its +// first AC-1 row, so the in-flight reservation taken at admission is no +// longer needed to count it — the row does that from here on. It is silent +// and does not signal the retry driver, because confirming a reservation +// does not free capacity. +func (s *Service) confirmCeilingReservation(sessionID string) { + if s.sessionCeiling == nil || sessionID == "" { + return + } + s.sessionCeiling.release(sessionID) +} + +// releaseCeilingReservation is AC-15/AC-51's release edge: a session left the +// AC-1 population, or a launch failed before ever reaching it, so its +// reservation is dropped and the retry driver is signalled to fill the freed +// slot. Releasing an id that holds no reservation is a defined no-op (AC-31), +// so callers are never required to know whether one was actually held. +func (s *Service) releaseCeilingReservation(sessionID string) { + if s.sessionCeiling == nil || sessionID == "" { + return + } + s.sessionCeiling.release(sessionID) + s.signalCeilingSweep() +} + +// ReleaseCeilingReservation is the exported form of releaseCeilingReservation, +// satisfying the narrow interfaces other packages (mcp/handlers, task/service) +// use to release a reservation for a write they make directly against the +// repository, bypassing this package's own persistence funnels (AC-51a). +func (s *Service) ReleaseCeilingReservation(sessionID string) { + s.releaseCeilingReservation(sessionID) +} + +// releaseCeilingIfLeftPopulation is the shared guard the two persistence +// funnels (persistTaskSessionState, persistStrictTaskSessionState) and the +// enumerated bypass writers (AC-51a) apply before releasing: a release fires +// only for a transition that actually left the AC-1 population, never for a +// transition that stayed inside it or moved into it (AC-51c). +func (s *Service) releaseCeilingIfLeftPopulation(sessionID string, priorState, nextState models.TaskSessionState) { + if !isAC1SessionState(priorState) || isAC1SessionState(nextState) { + return + } + s.releaseCeilingReservation(sessionID) +} + +// IsSessionCeilingBacked implements executor.CeilingBackingChecker (AC-41): +// the single permitted route by which Executor's observation-only bypass +// detector (AC-41a) reads whether a session is backed by a reservation or a +// counted AC-1 row. It is consulted for logging only and never gates a +// launch. +func (s *Service) IsSessionCeilingBacked(ctx context.Context, sessionID string) (bool, error) { + return s.sessionCeiling.isSessionCeilingBacked(ctx, sessionID) +} diff --git a/apps/backend/internal/orchestrator/ceiling_release_test.go b/apps/backend/internal/orchestrator/ceiling_release_test.go new file mode 100644 index 0000000000..6dbe481c62 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_release_test.go @@ -0,0 +1,125 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestIsAC1SessionState(t *testing.T) { + require.True(t, isAC1SessionState(models.TaskSessionStateStarting)) + require.True(t, isAC1SessionState(models.TaskSessionStateRunning)) + require.False(t, isAC1SessionState(models.TaskSessionStateCreated)) + require.False(t, isAC1SessionState(models.TaskSessionStateCompleted)) + require.False(t, isAC1SessionState(models.TaskSessionStateFailed)) + require.False(t, isAC1SessionState(models.TaskSessionStateCancelled)) + require.False(t, isAC1SessionState(models.TaskSessionStateWaitingForInput)) +} + +func TestConfirmCeilingReservation_ReleasesWithoutSignalling(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "t", sessionID: "s-confirm", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + + svc.confirmCeilingReservation("s-confirm") + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["s-confirm"] + svc.sessionCeiling.mu.Unlock() + require.False(t, held, "confirming must release the reservation") + + select { + case <-svc.ceilingSweeper.signal: + t.Fatal("confirming a reservation must not signal the retry sweep") + default: + } +} + +func TestReleaseCeilingReservation_ReleasesAndSignals(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "t", sessionID: "s-release", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + + svc.releaseCeilingReservation("s-release") + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["s-release"] + svc.sessionCeiling.mu.Unlock() + require.False(t, held) + + select { + case <-svc.ceilingSweeper.signal: + default: + t.Fatal("releasing a reservation must signal the retry sweep") + } +} + +func TestReleaseCeilingReservation_UnknownSessionIsANoOp(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + require.NotPanics(t, func() { svc.releaseCeilingReservation("never-reserved") }) + require.NotPanics(t, func() { svc.releaseCeilingReservation("") }) + require.NotPanics(t, func() { svc.confirmCeilingReservation("") }) +} + +func TestReleaseCeilingReservation_NilCeilingIsANoOp(t *testing.T) { + svc := &Service{} + require.NotPanics(t, func() { svc.releaseCeilingReservation("x") }) + require.NotPanics(t, func() { svc.confirmCeilingReservation("x") }) + require.NotPanics(t, func() { svc.ReleaseCeilingReservation("x") }) +} + +func TestReleaseCeilingIfLeftPopulation(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + // Unlimited: this test exercises releaseCeilingIfLeftPopulation's own + // predicate, not the ceiling's admission math, so each seed must succeed + // independently regardless of how many sessions are already reserved. + svc.sessionCeiling = newSessionCeilingController(unlimitedSessionCeiling, nil, nil) + svc.ceilingSweeper = newCeilingSweeper() + + seed := func(sessionID string) { + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "t", sessionID: sessionID, origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + } + held := func(sessionID string) bool { + svc.sessionCeiling.mu.Lock() + defer svc.sessionCeiling.mu.Unlock() + _, ok := svc.sessionCeiling.reservations[sessionID] + return ok + } + + // STARTING -> RUNNING stays inside the population: no release. + seed("stays-in") + svc.releaseCeilingIfLeftPopulation("stays-in", models.TaskSessionStateStarting, models.TaskSessionStateRunning) + require.True(t, held("stays-in")) + + // CREATED -> STARTING enters the population: no release (AC-51c). + seed("enters") + svc.releaseCeilingIfLeftPopulation("enters", models.TaskSessionStateCreated, models.TaskSessionStateStarting) + require.True(t, held("enters")) + + // RUNNING -> COMPLETED leaves the population: release. + seed("leaves") + svc.releaseCeilingIfLeftPopulation("leaves", models.TaskSessionStateRunning, models.TaskSessionStateCompleted) + require.False(t, held("leaves")) + + // CREATED -> FAILED never entered the population: no-op release, no panic. + require.NotPanics(t, func() { + svc.releaseCeilingIfLeftPopulation("never-existed", models.TaskSessionStateCreated, models.TaskSessionStateFailed) + }) +} diff --git a/apps/backend/internal/orchestrator/ceiling_replay.go b/apps/backend/internal/orchestrator/ceiling_replay.go new file mode 100644 index 0000000000..6f9160283b --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_replay.go @@ -0,0 +1,550 @@ +package orchestrator + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/watcher" + "github.com/kandev/kandev/internal/task/models" + workflowmove "github.com/kandev/kandev/internal/workflow/move" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// ceilingDeferredTaskLister is the narrow repository capability AC-50b's +// sweep needs. Reached through a type assertion on s.repo, following the +// admittedSessionLister idiom in session_ceiling_config.go, rather than +// growing sessionExecutorStore for a capability only the sweep uses. +type ceilingDeferredTaskLister interface { + ListTasksWithCeilingDeferred(ctx context.Context) ([]*models.Task, error) +} + +// ceilingReplayOutcome is the three-way disposition every per-kind replay +// function reduces to, however differently each of the 7 entry points +// reports it. +type ceilingReplayOutcome int + +const ( + // ceilingReplayStillDeferred means the launch asked for admission again + // and was refused again: the record is left exactly as stored (AC-32), + // including its original queued_at, for a later pass. + ceilingReplayStillDeferred ceilingReplayOutcome = iota + // ceilingReplaySucceeded means the launch was admitted and dispatched: + // the record is cleared. + ceilingReplaySucceeded + // ceilingReplayFailed means the launch failed for a reason unrelated to + // the ceiling (AC-15d): the record is retained for a later pass and the + // failure is logged, since — unlike a refusal — nothing else observes it. + ceilingReplayFailed +) + +// drainDeferredCeilingLaunches is AC-15c's pass driver: one sweep tick walks +// every ceiling-deferred task once, in id-ascending order (AC-50c), retrying +// each in turn. A candidate that fails for a non-ceiling reason is skipped +// for the rest of this pass (AC-15d) rather than retried immediately or +// stopping the pass. +func (s *Service) drainDeferredCeilingLaunches(ctx context.Context) { + lister, ok := s.repo.(ceilingDeferredTaskLister) + if !ok { + return + } + tasks, err := lister.ListTasksWithCeilingDeferred(ctx) + if err != nil { + s.logger.Zap().Warn("could not list ceiling-deferred tasks for the retry sweep", zap.Error(err)) + return + } + for _, task := range tasks { + s.retryOneDeferredCeilingLaunch(ctx, task) + } +} + +// retryOneDeferredCeilingLaunch reads one task's deferral, evaluates AC-17b's +// drop reasons, and otherwise dispatches to the kind's own replay. +func (s *Service) retryOneDeferredCeilingLaunch(ctx context.Context, task *models.Task) { + if task == nil || task.ID == "" { + return + } + raw, _ := task.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + deferral, err := models.ReadCeilingDeferral(raw) + if err != nil { + var unreplayable *models.UnreplayableCeilingRecordError + if errors.As(err, &unreplayable) { + s.dropCeilingDeferral(ctx, task, "", deferral, ceilingReasonDroppedUnreplayableRecord, + fmt.Sprintf("the deferred launch record could not be replayed: %v", err)) + } + // Any other read failure (most likely: a concurrent writer already + // cleared the record between the list and this read) is not this + // task's fault; leave it for the next tick to see the current state. + return + } + + if reasonCode, detail, drop := s.evaluateCeilingDropReasons(ctx, task, deferral); drop { + s.dropCeilingDeferral(ctx, task, sessionIDFromCeilingPayload(deferral), deferral, reasonCode, detail) + return + } + + switch s.replayCeilingDeferral(ctx, task, deferral) { + case ceilingReplaySucceeded: + s.clearCeilingDeferredRecord(ctx, task.ID) + s.publishTaskUpdatedByID(ctx, task.ID) + case ceilingReplayFailed: + s.logger.Zap().Warn("ceiling retry replay failed for a non-ceiling reason; will retry on a later sweep", + zap.String("task_id", task.ID), zap.String("kind", string(deferral.Kind))) + case ceilingReplayStillDeferred: + // Left exactly as stored: the replay's own admission gate already + // re-persisted this same record via deferCeilingRefusal's + // equivalence check (AC-32/AC-55a). AC-49g: retry the card note if + // an earlier attempt to write it failed. + s.attemptCeilingSurfaceWrite(ctx, task.ID) + } +} + +// sessionIDFromCeilingPayload extracts the session id a deferral's payload +// names, for the six kinds that carry one. CeilingLaunchStart carries none: +// no session exists yet at seam 1. +func sessionIDFromCeilingPayload(deferral models.CeilingDeferral) string { + if deferral.Kind == models.CeilingLaunchStart { + return "" + } + return stringField(deferral.Payload, metaKeySessionID) +} + +// evaluateCeilingDropReasons is AC-17b's closed set of drop reasons. Reasons +// (a)-(c) are universal; (d) and (e) are scoped to the "start" kind only — +// concrete tracing showed only startTask's own gate embeds the terminal-PR +// guard, and none of the other six entry points call shouldAutoStartStep +// internally, so applying either check to an unrelated kind risks dropping a +// legitimate continuation that has nothing to do with step-entry auto-start +// semantics. Reason (f), the unreplayable record, is handled by the caller +// before this is ever reached. +func (s *Service) evaluateCeilingDropReasons( + ctx context.Context, task *models.Task, deferral models.CeilingDeferral, +) (reasonCode, detail string, drop bool) { + if task.ArchivedAt != nil { + return ceilingReasonDroppedTaskIneligible, "task is archived", true + } + if task.State == v1.TaskStateCancelled { + return ceilingReasonDroppedTaskIneligible, "task is cancelled", true + } + if sessionID := sessionIDFromCeilingPayload(deferral); sessionID != "" { + if _, err := s.repo.GetTaskSession(ctx, sessionID); err != nil { + if errors.Is(err, models.ErrTaskSessionNotFound) { + return ceilingReasonDroppedTaskIneligible, "session no longer exists", true + } + // An unrelated read failure isn't grounds to drop the record; + // the replay attempt below will hit the same failure and retain it. + return "", "", false + } + } + if deferral.Kind == models.CeilingLaunchStart { + if !task.IsFromOffice && !s.shouldAutoStartStep(ctx, task.WorkflowStepID) { + return ceilingReasonDroppedTaskIneligible, "workflow step no longer auto-starts", true + } + if s.shouldSkipTerminalPRAutoStart(ctx, task) { + return ceilingReasonDroppedLaunchGateDeclined, "terminal PR auto-start gate declined the launch", true + } + } + return "", "", false +} + +// dropCeilingDeferral is AC-17b/AC-17c's disposition: clear the record, log +// the drop (AC-22/22a) and, where a session exists to attach it to, write a +// minimal one-shot drop note. The full AC-49 suppression-and-bounded-retry +// card surface for an ONGOING refusal is separate, later scope; this is only +// the terminal "this will never be retried again" notice. +func (s *Service) dropCeilingDeferral( + ctx context.Context, task *models.Task, sessionID string, deferral models.CeilingDeferral, reasonCode, detail string, +) { + s.logger.Zap().Warn("dropping a ceiling-deferred launch", + zap.String("task_id", task.ID), + zap.String("kind", string(deferral.Kind)), + zap.String(ceilingFieldReasonCode, reasonCode), + zap.String("detail", detail)) + + if sessionID != "" && s.messageCreator != nil { + if err := s.messageCreator.CreateSessionMessage( + ctx, task.ID, + fmt.Sprintf("The queued launch for this session was dropped: %s.", detail), + sessionID, string(v1.MessageTypeStatus), "", + map[string]interface{}{metaKeyVariant: metaVariantCeiling, ceilingFieldReasonCode: reasonCode}, + false, + ); err != nil { + s.logger.Zap().Warn("could not write the ceiling drop card note", + zap.String("task_id", task.ID), zap.String("session_id", sessionID), zap.Error(err)) + } + } + + s.clearCeilingDeferredRecord(ctx, task.ID) + s.publishTaskUpdatedByID(ctx, task.ID) +} + +// clearCeilingDeferredRecord is AC-17's "clear the whole record" disposition, +// applied on both a successful replay and a drop, under the same +// compare-and-set retry loop deferCeilingRefusal itself uses. +func (s *Service) clearCeilingDeferredRecord(ctx context.Context, taskID string) { + for attempt := 0; attempt < deferredLaunchCASRetryBudget; attempt++ { + existingRaw, prior, err := s.repo.GetTaskDeferredLaunch(ctx, taskID) + if err != nil { + s.logger.Zap().Warn("could not read deferred launch while clearing a ceiling record", + zap.String("task_id", taskID), zap.Error(err)) + return + } + record := stripCeilingRecordKeys(existingRaw) + stored, lostCompare, err := s.repo.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, prior, record) + if err != nil { + s.logger.Zap().Warn("could not clear a ceiling deferred record", + zap.String("task_id", taskID), zap.Error(err)) + return + } + if stored { + return + } + if lostCompare { + continue + } + return + } + s.logger.Zap().Warn("could not clear a ceiling deferred record after repeated compare-and-set losses", + zap.String("task_id", taskID)) +} + +// stripCeilingRecordKeys removes every ceiling_-prefixed key from a +// deferred_launch record, leaving any coexisting WIP-overflow or +// dependency-chain intent untouched: the ceiling_ prefix (IsCeilingRecordKey) +// is exactly how the two halves of the shared record partition without +// either side tracking the other's keys. SetTaskDeferredLaunchIfUnchanged +// rejects a nil value, so a record with no keys at all is an empty object, +// never nil. +func stripCeilingRecordKeys(existing interface{}) map[string]interface{} { + current, ok := existing.(map[string]interface{}) + if !ok { + return map[string]interface{}{} + } + stripped := make(map[string]interface{}, len(current)) + for key, value := range current { + if models.IsCeilingRecordKey(key) { + continue + } + stripped[key] = value + } + return stripped +} + +// replayCeilingDeferral dispatches to the one replay function matching the +// deferral's kind. The closed set is exhaustive; a kind outside it was +// already rejected as unreplayable by ReadCeilingDeferral before this point. +func (s *Service) replayCeilingDeferral(ctx context.Context, task *models.Task, deferral models.CeilingDeferral) ceilingReplayOutcome { + switch deferral.Kind { + case models.CeilingLaunchStart: + return s.replayCeilingLaunchStart(ctx, task, deferral.Payload) + case models.CeilingLaunchStartCreated: + return s.replayCeilingLaunchStartCreated(ctx, task, deferral.Payload) + case models.CeilingLaunchPromptEnsure: + return s.replayCeilingLaunchPromptEnsure(ctx, task, deferral.Payload) + case models.CeilingLaunchWorkflowStepEnsure: + return s.replayCeilingLaunchWorkflowStepEnsure(ctx, task, deferral.Payload) + case models.CeilingLaunchQueueDrainEnsure: + return s.replayCeilingLaunchQueueDrainEnsure(ctx, task, deferral.Payload) + case models.CeilingLaunchResume: + return s.replayCeilingLaunchResume(ctx, task, deferral.Payload) + case models.CeilingLaunchDynamicRelaunch: + return s.replayCeilingLaunchDynamicRelaunch(ctx, task, deferral.Payload) + default: + return ceilingReplayFailed + } +} + +// stringField and boolField read a primitive out of a generically-decoded +// payload map, defaulting to the zero value for an absent or mistyped key. +func stringField(payload map[string]interface{}, key string) string { + v, _ := payload[key].(string) + return v +} + +func boolField(payload map[string]interface{}, key string) bool { + v, _ := payload[key].(bool) + return v +} + +// int64Field reads an integer out of a generically-decoded payload map. A +// JSON round trip through map[string]interface{} always decodes numbers as +// float64, so this cannot type-assert int64 directly like boolField does. +func int64Field(payload map[string]interface{}, key string) int64 { + switch v := payload[key].(type) { + case float64: + return int64(v) + case int64: + return v + default: + return 0 + } +} + +// remintCeilingLaunchCredentials refreshes the short-lived Office runtime +// credentials redactedCeilingLaunchEnv stripped before persisting, using +// whichever CeilingLaunchCredentialReminter was wired (nil for a non-Office +// deployment, or when no reminter is registered — env round-trips unchanged +// in that case). A re-mint failure is not fatal to the replay attempt: it +// logs and falls back to the captured env, which is only stale — not +// wrong — for a non-Office launch, since only Office launches carry these +// credential keys in the first place. +func (s *Service) remintCeilingLaunchCredentials(ctx context.Context, taskID string, env map[string]string) map[string]string { + if s.ceilingCredentialReminter == nil || len(env) == 0 { + return env + } + refreshed, err := s.ceilingCredentialReminter.RemintCeilingLaunchCredentials(ctx, taskID, env) + if err != nil { + s.logger.Zap().Warn("ceiling replay: credential re-mint failed; replaying with the captured env", + zap.String("task_id", taskID), zap.Error(err)) + return env + } + return refreshed +} + +// decodeCeilingPayloadField reconstructs a typed replay field from its +// generically-decoded form. The original payload was built by placing typed +// Go values directly into a map[string]interface{} that the repository then +// json.Marshaled; marshaling this generic form back to bytes and unmarshaling +// into target recovers the original shape through the same struct tags. A +// malformed or absent value leaves target at its zero value. +func decodeCeilingPayloadField(raw interface{}, target interface{}) { + if raw == nil { + return + } + data, err := json.Marshal(raw) + if err != nil { + return + } + _ = json.Unmarshal(data, target) +} + +// replayCeilingLaunchStart replays an AC-42 "start" record by calling +// startTask with the payload seam1StartPayload originally captured. +func (s *Service) replayCeilingLaunchStart(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + var attachments []v1.MessageAttachment + decodeCeilingPayloadField(payload[metaKeyAttachments], &attachments) + var env map[string]string + decodeCeilingPayloadField(payload["env"], &env) + var route *executor.RouteOverride + decodeCeilingPayloadField(payload["route"], &route) + var additionalSkillSlugs []string + decodeCeilingPayloadField(payload["additional_skill_slugs"], &additionalSkillSlugs) + var entryOptions *workflowmove.EntryOptions + decodeCeilingPayloadField(payload["entry_options"], &entryOptions) + + env = s.remintCeilingLaunchCredentials(ctx, task.ID, env) + + opts := startTaskOptions{ + ProfileExplicit: boolField(payload, "profile_explicit"), + Env: env, + Route: route, + AdditionalSkillSlugs: additionalSkillSlugs, + EntryOptions: entryOptions, + WorkflowEntryID: int64Field(payload, "workflow_entry_id"), + // Origin must round-trip rather than be re-derived from auto_start: + // an AC-13d caller (Office-routed launch) passes autoStart=false with + // an explicit automatic Origin override, and re-deriving from + // auto_start alone would replay it as a manual override that bypasses + // the ceiling. + Origin: launchOrigin(stringField(payload, "origin")), + } + if spawnRaw, ok := payload["spawn_origin"].(map[string]interface{}); ok { + opts.SpawnOrigin = &SpawnOrigin{ + TaskID: stringField(spawnRaw, metaKeyTaskID), + SessionID: stringField(spawnRaw, metaKeySessionID), + SessionName: stringField(spawnRaw, "session_name"), + } + } + + execution, err := s.startTask( + ctx, task.ID, + stringField(payload, metaKeyAgentProfileID), + stringField(payload, "executor_id"), + stringField(payload, metaKeyExecutorProfile), + stringField(payload, "priority"), + stringField(payload, metaKeyPrompt), + stringField(payload, metaKeyWorkflowStepID), + boolField(payload, metaKeyPlanMode), + boolField(payload, "auto_start"), + attachments, + opts, + ) + // startTask reports a repeat refusal as ErrCeilingLaunchDeferred, not as + // (nil, nil) — ceilingReplayOutcomeFromExecution's execution==nil case + // would otherwise never fire and a still-refused replay would be + // misclassified as ceilingReplayFailed (a non-ceiling failure), skipping + // the AC-49g card-surface retry and logging a misleading warning. + if errors.Is(err, ErrCeilingLaunchDeferred) { + return ceilingReplayStillDeferred + } + return ceilingReplayOutcomeFromExecution(execution, err) +} + +// replayCeilingLaunchStartCreated replays an AC-42d "start_created" record. +func (s *Service) replayCeilingLaunchStartCreated(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + var attachments []v1.MessageAttachment + decodeCeilingPayloadField(payload[metaKeyAttachments], &attachments) + var references []v1.EntityReference + decodeCeilingPayloadField(payload["references"], &references) + + options := startCreatedSessionOptions{ + skipTaskDescriptionFallback: boolField(payload, "skip_task_description_fallback"), + promptAlreadyComposed: boolField(payload, "prompt_already_composed"), + retryPrompt: stringField(payload, "retry_prompt"), + canvasGuidanceResolved: boolField(payload, "canvas_guidance_resolved"), + includeCanvasGuidance: boolField(payload, "include_canvas_guidance"), + promptReferencesPrepared: boolField(payload, "prompt_references_prepared"), + preserveDirectPrompt: boolField(payload, "preserve_direct_prompt"), + } + + execution, err := s.startCreatedSession( + ctx, task.ID, + stringField(payload, metaKeySessionID), + stringField(payload, metaKeyAgentProfileID), + stringField(payload, metaKeyPrompt), + boolField(payload, "skip_message_record"), + boolField(payload, metaKeyPlanMode), + boolField(payload, "auto_start"), + attachments, + references, + stringField(payload, "prompt_reference_context"), + options, + ) + return ceilingReplayOutcomeFromExecution(execution, err) +} + +// replayCeilingLaunchPromptEnsure replays an AC-42 "prompt_ensure" record. +// promptTask has no seam-3 "deferred" return of its own; the caller (seam 3's +// own ensureSessionRunning gate) reports a refusal as a *seam3Refusal error, +// and an accept-then-fail-after dispatch as an acceptedPromptDispatchError — +// both distinguishable from an ordinary failure. +func (s *Service) replayCeilingLaunchPromptEnsure(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + var attachments []v1.MessageAttachment + decodeCeilingPayloadField(payload[metaKeyAttachments], &attachments) + + options := promptTaskOptions{ + lifecyclePrompt: boolField(payload, "lifecycle_prompt"), + preservePromptContext: boolField(payload, "preserve_prompt_context"), + reserveTurnUntilDispatch: boolField(payload, "reserve_turn_until_dispatch"), + requireNonterminalSession: boolField(payload, "require_nonterminal_session"), + promptAlreadyComposed: boolField(payload, "prompt_already_composed"), + fallbackLaunchPrompt: stringField(payload, "fallback_launch_prompt"), + fallbackRetryPrompt: stringField(payload, "fallback_retry_prompt"), + } + + _, err := s.promptTask( + ctx, task.ID, + stringField(payload, metaKeySessionID), + stringField(payload, metaKeyPrompt), + stringField(payload, sessionModelConfigKey), + boolField(payload, metaKeyPlanMode), + attachments, + boolField(payload, "dispatch_only"), + launchOriginAutomatic, + options, + ) + if err == nil { + return ceilingReplaySucceeded + } + if _, ok := isSeam3Refusal(err); ok { + return ceilingReplayStillDeferred + } + if accepted, ok := err.(interface{ DetachedResumeAccepted() bool }); ok && accepted.DetachedResumeAccepted() { + // agentctl accepted the prompt despite the reported error: the + // launch did happen, so this is not a ceiling matter any longer. + return ceilingReplaySucceeded + } + return ceilingReplayFailed +} + +// replayCeilingLaunchWorkflowStepEnsure replays an AC-42 "workflow_step_ensure" +// record. StartSessionForWorkflowStep reports its own still-deferred outcome +// via the distinguishable errSeam3WorkflowStepEnsureDeferred sentinel. +func (s *Service) replayCeilingLaunchWorkflowStepEnsure(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + err := s.StartSessionForWorkflowStep( + ctx, task.ID, + stringField(payload, metaKeySessionID), + stringField(payload, metaKeyWorkflowStepID), + ) + switch { + case err == nil: + return ceilingReplaySucceeded + case errors.Is(err, errSeam3WorkflowStepEnsureDeferred): + return ceilingReplayStillDeferred + default: + return ceilingReplayFailed + } +} + +// replayCeilingLaunchQueueDrainEnsure replays an AC-42 "queue_drain_ensure" +// record. tryEnsureExecution reports nothing at all — success is read back +// from the session's own state, the same signal AC-1 population counting +// itself uses. +func (s *Service) replayCeilingLaunchQueueDrainEnsure(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + sessionID := stringField(payload, metaKeySessionID) + s.tryEnsureExecution(ctx, sessionID, seam3CallShapeQueueDrain, launchOriginAutomatic, stringField(payload, "queued_message_id")) + + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return ceilingReplayStillDeferred + } + if isAC1SessionState(session.State) { + return ceilingReplaySucceeded + } + return ceilingReplayStillDeferred +} + +// replayCeilingLaunchResume replays an AC-42 "resume" record. +func (s *Service) replayCeilingLaunchResume(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + execution, err := s.ResumeTaskSessionWithOptions( + ctx, task.ID, + stringField(payload, metaKeySessionID), + executor.ResumeOptions{ + AllowBranchReplacement: boolField(payload, "allow_branch_replacement"), + Origin: string(launchOriginAutomatic), + }, + ) + return ceilingReplayOutcomeFromExecution(execution, err) +} + +// replayCeilingLaunchDynamicRelaunch replays an AC-42f "dynamic_relaunch" +// record. A repeated ceiling refusal remains deferred; a launch failure after +// admission is retained as a non-ceiling failure for the next sweep without +// being reported as another refusal. +func (s *Service) replayCeilingLaunchDynamicRelaunch(ctx context.Context, task *models.Task, payload map[string]interface{}) ceilingReplayOutcome { + data := watcher.AgentEventData{ + TaskID: stringField(payload, metaKeyTaskID), + SessionID: stringField(payload, metaKeySessionID), + AgentExecutionID: stringField(payload, "agent_execution_id"), + AgentProfileID: stringField(payload, metaKeyAgentProfileID), + } + switch s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, stringField(payload, "execution_profile_id"), launchOriginAutomatic) { + case dynamicRelaunchSucceeded: + return ceilingReplaySucceeded + case dynamicRelaunchDeferred: + return ceilingReplayStillDeferred + default: + return ceilingReplayFailed + } +} + +// ceilingReplayOutcomeFromExecution is the shared classification for the four +// entry points (start, start_created, resume — and, by the same contract, +// any future kind) that return (*executor.TaskExecution, error): a non-nil +// execution is success, a nil execution with a nil error is still-deferred +// (the seam's own gate re-persisted the record internally), and anything +// else is a non-ceiling failure. +func ceilingReplayOutcomeFromExecution(execution *executor.TaskExecution, err error) ceilingReplayOutcome { + switch { + case err != nil: + return ceilingReplayFailed + case execution == nil: + return ceilingReplayStillDeferred + default: + return ceilingReplaySucceeded + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_replay_test.go b/apps/backend/internal/orchestrator/ceiling_replay_test.go new file mode 100644 index 0000000000..90d8a01f1f --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_replay_test.go @@ -0,0 +1,332 @@ +package orchestrator + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/orchestrator/executor" + wfmodels "github.com/kandev/kandev/internal/workflow/models" + + "github.com/kandev/kandev/internal/task/models" + tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// --- pure unit tests ------------------------------------------------------- + +func TestStripCeilingRecordKeys_RemovesOnlyCeilingPrefixedKeys(t *testing.T) { + existing := map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: string(models.CeilingLaunchStart), + models.DeferredLaunchStartWhenUnblockedKey: true, + "wip_overflow_marker": true, + } + stripped := stripCeilingRecordKeys(existing) + require.NotContains(t, stripped, models.CeilingDeferredKey) + require.NotContains(t, stripped, models.CeilingLaunchKindKey) + require.Equal(t, true, stripped[models.DeferredLaunchStartWhenUnblockedKey]) + require.Equal(t, true, stripped["wip_overflow_marker"]) +} + +func TestStripCeilingRecordKeys_NonObjectValueYieldsEmptyMap(t *testing.T) { + stripped := stripCeilingRecordKeys("not-an-object") + require.NotNil(t, stripped) + require.Empty(t, stripped) + stripped = stripCeilingRecordKeys(nil) + require.NotNil(t, stripped) + require.Empty(t, stripped) +} + +func TestCeilingReplayOutcomeFromExecution(t *testing.T) { + require.Equal(t, ceilingReplaySucceeded, ceilingReplayOutcomeFromExecution(&executor.TaskExecution{}, nil)) + require.Equal(t, ceilingReplayStillDeferred, ceilingReplayOutcomeFromExecution(nil, nil)) + require.Equal(t, ceilingReplayFailed, ceilingReplayOutcomeFromExecution(nil, context.Canceled)) +} + +func TestSessionIDFromCeilingPayload(t *testing.T) { + require.Empty(t, sessionIDFromCeilingPayload(models.CeilingDeferral{ + Kind: models.CeilingLaunchStart, + Payload: map[string]interface{}{metaKeySessionID: "should-be-ignored"}, + })) + require.Equal(t, "session-x", sessionIDFromCeilingPayload(models.CeilingDeferral{ + Kind: models.CeilingLaunchResume, + Payload: map[string]interface{}{metaKeySessionID: "session-x"}, + })) +} + +// --- clearCeilingDeferredRecord -------------------------------------------- + +func TestClearCeilingDeferredRecord_PreservesCoexistingWIPIntent(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: "clear-task-a", Title: "t", State: v1.TaskStateInProgress, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + })) + + record := models.CeilingRecordKeys(models.CeilingDeferral{ + Kind: models.CeilingLaunchResume, + Payload: map[string]interface{}{metaKeySessionID: "session-a"}, + Origin: string(launchOriginAutomatic), + ReasonCode: ceilingReasonRefused, + QueuedAt: time.Now(), + }) + record[models.DeferredLaunchStartWhenUnblockedKey] = true + _, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "clear-task-a", tasksqlite.AbsentDeferredLaunch(), record) + require.NoError(t, err) + + svc.clearCeilingDeferredRecord(ctx, "clear-task-a") + + after := deferredLaunchOf(t, svc, "clear-task-a") + require.False(t, models.HasCeilingDeferredIntent(&models.Task{Metadata: after})) + require.Equal(t, true, after[models.DeferredLaunchStartWhenUnblockedKey], "clearing the ceiling half must not disturb the WIP-overflow half") +} + +// --- ListTasksWithCeilingDeferred ------------------------------------------- + +func TestListTasksWithCeilingDeferred_IncludesArchivedAndOrdersByID(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + + mkTask := func(id string) { + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: id, Title: "t", State: v1.TaskStateInProgress, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + })) + record := models.CeilingRecordKeys(models.CeilingDeferral{ + Kind: models.CeilingLaunchResume, + Payload: map[string]interface{}{metaKeySessionID: id + "-session"}, + Origin: string(launchOriginAutomatic), + ReasonCode: ceilingReasonRefused, + QueuedAt: time.Now(), + }) + _, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, id, tasksqlite.AbsentDeferredLaunch(), record) + require.NoError(t, err) + } + mkTask("list-task-b") + mkTask("list-task-a") + mkTask("list-task-c") + require.NoError(t, repo.ArchiveTask(ctx, "list-task-c")) + + // A task with no deferral at all must never appear. + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: "list-task-no-deferral", Title: "t", State: v1.TaskStateInProgress, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + })) + + lister, ok := svc.repo.(ceilingDeferredTaskLister) + require.True(t, ok, "the production sqlite repository must satisfy ceilingDeferredTaskLister") + tasks, err := lister.ListTasksWithCeilingDeferred(ctx) + require.NoError(t, err) + + var ids []string + for _, task := range tasks { + ids = append(ids, task.ID) + } + require.Equal(t, []string{"list-task-a", "list-task-b", "list-task-c"}, ids, + "archived tasks must still be returned, in id-ascending order") +} + +// --- evaluateCeilingDropReasons --------------------------------------------- + +func TestEvaluateCeilingDropReasons_ArchivedTask(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + task := &models.Task{ID: "drop-archived", Title: "t", State: v1.TaskStateInProgress, CreatedAt: time.Now(), UpdatedAt: time.Now()} + require.NoError(t, repo.CreateTask(ctx, task)) + require.NoError(t, repo.ArchiveTask(ctx, "drop-archived")) + task, err := repo.GetTask(ctx, "drop-archived") + require.NoError(t, err) + + reasonCode, _, drop := svc.evaluateCeilingDropReasons(ctx, task, models.CeilingDeferral{Kind: models.CeilingLaunchResume}) + require.True(t, drop) + require.Equal(t, ceilingReasonDroppedTaskIneligible, reasonCode) +} + +func TestEvaluateCeilingDropReasons_CancelledTask(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + task := &models.Task{ID: "drop-cancelled", Title: "t", State: v1.TaskStateInProgress, CreatedAt: time.Now(), UpdatedAt: time.Now()} + require.NoError(t, repo.CreateTask(ctx, task)) + require.NoError(t, repo.UpdateTaskState(ctx, "drop-cancelled", v1.TaskStateCancelled)) + task, err := repo.GetTask(ctx, "drop-cancelled") + require.NoError(t, err) + + reasonCode, _, drop := svc.evaluateCeilingDropReasons(ctx, task, models.CeilingDeferral{Kind: models.CeilingLaunchResume}) + require.True(t, drop) + require.Equal(t, ceilingReasonDroppedTaskIneligible, reasonCode) +} + +func TestEvaluateCeilingDropReasons_SessionNoLongerExists(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + task := &models.Task{ID: "drop-ghost-session", Title: "t", State: v1.TaskStateInProgress, CreatedAt: time.Now(), UpdatedAt: time.Now()} + require.NoError(t, repo.CreateTask(ctx, task)) + + deferral := models.CeilingDeferral{ + Kind: models.CeilingLaunchResume, + Payload: map[string]interface{}{metaKeySessionID: "session-that-was-deleted"}, + } + reasonCode, detail, drop := svc.evaluateCeilingDropReasons(ctx, task, deferral) + require.True(t, drop) + require.Equal(t, ceilingReasonDroppedTaskIneligible, reasonCode) + require.Contains(t, detail, "session no longer exists") +} + +func TestEvaluateCeilingDropReasons_StartKindStepNoLongerAutoStarts(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + stepGetter := newMockStepGetter() + svc.workflowStepGetter = stepGetter + ctx := context.Background() + + stepGetter.steps["step-no-auto-start"] = &wfmodels.WorkflowStep{ID: "step-no-auto-start"} + task := &models.Task{ + ID: "drop-step-no-auto-start", Title: "t", State: v1.TaskStateInProgress, + WorkflowStepID: "step-no-auto-start", CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + require.NoError(t, repo.CreateTask(ctx, task)) + + reasonCode, detail, drop := svc.evaluateCeilingDropReasons(ctx, task, models.CeilingDeferral{Kind: models.CeilingLaunchStart, Payload: map[string]interface{}{}}) + require.True(t, drop) + require.Equal(t, ceilingReasonDroppedTaskIneligible, reasonCode) + require.Contains(t, detail, "no longer auto-starts") +} + +func TestEvaluateCeilingDropReasons_OfficeStartIgnoresWorkflowAutoStartEligibility(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + stepGetter := newMockStepGetter() + svc.workflowStepGetter = stepGetter + ctx := context.Background() + + stepGetter.steps["office-step-no-auto-start"] = &wfmodels.WorkflowStep{ID: "office-step-no-auto-start"} + task := &models.Task{ + ID: "keep-office-start", Title: "Office task", State: v1.TaskStateInProgress, + WorkflowStepID: "office-step-no-auto-start", IsFromOffice: true, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + require.NoError(t, repo.CreateTask(ctx, task)) + + _, detail, drop := svc.evaluateCeilingDropReasons(ctx, task, models.CeilingDeferral{ + Kind: models.CeilingLaunchStart, Payload: map[string]interface{}{}, + }) + require.False(t, drop, "an Office automatic start is not governed by workflow auto-start eligibility") + require.Empty(t, detail) +} + +func TestEvaluateCeilingDropReasons_NonStartKindIgnoresStepAutoStart(t *testing.T) { + // A resume/prompt_ensure/etc. record must not be dropped just because the + // task's current step no longer auto-starts (AC-17b(d) is scoped to "start"). + svc, repo := newServiceWithRealRepo(t) + stepGetter := newMockStepGetter() + svc.workflowStepGetter = stepGetter + ctx := context.Background() + + stepGetter.steps["step-no-auto-start"] = &wfmodels.WorkflowStep{ID: "step-no-auto-start"} + task := &models.Task{ + ID: "keep-resume-despite-step", Title: "t", State: v1.TaskStateInProgress, + WorkflowStepID: "step-no-auto-start", CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + require.NoError(t, repo.CreateTask(ctx, task)) + + _, _, drop := svc.evaluateCeilingDropReasons(ctx, task, models.CeilingDeferral{Kind: models.CeilingLaunchResume, Payload: map[string]interface{}{}}) + require.False(t, drop) +} + +// --- end-to-end drain behaviour, through the real seam 2 gate --------------- + +func TestDrainDeferredCeilingLaunches_StillDeferredThenSucceedsOnceSlotFrees(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "drain-task-a", "drain-session-a", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "drain-session-a", "drain-task-a", "exec-a") + seedTaskAndSession(t, repo, "drain-task-b", "drain-session-b", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "drain-session-b", "drain-task-b", "exec-b") + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "drain-task-a", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "drain-task-b", v1.TaskStateInProgress) + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + events := &capturingTaskEvents{} + svc.SetTaskEventPublisher(events) + + // Occupy the only slot directly against the controller (session-a "holds" + // the ceiling's one unit for this test, without needing a real launch). + decision := svc.sessionCeiling.admit(ctx, admissionRequest{ + taskID: "drain-task-a", sessionID: "drain-session-a", origin: launchOriginAutomatic, seam: "test-setup", + }) + require.True(t, decision.admitted) + + // Task b's automatic launch is refused and deferred by the real seam 2 gate. + exec, err := svc.StartCreatedSession(ctx, "drain-task-b", "drain-session-b", "profile-1", "go", false, false, true, nil, nil) + require.NoError(t, err) + require.Nil(t, exec) + require.NotNil(t, deferredLaunchOf(t, svc, "drain-task-b")) + + // A drain pass while the slot is still occupied leaves the record in place. + svc.drainDeferredCeilingLaunches(ctx) + record := deferredLaunchOf(t, svc, "drain-task-b") + require.NotNil(t, record, "a still-refused replay must not clear the record") + require.Equal(t, true, record[models.CeilingDeferredKey]) + + agentMgr.mu.Lock() + callsWhileFull := len(agentMgr.setExecutionDescriptionCalls) + agentMgr.mu.Unlock() + require.Zero(t, callsWhileFull, "the still-deferred replay must not have dispatched an agent") + + // Free the slot and drain again: the replay must now succeed and clear the record. + svc.sessionCeiling.release("drain-session-a") + svc.drainDeferredCeilingLaunches(ctx) + + require.False(t, models.HasCeilingDeferredIntent(&models.Task{Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: deferredLaunchOf(t, svc, "drain-task-b"), + }}), "a successful replay must clear the ceiling record") + + published := events.last() + require.NotNil(t, published, "a successful replay must publish task.updated so the WS-driven UI clears the queued state") + require.Equal(t, "drain-task-b", published.ID) + require.False(t, models.HasCeilingDeferredIntent(published), + "the published task after a successful replay must reflect the cleared record, not a stale queued snapshot") + agentMgr.mu.Lock() + callsAfterFree := len(agentMgr.setExecutionDescriptionCalls) + agentMgr.mu.Unlock() + require.Equal(t, 1, callsAfterFree, "the now-admitted replay must have dispatched exactly one agent") +} + +func TestDrainDeferredCeilingLaunches_DropsArchivedTaskAndWritesCardNote(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "drain-archived-task", "drain-archived-session", models.TaskSessionStateCreated) + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "drain-archived-task", v1.TaskStateInProgress) + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, &mockAgentManager{repoForExecutionLookup: repo}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + msgCreator := &mockMessageCreator{} + svc.messageCreator = msgCreator + + record := models.CeilingRecordKeys(models.CeilingDeferral{ + Kind: models.CeilingLaunchResume, + Payload: map[string]interface{}{metaKeySessionID: "drain-archived-session"}, + Origin: string(launchOriginAutomatic), + ReasonCode: ceilingReasonRefused, + QueuedAt: time.Now(), + }) + _, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "drain-archived-task", tasksqlite.AbsentDeferredLaunch(), record) + require.NoError(t, err) + require.NoError(t, repo.ArchiveTask(ctx, "drain-archived-task")) + + svc.drainDeferredCeilingLaunches(ctx) + + require.False(t, models.HasCeilingDeferredIntent(&models.Task{Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: deferredLaunchOf(t, svc, "drain-archived-task"), + }}), "a dropped record must be cleared") + msgCreator.mu.Lock() + attempts := msgCreator.sessionMessageAttempts + msgCreator.mu.Unlock() + require.Equal(t, 1, attempts, "dropping a record for a session that still exists must write one card note") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam1_integration_test.go b/apps/backend/internal/orchestrator/ceiling_seam1_integration_test.go new file mode 100644 index 0000000000..b9dc79e3f5 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam1_integration_test.go @@ -0,0 +1,121 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +// TestStartTask_SecondAutomaticLaunchOverCeilingIsDeferred pins AC-4a/AC-11 at +// the real seam: StartTask itself, gated before claimDeferredLaunchForStart, +// must refuse the second automatic launch once the ceiling's single slot is +// already held by the first, and must not create a session or dispatch an +// agent for the refused task. +func TestStartTask_SecondAutomaticLaunchOverCeilingIsDeferred(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam1-it-first", "seam1-it-first-session", models.TaskSessionStateCompleted) + seedTaskAndSession(t, repo, "seam1-it-second", "seam1-it-second-session", models.TaskSessionStateCompleted) + + var launches int + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam1-it-first", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "seam1-it-second", v1.TaskStateInProgress) + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + launches++ + return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil + }, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + exec1, err := svc.StartTask(ctx, "seam1-it-first", "profile-1", "", "", "", "go", "", false, true, nil) + require.NoError(t, err) + require.NotNil(t, exec1) + require.Equal(t, 1, launches) + + exec2, err := svc.StartTask(ctx, "seam1-it-second", "profile-1", "", "", "", "go", "", false, true, nil) + require.ErrorIs(t, err, ErrCeilingLaunchDeferred, + "a refused automatic launch must report a distinguishable deferred error, not a silent no-op") + require.Nil(t, exec2, "a refused automatic launch must not produce a launched execution") + require.Equal(t, 1, launches, "the refused launch must not reach the agent manager") + + record := deferredLaunchOf(t, svc, "seam1-it-second") + require.NotNil(t, record, "a ceiling refusal must persist a deferred_launch record") + require.Equal(t, true, record[models.CeilingDeferredKey]) +} + +// TestStartTask_ManualLaunchOverCeilingIsAdmitted pins AC-14: a manual launch +// at the ceiling is always admitted, never deferred, through the real +// StartTask entry point. +func TestStartTask_ManualLaunchOverCeilingIsAdmitted(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam1-it-manual-first", "seam1-it-manual-first-session", models.TaskSessionStateCompleted) + seedTaskAndSession(t, repo, "seam1-it-manual-second", "seam1-it-manual-second-session", models.TaskSessionStateCompleted) + + var launches int + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam1-it-manual-first", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "seam1-it-manual-second", v1.TaskStateInProgress) + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + launches++ + return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil + }, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svc.StartTask(ctx, "seam1-it-manual-first", "profile-1", "", "", "", "go", "", false, true, nil) + require.NoError(t, err) + require.Equal(t, 1, launches) + + // autoStart=false: a direct, manual start. AC-14 requires this be admitted + // even though the ceiling's only slot is already held. + exec2, err := svc.StartTask(ctx, "seam1-it-manual-second", "profile-1", "", "", "", "go", "", false, false, nil) + require.NoError(t, err) + require.NotNil(t, exec2, "a manual launch must be admitted even over the ceiling") + require.Equal(t, 2, launches) + + record := deferredLaunchOf(t, svc, "seam1-it-manual-second") + require.Nil(t, record, "a manual override must never write a ceiling_deferred record") +} + +// TestStartTask_FailedLaunchAfterSessionCreationReleasesTheReservation pins +// the reservation-leak fix: seam1's reservation is rebound onto the new +// session partway through StartTask, well before the launch attempt that can +// still fail. A failure at that point must still release the reservation +// through the real StartTask entry point, not just at the seam1Reservation +// unit level — otherwise the ceiling's only slot stays phantom-held until the +// stale-reservation sweep eventually reclaims it. +func TestStartTask_FailedLaunchAfterSessionCreationReleasesTheReservation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam1-it-leak", "seam1-it-leak-session", models.TaskSessionStateCompleted) + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam1-it-leak", v1.TaskStateInProgress) + launchErr := errors.New("workspace launch failed") + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + return nil, launchErr + }, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svc.StartTask(ctx, "seam1-it-leak", "profile-1", "", "", "", "go", "", false, true, nil) + require.Error(t, err, "the forced launch failure must still surface") + + population, popErr := svc.sessionCeiling.population(ctx) + require.NoError(t, popErr) + require.Equal(t, 0, population, + "a launch failure after session creation must release the seam1 reservation, not leak it") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam2.go b/apps/backend/internal/orchestrator/ceiling_seam2.go new file mode 100644 index 0000000000..52787c27cd --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam2.go @@ -0,0 +1,58 @@ +package orchestrator + +import ( + "context" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// seam2StartCreatedPayload builds the AC-42d "start_created" replay row from +// startCreatedSession's own frame, at the point the gate is consulted — before +// the profile override, prompt composition, and workflow/plan transforms below +// it, so the recorded payload is the caller's original request. +func seam2StartCreatedPayload( + sessionID, agentProfileID, prompt string, + skipMessageRecord, planMode, autoStart bool, + attachments []v1.MessageAttachment, + references []v1.EntityReference, + promptReferenceContext string, + options startCreatedSessionOptions, +) map[string]interface{} { + return map[string]interface{}{ + metaKeySessionID: sessionID, + metaKeyAgentProfileID: agentProfileID, + metaKeyPrompt: prompt, + metaKeyPlanMode: planMode, + metaKeyAttachments: attachments, + "references": references, + "prompt_reference_context": promptReferenceContext, + "skip_message_record": skipMessageRecord, + "auto_start": autoStart, + "skip_task_description_fallback": options.skipTaskDescriptionFallback, + "prompt_already_composed": options.promptAlreadyComposed, + "retry_prompt": options.retryPrompt, + "canvas_guidance_resolved": options.canvasGuidanceResolved, + "include_canvas_guidance": options.includeCanvasGuidance, + "prompt_references_prepared": options.promptReferencesPrepared, + "preserve_direct_prompt": options.preserveDirectPrompt, + } +} + +// admitOrDeferSeam2 is Service.startCreatedSession's AC-4d gate: consulted +// before its own claimDeferredLaunchForStart, keyed by the session that already +// exists (AC-4d — unlike seam 1, there is no launch-scoped window here). On +// refusal it persists the ceiling_deferred record from the caller-supplied +// AC-42d "start_created" payload and reports the decision so the caller returns +// without claiming the launch intent it holds or dispatching an agent. +// +// The returned reservation is non-nil only when the launch is admitted. The +// caller is responsible for deferring reservation.releaseIfNotConsumed, for +// calling reservation.rekeyToSession if the on_turn_start redirect switches +// sessions, and for calling reservation.consume once the launch succeeds. +func (s *Service) admitOrDeferSeam2( + ctx context.Context, taskID, sessionID string, origin launchOrigin, startPayload map[string]interface{}, +) (*sessionKeyedCeilingReservation, bool, error) { + return s.admitOrDeferSessionKeyedLaunch(ctx, taskID, sessionID, origin, "startCreatedSession", + models.CeilingLaunchStartCreated, startPayload, "the launch could not be admitted or recorded") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam2_integration_test.go b/apps/backend/internal/orchestrator/ceiling_seam2_integration_test.go new file mode 100644 index 0000000000..0e7e18c410 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam2_integration_test.go @@ -0,0 +1,130 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +// TestStartCreatedSession_SecondAutomaticLaunchOverCeilingIsDeferred pins +// AC-4d/AC-11 at the real seam: StartCreatedSession itself, gated before +// claimDeferredLaunchForStart, must refuse the second automatic launch once +// the ceiling's single slot is already held, and must not dispatch an agent +// for the refused session. +func TestStartCreatedSession_SecondAutomaticLaunchOverCeilingIsDeferred(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam2-it-task-a", "seam2-it-session-a", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "seam2-it-session-a", "seam2-it-task-a", "exec-a") + seedTaskAndSession(t, repo, "seam2-it-task-b", "seam2-it-session-b", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "seam2-it-session-b", "seam2-it-task-b", "exec-b") + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam2-it-task-a", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "seam2-it-task-b", v1.TaskStateInProgress) + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + exec1, err := svc.StartCreatedSession(ctx, "seam2-it-task-a", "seam2-it-session-a", "profile-1", "go", false, false, true, nil, nil) + require.NoError(t, err) + require.NotNil(t, exec1) + + agentMgr.mu.Lock() + firstCalls := len(agentMgr.setExecutionDescriptionCalls) + agentMgr.mu.Unlock() + require.Equal(t, 1, firstCalls) + + exec2, err := svc.StartCreatedSession(ctx, "seam2-it-task-b", "seam2-it-session-b", "profile-1", "go", false, false, true, nil, nil) + require.NoError(t, err) + require.Nil(t, exec2, "a refused automatic launch must not produce a launched execution") + + agentMgr.mu.Lock() + secondCalls := len(agentMgr.setExecutionDescriptionCalls) + agentMgr.mu.Unlock() + require.Equal(t, 1, secondCalls, "the refused launch must not reach the agent manager") + + record := deferredLaunchOf(t, svc, "seam2-it-task-b") + require.NotNil(t, record, "a ceiling refusal must persist a deferred_launch record") + require.Equal(t, true, record[models.CeilingDeferredKey]) + require.Equal(t, string(models.CeilingLaunchStartCreated), record[models.CeilingLaunchKindKey]) +} + +// TestStartCreatedSession_ManualLaunchOverCeilingIsAdmitted pins AC-14 through +// the real StartCreatedSession entry point. +func TestStartCreatedSession_ManualLaunchOverCeilingIsAdmitted(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam2-it-manual-a", "seam2-it-manual-session-a", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "seam2-it-manual-session-a", "seam2-it-manual-a", "exec-a") + seedTaskAndSession(t, repo, "seam2-it-manual-b", "seam2-it-manual-session-b", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "seam2-it-manual-session-b", "seam2-it-manual-b", "exec-b") + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam2-it-manual-a", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "seam2-it-manual-b", v1.TaskStateInProgress) + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svc.StartCreatedSession(ctx, "seam2-it-manual-a", "seam2-it-manual-session-a", "profile-1", "go", false, false, true, nil, nil) + require.NoError(t, err) + + // autoStart=false: a direct, manual start. AC-14 requires this be admitted + // even though the ceiling's only slot is already held. + exec2, err := svc.StartCreatedSession(ctx, "seam2-it-manual-b", "seam2-it-manual-session-b", "profile-1", "go", false, false, false, nil, nil) + require.NoError(t, err) + require.NotNil(t, exec2, "a manual launch must be admitted even over the ceiling") + + record := deferredLaunchOf(t, svc, "seam2-it-manual-b") + require.Nil(t, record, "a manual override must never write a ceiling_deferred record") +} + +// TestStartCreatedSession_ManualLaunchOverCeilingIsAuditedEvenWhenTheLaunchFails +// pins AC-14/AC-53's unconditional audit contract: the record must be written +// once the reservation is admitted onto a real session, not only once the +// subsequent launch attempt has also succeeded. A launch failure between +// admission and agent start must not erase the only evidence that a ceiling +// override happened. +func TestStartCreatedSession_ManualLaunchOverCeilingIsAuditedEvenWhenTheLaunchFails(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam2-it-fail-a", "seam2-it-fail-session-a", models.TaskSessionStateCreated) + seedTaskAndSession(t, repo, "seam2-it-fail-b", "seam2-it-fail-session-b", models.TaskSessionStateCreated) + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "seam2-it-fail-a", v1.TaskStateInProgress) + seedMockTaskState(taskRepo, "seam2-it-fail-b", v1.TaskStateInProgress) + launchErr := errors.New("workspace launch failed") + agentMgr := &mockAgentManager{ + repoForExecutionLookup: repo, + launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + if req.SessionID == "seam2-it-fail-session-b" { + return nil, launchErr + } + return &executor.LaunchAgentResponse{AgentExecutionID: "mock-launch-" + req.SessionID}, nil + }, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + _, err := svc.StartCreatedSession(ctx, "seam2-it-fail-a", "seam2-it-fail-session-a", "profile-1", "go", false, false, true, nil, nil) + require.NoError(t, err) + + // autoStart=false: a manual start admitted over the ceiling whose launch + // then fails. + _, err = svc.StartCreatedSession(ctx, "seam2-it-fail-b", "seam2-it-fail-session-b", "profile-1", "go", false, false, false, nil, nil) + require.Error(t, err, "the forced launch failure must still surface") + + session, err := repo.GetTaskSession(ctx, "seam2-it-fail-session-b") + require.NoError(t, err) + require.NotNil(t, session.Metadata[ceilingManualOverrideMetadataKey], + "a manual override must be audited even when its launch subsequently fails") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam2_test.go b/apps/backend/internal/orchestrator/ceiling_seam2_test.go new file mode 100644 index 0000000000..d74499048c --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam2_test.go @@ -0,0 +1,160 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestAdmitOrDeferSeam2AdmitsUnderCeiling covers the ordinary case: population +// below the ceiling, admitted, no record written, reservation keyed by the +// session that already exists. +func TestAdmitOrDeferSeam2AdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam2-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, deferred, err := svc.admitOrDeferSeam2(ctx, "seam2-admit", "seam2-admit-session", launchOriginAutomatic, map[string]interface{}{"prompt": "p"}) + if err != nil { + t.Fatalf("admitOrDeferSeam2: %v", err) + } + if deferred { + t.Fatal("launch was deferred while under the ceiling") + } + if reservation == nil || reservation.key != "seam2-admit-session" { + t.Fatalf("reservation not keyed by the existing session id: %+v", reservation) + } + + record := deferredLaunchOf(t, svc, "seam2-admit") + if record != nil { + t.Fatalf("a deferred_launch record was written for an admitted launch: %+v", record) + } +} + +// TestAdmitOrDeferSeam2DefersAutomaticOverCeiling covers AC-11/AC-42d: an +// automatic launch at the ceiling is refused and its start_created payload is +// recorded, not dropped. +func TestAdmitOrDeferSeam2DefersAutomaticOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam2-first", "seam2-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + + if _, deferred, err := svc.admitOrDeferSeam2(ctx, "seam2-first", "seam2-first-session", launchOriginAutomatic, map[string]interface{}{"prompt": "first"}); err != nil || deferred { + t.Fatalf("first launch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam2(ctx, "seam2-second", "seam2-second-session", launchOriginAutomatic, map[string]interface{}{"prompt": "second"}) + if err != nil { + t.Fatalf("admitOrDeferSeam2: %v", err) + } + if !deferred { + t.Fatal("second automatic launch over the ceiling was admitted, want deferred") + } + if reservation != nil { + t.Fatal("a deferred launch must not hold a reservation") + } + + record := deferredLaunchOf(t, svc, "seam2-second") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the refused launch's payload was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchStartCreated) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchStartCreated) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "second" { + t.Fatalf("the recorded payload does not match the refused launch: %+v", record) + } +} + +// TestAdmitOrDeferSeam2AdmitsManualOverCeiling covers AC-14: a manual launch is +// always admitted, never deferred, even at the ceiling. +func TestAdmitOrDeferSeam2AdmitsManualOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam2-manual-first", "seam2-manual-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam2(ctx, "seam2-manual-first", "seam2-manual-first-session", launchOriginAutomatic, map[string]interface{}{"prompt": "first"}); err != nil || deferred { + t.Fatalf("first launch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam2(ctx, "seam2-manual-second", "seam2-manual-second-session", launchOriginManual, map[string]interface{}{"prompt": "second"}) + if err != nil { + t.Fatalf("admitOrDeferSeam2: %v", err) + } + if deferred { + t.Fatal("a manual launch was deferred; AC-14 requires it always be admitted") + } + if reservation == nil { + t.Fatal("a manual override still consumes a reservation") + } + if record := deferredLaunchOf(t, svc, "seam2-manual-second"); record != nil { + t.Fatalf("a manual override must never write a ceiling_deferred record: %+v", record) + } +} + +// TestSeam2ReservationRekeyMovesThePopulationUnit pins AC-4e: the redirect to +// an already-active session must move the reservation, not release-then-admit +// a second unit for the same launch. +func TestSeam2ReservationRekeyMovesThePopulationUnit(t *testing.T) { + ctx := context.Background() + controller := newSessionCeilingController(1, nil, nil) + decision := controller.admit(ctx, admissionRequest{taskID: "t", sessionID: "gated-session", origin: launchOriginAutomatic, seam: "startCreatedSession"}) + if !decision.admitted { + t.Fatal("expected admission") + } + reservation := &sessionKeyedCeilingReservation{controller: controller, key: decision.reservationKey} + + reservation.rekeyToSession(ctx, "redirect-session") + if reservation.key != "redirect-session" { + t.Fatalf("reservation key = %q, want %q", reservation.key, "redirect-session") + } + + population, err := controller.population(ctx) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 1 { + t.Fatalf("population = %d, want 1 (rekey must not consume a second unit)", population) + } + + // A second launch should now be refused: the ceiling's only slot is still + // held, under the redirected key. + second := controller.admit(ctx, admissionRequest{taskID: "t2", sessionID: "other-session", origin: launchOriginAutomatic, seam: "startCreatedSession"}) + if second.admitted { + t.Fatal("the ceiling's slot should still be held after rekey") + } + + reservation.consume() + reservation.releaseIfNotConsumed() + population, err = controller.population(ctx) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 1 { + t.Fatalf("population = %d, want 1 (consumed reservation must not be released)", population) + } +} + +// TestAdmitOrDeferSeam2ReportsWriteFailure covers AC-45: a refusal that cannot +// be persisted must not be swallowed. +func TestAdmitOrDeferSeam2ReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + ctx := context.Background() + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "startCreatedSession"}) + + _, _, err := svc.admitOrDeferSeam2(ctx, "missing-task", "missing-session", launchOriginAutomatic, map[string]interface{}{"prompt": "p"}) + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam3.go b/apps/backend/internal/orchestrator/ceiling_seam3.go new file mode 100644 index 0000000000..6a89dbe4a3 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam3.go @@ -0,0 +1,231 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// seam3Refusal is ensureSessionRunning's distinguishable ceiling-refusal +// sentinel (AC-47): a plain error cannot be told apart from a materialization +// failure, so each of seam 3's four call shapes needs to recognize this exact +// type to choose its own disposition. ensureSessionRunning's own signature +// carries no replay fields, so it never persists a deferral itself — it only +// decides admit/refuse and reports the refusal for its caller to record. +type seam3Refusal struct { + reasonCode string + // population, populationKnown and ceiling are the admission decision's own + // reading, carried so a caller-side disposition can pass them on to + // deferCeilingRefusal for the AC-49 card carrier's content. + population int + populationKnown bool + ceiling int + // deferred is set by the caller-side disposition once a ceiling_deferred + // record has actually been written for this refusal (AC-47c2): a caller + // with two possible dispositions (promptTask's prompt_ensure vs AC-47c1's + // refuse-unchanged) inspects it to decide whether it or the ceiling now + // owns restoration. + deferred bool +} + +func (e *seam3Refusal) Error() string { + return fmt.Sprintf("session ceiling refused seam 3 admission (%s)", e.reasonCode) +} + +func isSeam3Refusal(err error) (*seam3Refusal, bool) { + var refusal *seam3Refusal + if errors.As(err, &refusal) { + return refusal, true + } + return nil, false +} + +// errSeam3WorkflowStepEnsureDeferred is returned by StartSessionForWorkflowStep +// when its own AC-47f pre-consultation is refused: the workflow_step_ensure +// record has already been written, and (per AC-47f1) advanceTaskWorkflowStep +// was never called, so the card is left exactly where it was. +var errSeam3WorkflowStepEnsureDeferred = errors.New("session ceiling deferred the workflow step launch") + +// seam3CallShape distinguishes tryEnsureExecution's two production callers +// (AC-47e): an explicit token supplied by each caller, never inferred from +// context or goroutine identity. +type seam3CallShape string + +const ( + // seam3CallShapeViewing is EnsureSession's call shape (AC-47a): nothing to + // replay and nobody waiting, so a refusal is swallowed, not deferred. + seam3CallShapeViewing seam3CallShape = "viewing" + // seam3CallShapeQueueDrain is scheduleAutoResumeForWorkflowQueue's call + // shape (AC-47d): a refusal defers, carrying the queued message id. + seam3CallShapeQueueDrain seam3CallShape = "queue_drain" +) + +// admitSeam3 is ensureSessionRunning's own gate: consulted once, after the +// already-running early return and before the first attemptColdResume +// (AC-4e1 — the loop's retry is recovering an already-admitted launch, not +// requesting a new one). +func (s *Service) admitSeam3( + ctx context.Context, taskID, sessionID string, origin launchOrigin, +) (*sessionKeyedCeilingReservation, *seam3Refusal) { + decision := s.sessionCeiling.admit(ctx, admissionRequest{ + taskID: taskID, sessionID: sessionID, origin: origin, seam: "ensureSessionRunning", + }) + if decision.admitted { + return &sessionKeyedCeilingReservation{ + controller: s.sessionCeiling, key: decision.reservationKey, + manualOverride: decision.manualOverride, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + }, nil + } + return nil, &seam3Refusal{ + reasonCode: decision.reasonCode, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + } +} + +// admitOrDeferWorkflowStepEnsure is StartSessionForWorkflowStep's AC-47f/AC-47f1 +// pre-consultation: an ordinary admission request for the session id it +// already holds, consulted before advanceTaskWorkflowStep mutates the task. +// On refusal it writes the workflow_step_ensure record itself (AC-47's +// caller-owns-the-disposition split extends to this pre-check, not only to +// the in-function gate) so the caller can return without ever calling +// advanceTaskWorkflowStep. +func (s *Service) admitOrDeferWorkflowStepEnsure( + ctx context.Context, taskID, sessionID, workflowStepID string, +) (reservation *sessionKeyedCeilingReservation, deferred bool, err error) { + decision := s.sessionCeiling.admit(ctx, admissionRequest{ + taskID: taskID, sessionID: sessionID, origin: launchOriginAutomatic, seam: "startSessionForWorkflowStepPreConsult", + }) + if decision.admitted { + return &sessionKeyedCeilingReservation{ + controller: s.sessionCeiling, key: decision.reservationKey, + manualOverride: decision.manualOverride, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + }, false, nil + } + payload := map[string]interface{}{ + metaKeySessionID: sessionID, + metaKeyWorkflowStepID: workflowStepID, + } + if err := s.deferCeilingRefusal(ctx, taskID, sessionID, models.CeilingLaunchWorkflowStepEnsure, payload, decision.reasonCode, + decision.population, decision.populationKnown, decision.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the workflow step launch could not be admitted or recorded", + zap.String("task_id", taskID), zap.String("session_id", sessionID), + zap.String("workflow_step_id", workflowStepID), zap.Error(err)) + return nil, false, err + } + return nil, true, nil +} + +// deferSeam3QueueDrainRefusal is tryEnsureExecution's AC-47d disposition for +// its queue-drain call shape: the prompt, attachments and metadata are +// already durable in the message queue, so the record needs only the +// session id and the queued message's own id (AC-47d1's already-drained +// check at retry time is undecidable without it). +func (s *Service) deferSeam3QueueDrainRefusal(ctx context.Context, taskID, sessionID, queuedMessageID string, refusal *seam3Refusal) { + payload := map[string]interface{}{ + metaKeySessionID: sessionID, + "queued_message_id": queuedMessageID, + } + if err := s.deferCeilingRefusal(ctx, taskID, sessionID, models.CeilingLaunchQueueDrainEnsure, payload, refusal.reasonCode, + refusal.population, refusal.populationKnown, refusal.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the workflow queue drain could not be recorded", + zap.String("task_id", taskID), zap.String("session_id", sessionID), + zap.String("queued_message_id", queuedMessageID), zap.Error(err)) + } +} + +// seam3PromptEnsureNonReconstructableOptionsSet reports AC-47c1: whether +// options carries any AC-47c(a) field at other than its zero value. Function +// values and in-flight claim or session identity are meaningless outside the +// originating call, and every production caller that sets one already holds +// its own failure path, so a refusal here is returned to the caller unchanged +// rather than deferred. +func seam3PromptEnsureNonReconstructableOptionsSet(options promptTaskOptions) bool { + return options.afterClaim != nil || + options.onAccepted != nil || + options.claimEntryID != "" || + options.expectedCurrentTurnID != "" || + options.promptDispatchRecovery != nil || + options.expectedSessionIdentity != nil || + options.afterDispatch != nil || + options.beforeDispatch != nil || + options.disableDispatchRetry +} + +// seam3PromptEnsurePayload builds the AC-42 "prompt_ensure" replay row from +// promptTask's own frame, at the point the gate is consulted. +func seam3PromptEnsurePayload( + sessionID, prompt, model string, planMode bool, attachments []v1.MessageAttachment, + dispatchOnly bool, options promptTaskOptions, +) map[string]interface{} { + return map[string]interface{}{ + metaKeySessionID: sessionID, + metaKeyPrompt: prompt, + sessionModelConfigKey: model, + metaKeyPlanMode: planMode, + metaKeyAttachments: attachments, + "dispatch_only": dispatchOnly, + "lifecycle_prompt": options.lifecyclePrompt, + "preserve_prompt_context": options.preservePromptContext, + "reserve_turn_until_dispatch": options.reserveTurnUntilDispatch, + "require_nonterminal_session": options.requireNonterminalSession, + "prompt_already_composed": options.promptAlreadyComposed, + "fallback_launch_prompt": options.fallbackLaunchPrompt, + "fallback_retry_prompt": options.fallbackRetryPrompt, + } +} + +// disposeSeam3PromptEnsureRefusal is promptTask's AC-47/AC-47c1/AC-47c2 +// disposition for a seam-3 refusal: writes a prompt_ensure record from its own +// frame UNLESS the call carries a non-reconstructable AC-47c(a) field, in +// which case the refusal is returned to the caller unchanged (deferred=false) +// so the caller's own restoration path owns it, exactly as AC-47d already +// argues for queue_drain_ensure. The two dispositions are distinguishable via +// the returned bool, per AC-47c2's requirement that a caller not infer which +// happened from the presence of a record it would have to re-read. +func (s *Service) disposeSeam3PromptEnsureRefusal( + ctx context.Context, taskID, sessionID, prompt, model string, planMode bool, + attachments []v1.MessageAttachment, dispatchOnly bool, options promptTaskOptions, refusal *seam3Refusal, +) error { + if seam3PromptEnsureNonReconstructableOptionsSet(options) { + return nil + } + payload := seam3PromptEnsurePayload(sessionID, prompt, model, planMode, attachments, dispatchOnly, options) + if err := s.deferCeilingRefusal(ctx, taskID, sessionID, models.CeilingLaunchPromptEnsure, payload, refusal.reasonCode, + refusal.population, refusal.populationKnown, refusal.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the prompt could not be admitted or recorded", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err)) + return err + } + refusal.deferred = true + return nil +} + +// classifyEnsureSessionRunningFailureForPrompt is promptTask's single error +// path out of ensureSessionRunning: a seam-3 refusal is disposed of via +// disposeSeam3PromptEnsureRefusal (AC-47/AC-47c1/AC-47c2) and returned to the +// caller unchanged either way, and every other failure keeps promptTask's +// pre-existing classification (runtime-unavailable vs. a generic wrap). +func (s *Service) classifyEnsureSessionRunningFailureForPrompt( + ctx context.Context, taskID, sessionID, prompt, model string, planMode bool, + attachments []v1.MessageAttachment, dispatchOnly bool, options promptTaskOptions, err error, +) error { + if refusal, ok := isSeam3Refusal(err); ok { + if deferErr := s.disposeSeam3PromptEnsureRefusal( + ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, options, refusal, + ); deferErr != nil { + return deferErr + } + return err + } + if errors.Is(err, errSessionAwaitingRuntimeLaunch) { + return fmt.Errorf("%w: failed to ensure session is running: %w", ErrSessionRuntimeUnavailable, err) + } + return fmt.Errorf("failed to ensure session is running: %w", err) +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam3_integration_test.go b/apps/backend/internal/orchestrator/ceiling_seam3_integration_test.go new file mode 100644 index 0000000000..3a23e996de --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam3_integration_test.go @@ -0,0 +1,312 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + wfmodels "github.com/kandev/kandev/internal/workflow/models" + "github.com/stretchr/testify/require" +) + +// TestPromptTask_AutomaticOverCeilingDefersPromptEnsure pins AC-47/AC-47c2 at +// the real promptTask entry point: an automatic prompt whose ensureSessionRunning +// call is refused by the ceiling gets a prompt_ensure record written from +// promptTask's own frame, and the returned error is the seam-3 sentinel marked +// deferred. +func TestPromptTask_AutomaticOverCeilingDefersPromptEnsure(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + _, err := svc.promptTask(ctx, "task1", "session1", "hello", "", false, nil, false, launchOriginAutomatic, promptTaskOptions{}) + if err == nil { + t.Fatal("expected the prompt to be refused by the session ceiling") + } + refusal, ok := isSeam3Refusal(err) + if !ok { + t.Fatalf("expected a seam3Refusal, got: %v", err) + } + if !refusal.deferred { + t.Fatal("a reconstructable prompt refusal must be marked deferred") + } + + record := deferredLaunchOf(t, svc, "task1") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("prompt_ensure was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchPromptEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchPromptEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "hello" { + t.Fatalf("recorded payload does not match the refused prompt: %+v", record) + } +} + +// TestPromptTask_ManualOverCeilingIsAdmitted covers AC-14 through the real +// promptTask entry point: PromptTask's own public wrapper always hardcodes +// manual, so it must be admitted even at the ceiling, and never write a +// deferral record. +func TestPromptTask_ManualOverCeilingIsAdmitted(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + _, err := svc.PromptTask(ctx, "task1", "session1", "hello", "", false, nil, false) + if err == nil { + t.Fatal("expected an error (no resumable executor), but not a ceiling refusal") + } + if _, ok := isSeam3Refusal(err); ok { + t.Fatalf("a manual prompt must never be refused by the ceiling, got: %v", err) + } + if record := deferredLaunchOf(t, svc, "task1"); record != nil { + t.Fatalf("a manual prompt must never write a ceiling_deferred record: %+v", record) + } +} + +// TestPromptTask_ManualOverCeilingIsAuditedEvenWhenTheResumeFails pins +// AC-14/AC-53's unconditional audit contract at seam 3's own gate +// (ensureSessionRunning's admitSeam3): the record must be written once the +// reservation is admitted, not only once the subsequent resume attempt has +// also succeeded. A resume failure right after admission (no executors_running +// row for the session) must not erase the only evidence a ceiling override +// happened. +func TestPromptTask_ManualOverCeilingIsAuditedEvenWhenTheResumeFails(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "seam3-fail-task", "seam3-fail-session", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + // PromptTask's public wrapper hardcodes manual origin (AC-14), so this + // resume is admitted over the ceiling's only slot. No executors_running row + // exists for the session, so attemptColdResume fails synchronously and + // deterministically before seam3Res.consume() is ever reached. + _, err := svc.PromptTask(ctx, "seam3-fail-task", "seam3-fail-session", "hello", "", false, nil, false) + require.Error(t, err, "the resume must still fail: no executors_running row exists for the session") + + session, getErr := repo.GetTaskSession(ctx, "seam3-fail-session") + require.NoError(t, getErr) + require.NotNil(t, session.Metadata[ceilingManualOverrideMetadataKey], + "a manual override must be audited even when the resume subsequently fails") +} + +// TestStartSessionForWorkflowStep_PreConsultationRefusalOverCeiling pins +// AC-47f/AC-47f1: the pre-consultation admission runs before +// advanceTaskWorkflowStep mutates the task, so a refusal must leave the +// task's workflow step untouched and record workflow_step_ensure itself. +func TestStartSessionForWorkflowStep_PreConsultationRefusalOverCeiling(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + + stepGetter := newMockStepGetter() + stepGetter.steps["step2"] = &wfmodels.WorkflowStep{ID: "step2", WorkflowID: "wf1"} + svc := createTestService(repo, stepGetter, newMockTaskRepo()) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + err := svc.StartSessionForWorkflowStep(ctx, "t1", "s1", "step2") + if !errors.Is(err, errSeam3WorkflowStepEnsureDeferred) { + t.Fatalf("StartSessionForWorkflowStep error = %v, want errSeam3WorkflowStepEnsureDeferred", err) + } + + task, getErr := repo.GetTask(ctx, "t1") + if getErr != nil { + t.Fatalf("get task: %v", getErr) + } + if task.WorkflowStepID != "step1" { + t.Fatalf("task.WorkflowStepID = %q, want unchanged %q — advanceTaskWorkflowStep must not run before the pre-consultation admits", task.WorkflowStepID, "step1") + } + + record := deferredLaunchOf(t, svc, "t1") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("workflow_step_ensure was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchWorkflowStepEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchWorkflowStepEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested[metaKeySessionID] != "s1" || nested[metaKeyWorkflowStepID] != "step2" { + t.Fatalf("recorded payload does not match the refused pre-consultation: %+v", record) + } +} + +// TestStartSessionForWorkflowStep_PreConsultReservationReleasedWhenResumeFails +// closes the coverage gap ccdac03c2 left at StartSessionForWorkflowStep's own +// pre-consult gate (admitOrDeferWorkflowStepEnsure): that commit moved this +// call site's recordManualOverrideIfAdmitted audit next to admission, mirroring +// ensureSessionRunning's own move, but only ensureSessionRunning's half was +// covered by a regression test (TestPromptTask_ManualOverCeilingIsAuditedEvenWhenTheResumeFails, +// reached through PromptTask's hardcoded manual origin). +// +// The pre-consult gate itself always admits with launchOriginAutomatic +// (admitOrDeferWorkflowStepEnsure hardcodes it), and decideLocked only ever +// sets manualOverride for a manual-origin request, so this specific gate can +// never produce a manual override to audit — recordManualOverrideIfAdmitted is +// unconditionally a no-op here regardless of where it sits relative to the +// downstream resume. What is genuinely load-bearing for this call site's +// failure path, and was not otherwise covered, is that its reservation itself +// does not leak when the resume it precedes fails after admission (no +// executors_running row for the session, so attemptColdResume fails +// deterministically before either reservation's consume() is reached). +func TestStartSessionForWorkflowStep_PreConsultReservationReleasedWhenResumeFails(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "wfstep-fail-task", "wfstep-fail-session", models.TaskSessionStateWaitingForInput) + + stepGetter := newMockStepGetter() + stepGetter.steps["step2"] = &wfmodels.WorkflowStep{ID: "step2", WorkflowID: "wf1"} + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, stepGetter, newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + + err := svc.StartSessionForWorkflowStep(ctx, "wfstep-fail-task", "wfstep-fail-session", "step2") + require.Error(t, err, "the resume must still fail: no executors_running row exists for the session") + + population, popErr := svc.sessionCeiling.population(ctx) + require.NoError(t, popErr) + require.Equal(t, 0, population, + "a resume failure after the pre-consult admission must release its reservation, not leak it") +} + +// TestTryEnsureExecution_ViewingShapeSwallowsRefusal covers AC-47a: a +// ceiling refusal reached from the viewing call shape (EnsureSession's +// resume-for-viewing path) has nothing to replay and nobody waiting, so it +// must be swallowed — no deferred_launch record, and the call must not +// panic despite tryEnsureExecution having no return value to report through. +func TestTryEnsureExecution_ViewingShapeSwallowsRefusal(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + svc.tryEnsureExecution(ctx, "session1", seam3CallShapeViewing, launchOriginAutomatic, "") + + if record := deferredLaunchOf(t, svc, "task1"); record != nil { + t.Fatalf("the viewing call shape must swallow a ceiling refusal, not record it: %+v", record) + } +} + +// TestTryEnsureExecution_QueueDrainShapeDefersRefusal covers AC-47d: a +// refusal reached from the queue-drain call shape defers, carrying the +// queued message id needed for AC-47d1's already-drained check at retry +// time. +func TestTryEnsureExecution_QueueDrainShapeDefersRefusal(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + svc.tryEnsureExecution(ctx, "session1", seam3CallShapeQueueDrain, launchOriginAutomatic, "queued-msg-42") + + record := deferredLaunchOf(t, svc, "task1") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the queue-drain call shape must record the refusal: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchQueueDrainEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchQueueDrainEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested[metaKeySessionID] != "session1" || nested["queued_message_id"] != "queued-msg-42" { + t.Fatalf("recorded payload does not match the refused drain: %+v", record) + } +} + +// TestTryEnsureExecution_UnrecognizedShapeDefaultsToDeferring covers AC-47e1: +// an absent/unrecognized call-shape token must default to the deferring +// shape rather than silently dropping the launch. +func TestTryEnsureExecution_UnrecognizedShapeDefaultsToDeferring(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + svc.tryEnsureExecution(ctx, "session1", seam3CallShape("unrecognized"), launchOriginAutomatic, "queued-msg-99") + + record := deferredLaunchOf(t, svc, "task1") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("an unrecognized call shape must default to deferring, not dropping: %+v", record) + } +} + +// TestQueueAutoStartPrompt_ThreadsQueuedMessageIDToDeferral is an end-to-end +// check that queueAutoStartPrompt's queued message id survives through +// scheduleAutoResumeForWorkflowQueue's detached goroutine into the +// queue_drain_ensure record, when the ceiling is over capacity and no +// execution is tracked for the session. +func TestQueueAutoStartPrompt_ThreadsQueuedMessageIDToDeferral(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput) + + agentMgr := &mockAgentManager{isAgentRunning: false} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "filler"}) + + err := svc.queueAutoStartPrompt(ctx, "task1", "session1", "queued prompt", false, nil, workflowMessageOrigin{}, false, nil, "") + if err != nil { + t.Fatalf("queueAutoStartPrompt: %v", err) + } + + require.Eventually(t, func() bool { + task, getErr := svc.repo.GetTask(ctx, "task1") + if getErr != nil || task.Metadata == nil { + return false + } + record, _ := task.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + return record != nil && record[models.CeilingDeferredKey] == true + }, 2*time.Second, 10*time.Millisecond, "expected the detached goroutine to record a queue_drain_ensure deferral") + + record := deferredLaunchOf(t, svc, "task1") + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchQueueDrainEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchQueueDrainEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + queuedMessageID, _ := nested["queued_message_id"].(string) + if queuedMessageID == "" { + t.Fatalf("recorded payload is missing the queued message id: %+v", record) + } + status := svc.messageQueue.GetStatus(ctx, "session1") + if len(status.Entries) != 1 || status.Entries[0].ID != queuedMessageID { + t.Fatalf("recorded queued_message_id %q does not match the queue's own message: %+v", queuedMessageID, status.Entries) + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam3_test.go b/apps/backend/internal/orchestrator/ceiling_seam3_test.go new file mode 100644 index 0000000000..9ba4681471 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam3_test.go @@ -0,0 +1,285 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestAdmitSeam3AdmitsUnderCeiling covers the ordinary case: population below +// the ceiling, admitted, reservation keyed by the session being resumed, and +// (unlike every other seam) no record is written by the gate itself — seam 3 +// never persists, only its four callers do. +func TestAdmitSeam3AdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam3-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, refusal := svc.admitSeam3(ctx, "seam3-admit", "seam3-admit-session", launchOriginAutomatic) + if refusal != nil { + t.Fatalf("ensureSessionRunning's own gate refused while under the ceiling: %v", refusal) + } + if reservation == nil || reservation.key != "seam3-admit-session" { + t.Fatalf("reservation not keyed by the session being resumed: %+v", reservation) + } +} + +// TestAdmitSeam3RefusesAutomaticOverCeiling covers AC-47: an automatic +// ensureSessionRunning call at the ceiling is refused via the distinguishable +// sentinel, with no record written by the gate itself (disposition belongs to +// each of the four call shapes, not to admitSeam3). +func TestAdmitSeam3RefusesAutomaticOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam3-first", "seam3-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + + if _, refusal := svc.admitSeam3(ctx, "seam3-first", "seam3-first-session", launchOriginAutomatic); refusal != nil { + t.Fatalf("first ensure was not admitted: %v", refusal) + } + + reservation, refusal := svc.admitSeam3(ctx, "seam3-second", "seam3-second-session", launchOriginAutomatic) + if refusal == nil { + t.Fatal("second automatic ensure over the ceiling was admitted, want refused") + } + if reservation != nil { + t.Fatal("a refused ensure must not hold a reservation") + } + if refusal.deferred { + t.Fatal("admitSeam3 must never set deferred itself; that belongs to the caller's disposition") + } + + record := deferredLaunchOf(t, svc, "seam3-second") + if record != nil { + t.Fatalf("admitSeam3 must never persist a record itself: %+v", record) + } +} + +// TestAdmitSeam3AdmitsManualOverCeiling covers AC-14: a manual +// ensureSessionRunning call is always admitted, never refused, even at the +// ceiling. +func TestAdmitSeam3AdmitsManualOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam3-manual-first", "seam3-manual-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, refusal := svc.admitSeam3(ctx, "seam3-manual-first", "seam3-manual-first-session", launchOriginAutomatic); refusal != nil { + t.Fatalf("first ensure was not admitted: %v", refusal) + } + + reservation, refusal := svc.admitSeam3(ctx, "seam3-manual-second", "seam3-manual-second-session", launchOriginManual) + if refusal != nil { + t.Fatalf("a manual ensure was refused; AC-14 requires it always be admitted: %v", refusal) + } + if reservation == nil { + t.Fatal("a manual override still consumes a reservation") + } +} + +// TestAdmitOrDeferWorkflowStepEnsureAdmitsUnderCeiling covers AC-47f's +// pre-consultation in the ordinary case. +func TestAdmitOrDeferWorkflowStepEnsureAdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam3-wf-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, deferred, err := svc.admitOrDeferWorkflowStepEnsure(ctx, "seam3-wf-admit", "seam3-wf-admit-session", "step-1") + if err != nil { + t.Fatalf("admitOrDeferWorkflowStepEnsure: %v", err) + } + if deferred { + t.Fatal("pre-consultation was deferred while under the ceiling") + } + if reservation == nil || reservation.key != "seam3-wf-admit-session" { + t.Fatalf("reservation not keyed by the session: %+v", reservation) + } + if record := deferredLaunchOf(t, svc, "seam3-wf-admit"); record != nil { + t.Fatalf("a deferred_launch record was written for an admitted pre-consultation: %+v", record) + } +} + +// TestAdmitOrDeferWorkflowStepEnsureDefersOverCeilingAndRecords covers +// AC-47f/AC-47f1: a pre-consultation refused at the ceiling writes the +// workflow_step_ensure record itself, carrying session_id and +// workflow_step_id, so the caller can return before ever calling +// advanceTaskWorkflowStep. +func TestAdmitOrDeferWorkflowStepEnsureDefersOverCeilingAndRecords(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam3-wf-first", "seam3-wf-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferWorkflowStepEnsure(ctx, "seam3-wf-first", "seam3-wf-first-session", "step-1"); err != nil || deferred { + t.Fatalf("first pre-consultation was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferWorkflowStepEnsure(ctx, "seam3-wf-second", "seam3-wf-second-session", "step-2") + if err != nil { + t.Fatalf("admitOrDeferWorkflowStepEnsure: %v", err) + } + if !deferred { + t.Fatal("second pre-consultation over the ceiling was admitted, want deferred") + } + if reservation != nil { + t.Fatal("a deferred pre-consultation must not hold a reservation") + } + + record := deferredLaunchOf(t, svc, "seam3-wf-second") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the refused pre-consultation was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchWorkflowStepEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchWorkflowStepEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested[metaKeySessionID] != "seam3-wf-second-session" || nested[metaKeyWorkflowStepID] != "step-2" { + t.Fatalf("the recorded payload does not match the refused pre-consultation: %+v", record) + } +} + +// TestAdmitOrDeferWorkflowStepEnsureReportsWriteFailure covers AC-45: a +// pre-consultation refusal that cannot be persisted must not be swallowed. +func TestAdmitOrDeferWorkflowStepEnsureReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + ctx := context.Background() + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "startSessionForWorkflowStepPreConsult"}) + + _, _, err := svc.admitOrDeferWorkflowStepEnsure(ctx, "missing-task", "missing-session", "step-1") + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } +} + +// TestDeferSeam3QueueDrainRefusalRecordsQueuedMessageID covers AC-47d: the +// queue-drain disposition records session_id and the queued message's own id, +// needed for AC-47d1's already-drained check at retry time. +func TestDeferSeam3QueueDrainRefusalRecordsQueuedMessageID(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam3-qd", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + svc.deferSeam3QueueDrainRefusal(ctx, "seam3-qd", "seam3-qd-session", "queued-msg-1", &seam3Refusal{reasonCode: ceilingReasonRefused}) + + record := deferredLaunchOf(t, svc, "seam3-qd") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the queue-drain refusal was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchQueueDrainEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchQueueDrainEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested[metaKeySessionID] != "seam3-qd-session" || nested["queued_message_id"] != "queued-msg-1" { + t.Fatalf("the recorded payload does not match the refused drain: %+v", record) + } +} + +// TestDisposeSeam3PromptEnsureRefusalWritesRecordWhenReconstructable covers +// AC-47/AC-47c2's first disposition: with no AC-47c(a) field set, the refusal +// is fully reconstructable from promptTask's own frame, so a prompt_ensure +// record is written and the refusal is marked deferred. +func TestDisposeSeam3PromptEnsureRefusalWritesRecordWhenReconstructable(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam3-prompt-a", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + refusal := &seam3Refusal{reasonCode: ceilingReasonRefused} + + err := svc.disposeSeam3PromptEnsureRefusal( + ctx, "seam3-prompt-a", "seam3-prompt-a-session", "hello", "", false, nil, false, + promptTaskOptions{lifecyclePrompt: true}, refusal, + ) + if err != nil { + t.Fatalf("disposeSeam3PromptEnsureRefusal: %v", err) + } + if !refusal.deferred { + t.Fatal("a reconstructable refusal must be marked deferred once the record is written") + } + + record := deferredLaunchOf(t, svc, "seam3-prompt-a") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the prompt refusal was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchPromptEnsure) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchPromptEnsure) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "hello" { + t.Fatalf("the recorded payload does not match the refused prompt: %+v", record) + } +} + +// TestDisposeSeam3PromptEnsureRefusalReturnsUnchangedWhenNonReconstructable +// covers AC-47c1: a call carrying a class-(a) field (here, an in-flight +// afterClaim callback) is not reconstructable from a stored record, so no +// prompt_ensure record is written and the refusal is left undeferred for the +// caller's own restoration path to own. +func TestDisposeSeam3PromptEnsureRefusalReturnsUnchangedWhenNonReconstructable(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam3-prompt-b", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + refusal := &seam3Refusal{reasonCode: ceilingReasonRefused} + + err := svc.disposeSeam3PromptEnsureRefusal( + ctx, "seam3-prompt-b", "seam3-prompt-b-session", "hello", "", false, nil, false, + promptTaskOptions{afterClaim: func() error { return nil }}, refusal, + ) + if err != nil { + t.Fatalf("disposeSeam3PromptEnsureRefusal: %v", err) + } + if refusal.deferred { + t.Fatal("a non-reconstructable refusal must not be marked deferred") + } + if record := deferredLaunchOf(t, svc, "seam3-prompt-b"); record != nil { + t.Fatalf("a non-reconstructable refusal must not write a record: %+v", record) + } +} + +// TestDisposeSeam3PromptEnsureRefusalReportsWriteFailure covers AC-45 for the +// prompt_ensure disposition. +func TestDisposeSeam3PromptEnsureRefusalReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + refusal := &seam3Refusal{reasonCode: ceilingReasonRefused} + + err := svc.disposeSeam3PromptEnsureRefusal( + context.Background(), "missing-task", "missing-session", "hello", "", false, nil, false, + promptTaskOptions{}, refusal, + ) + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } + if refusal.deferred { + t.Fatal("a refusal must not be marked deferred when the write itself failed") + } +} + +// TestIsSeam3Refusal covers the sentinel's errors.As compatibility, including +// through fmt.Errorf's %w wrapping used by promptTask's own error paths. +func TestIsSeam3Refusal(t *testing.T) { + refusal := &seam3Refusal{reasonCode: ceilingReasonRefused} + wrapped := errors.New("wrapped: " + refusal.Error()) + if _, ok := isSeam3Refusal(wrapped); ok { + t.Fatal("a plain error must not be misidentified as a seam3Refusal") + } + if got, ok := isSeam3Refusal(refusal); !ok || got != refusal { + t.Fatal("isSeam3Refusal must recognize its own sentinel and return the same pointer") + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam4.go b/apps/backend/internal/orchestrator/ceiling_seam4.go new file mode 100644 index 0000000000..2609f1e938 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam4.go @@ -0,0 +1,35 @@ +package orchestrator + +import ( + "context" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" +) + +// seam4ResumePayload builds the AC-42 "resume" replay row from +// ResumeTaskSessionWithOptions's own frame, at the point the gate is consulted. +func seam4ResumePayload(sessionID string, options executor.ResumeOptions) map[string]interface{} { + return map[string]interface{}{ + metaKeySessionID: sessionID, + "allow_branch_replacement": options.AllowBranchReplacement, + } +} + +// admitOrDeferSeam4 is Service.ResumeTaskSessionWithOptions's gate: consulted +// after the resumability and Office checks, before the executor's resume call, +// keyed by the session being resumed — it already exists, so there is no +// launch-scoped window and no redirect to re-key onto (unlike seam 2). On +// refusal it persists the ceiling_deferred record from the caller-supplied +// AC-42 "resume" payload and reports the decision so the caller returns +// without calling the executor. +// +// The returned reservation is non-nil only when the resume is admitted. The +// caller is responsible for deferring reservation.releaseIfNotConsumed and for +// calling reservation.consume once the resume succeeds. +func (s *Service) admitOrDeferSeam4( + ctx context.Context, taskID, sessionID string, origin launchOrigin, startPayload map[string]interface{}, +) (*sessionKeyedCeilingReservation, bool, error) { + return s.admitOrDeferSessionKeyedLaunch(ctx, taskID, sessionID, origin, "resumeTaskSession", + models.CeilingLaunchResume, startPayload, "the resume could not be admitted or recorded") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam4_integration_test.go b/apps/backend/internal/orchestrator/ceiling_seam4_integration_test.go new file mode 100644 index 0000000000..2b0a56fa1d --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam4_integration_test.go @@ -0,0 +1,173 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +// newSeam4TestService builds a cancelled, resumable session (no executor +// record required — AC-4's seam 4 resumability check waives it for +// CANCELLED/FAILED states) backed by a real sqlite repo and a mock agent +// manager that transitions the session to WAITING_FOR_INPUT on launch, the +// same fixture shape as TestCompletedSessionResumePreservesConversation. +func newSeam4TestService(t *testing.T, taskID, sessionID string) *Service { + t.Helper() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateCancelled) + + ctx := context.Background() + session, err := repo.GetTaskSession(ctx, sessionID) + require.NoError(t, err) + session.AgentProfileID = "profile-1" + require.NoError(t, repo.UpdateTaskSession(ctx, session)) + + taskRepo := newMockTaskRepo() + taskRepo.tasks[taskID] = &v1.Task{ID: taskID, Title: "T", State: v1.TaskStateInProgress} + + started := false + agentMgr := &sessionUpdatingAgentManager{ + mockAgentManager: &mockAgentManager{}, + repo: repo, + sessionID: sessionID, + taskID: taskID, + onStartCalled: &started, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + return svc +} + +// TestResumeTaskSessionWithOptions_SecondAutomaticLaunchOverCeilingIsDeferred +// pins AC-4/AC-11 at the real seam: ResumeTaskSessionWithOptions itself, gated +// after the resumability/Office checks and before the executor call, must +// refuse the second automatic resume once the ceiling's single slot is held. +func TestResumeTaskSessionWithOptions_SecondAutomaticLaunchOverCeilingIsDeferred(t *testing.T) { + ctx := context.Background() + svcA := newSeam4TestService(t, "seam4-it-task-a", "seam4-it-session-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + exec1, err := svcA.ResumeTaskSessionWithOptions(ctx, "seam4-it-task-a", "seam4-it-session-a", executor.ResumeOptions{ + Origin: "automatic", + }) + require.NoError(t, err) + require.NotNil(t, exec1) + + // A second, independent service sharing the same controller instance + // simulates a second resume request arriving while the first's reservation + // is still held — ResumeTaskSessionWithOptions never releases the + // reservation for a session it successfully resumed within this process + // lifetime (it is consumed, matching the population's own semantics). + svcB := newSeam4TestService(t, "seam4-it-task-b", "seam4-it-session-b") + svcB.sessionCeiling = svcA.sessionCeiling + + exec2, err := svcB.ResumeTaskSessionWithOptions(ctx, "seam4-it-task-b", "seam4-it-session-b", executor.ResumeOptions{ + Origin: "automatic", + }) + require.NoError(t, err) + require.Nil(t, exec2, "a refused automatic resume must not produce a launched execution") + + record := deferredLaunchOf(t, svcB, "seam4-it-task-b") + require.NotNil(t, record, "a ceiling refusal must persist a deferred_launch record") + require.Equal(t, true, record[models.CeilingDeferredKey]) + require.Equal(t, string(models.CeilingLaunchResume), record[models.CeilingLaunchKindKey]) +} + +// TestResumeTaskSessionWithOptions_ManualLaunchOverCeilingIsAdmitted pins +// AC-14 through the real ResumeTaskSessionWithOptions entry point. +func TestResumeTaskSessionWithOptions_ManualLaunchOverCeilingIsAdmitted(t *testing.T) { + ctx := context.Background() + svcA := newSeam4TestService(t, "seam4-it-manual-a", "seam4-it-manual-session-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svcA.ResumeTaskSessionWithOptions(ctx, "seam4-it-manual-a", "seam4-it-manual-session-a", executor.ResumeOptions{ + Origin: "automatic", + }) + require.NoError(t, err) + + svcB := newSeam4TestService(t, "seam4-it-manual-b", "seam4-it-manual-session-b") + svcB.sessionCeiling = svcA.sessionCeiling + + exec2, err := svcB.ResumeTaskSessionWithOptions(ctx, "seam4-it-manual-b", "seam4-it-manual-session-b", executor.ResumeOptions{ + Origin: "manual", + }) + require.NoError(t, err) + require.NotNil(t, exec2, "a manual resume must be admitted even over the ceiling") + + record := deferredLaunchOf(t, svcB, "seam4-it-manual-b") + require.Nil(t, record, "a manual override must never write a ceiling_deferred record") +} + +// TestResumeTaskSessionWithOptions_ManualLaunchOverCeilingIsAuditedEvenWhenTheLaunchFails +// pins AC-14/AC-53's unconditional audit contract: the record must be written +// once the reservation is admitted onto the session being resumed, not only +// once the executor's subsequent LaunchAgent call has also succeeded. A launch +// failure between admission and agent start must not erase the only evidence +// a ceiling override happened. +func TestResumeTaskSessionWithOptions_ManualLaunchOverCeilingIsAuditedEvenWhenTheLaunchFails(t *testing.T) { + ctx := context.Background() + svcA := newSeam4TestService(t, "seam4-it-fail-a", "seam4-it-fail-session-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svcA.ResumeTaskSessionWithOptions(ctx, "seam4-it-fail-a", "seam4-it-fail-session-a", executor.ResumeOptions{ + Origin: "automatic", + }) + require.NoError(t, err) + + repoB := setupTestRepo(t) + seedTaskAndSession(t, repoB, "seam4-it-fail-b", "seam4-it-fail-session-b", models.TaskSessionStateCancelled) + sessionB, err := repoB.GetTaskSession(ctx, "seam4-it-fail-session-b") + require.NoError(t, err) + sessionB.AgentProfileID = "profile-1" + require.NoError(t, repoB.UpdateTaskSession(ctx, sessionB)) + taskRepoB := newMockTaskRepo() + taskRepoB.tasks["seam4-it-fail-b"] = &v1.Task{ID: "seam4-it-fail-b", Title: "T", State: v1.TaskStateInProgress} + launchErr := errors.New("resume launch failed") + agentMgrB := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + return nil, launchErr + }, + } + svcB := createTestServiceWithAgent(repoB, newMockStepGetter(), taskRepoB, agentMgrB) + svcB.executor = executor.NewExecutor(agentMgrB, repoB, testLogger(), executor.ExecutorConfig{}) + svcB.sessionCeiling = svcA.sessionCeiling + + _, err = svcB.ResumeTaskSessionWithOptions(ctx, "seam4-it-fail-b", "seam4-it-fail-session-b", executor.ResumeOptions{ + Origin: "manual", + }) + require.Error(t, err, "the forced resume failure must still surface") + + session, err := repoB.GetTaskSession(ctx, "seam4-it-fail-session-b") + require.NoError(t, err) + require.NotNil(t, session.Metadata[ceilingManualOverrideMetadataKey], + "a manual override must be audited even when the resume subsequently fails") +} + +// TestResumeTaskSessionWithOptions_UnsetOriginDefaultsAutomatic pins AC-13b +// through the real entry point: launchResume is the only caller that sets +// Origin explicitly today, so every other caller's zero-value Origin must +// still be gated (and refused/deferred, not silently treated as manual) once +// the ceiling is reached. +func TestResumeTaskSessionWithOptions_UnsetOriginDefaultsAutomatic(t *testing.T) { + ctx := context.Background() + svcA := newSeam4TestService(t, "seam4-it-unset-a", "seam4-it-unset-session-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + _, err := svcA.ResumeTaskSessionWithOptions(ctx, "seam4-it-unset-a", "seam4-it-unset-session-a", executor.ResumeOptions{ + Origin: "automatic", + }) + require.NoError(t, err) + + svcB := newSeam4TestService(t, "seam4-it-unset-b", "seam4-it-unset-session-b") + svcB.sessionCeiling = svcA.sessionCeiling + + // No Origin set at all. + exec2, err := svcB.ResumeTaskSessionWithOptions(ctx, "seam4-it-unset-b", "seam4-it-unset-session-b", executor.ResumeOptions{}) + require.NoError(t, err) + require.Nil(t, exec2, "an unset origin must default to automatic and be refused over the ceiling, not silently admitted") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam4_test.go b/apps/backend/internal/orchestrator/ceiling_seam4_test.go new file mode 100644 index 0000000000..845cf2322c --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam4_test.go @@ -0,0 +1,144 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" +) + +// TestAdmitOrDeferSeam4AdmitsUnderCeiling covers the ordinary case: population +// below the ceiling, admitted, no record written, reservation keyed by the +// session being resumed. +func TestAdmitOrDeferSeam4AdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam4-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-admit", "seam4-admit-session", launchOriginAutomatic, map[string]interface{}{"session_id": "seam4-admit-session"}) + if err != nil { + t.Fatalf("admitOrDeferSeam4: %v", err) + } + if deferred { + t.Fatal("resume was deferred while under the ceiling") + } + if reservation == nil || reservation.key != "seam4-admit-session" { + t.Fatalf("reservation not keyed by the session being resumed: %+v", reservation) + } + + record := deferredLaunchOf(t, svc, "seam4-admit") + if record != nil { + t.Fatalf("a deferred_launch record was written for an admitted resume: %+v", record) + } +} + +// TestAdmitOrDeferSeam4DefersAutomaticOverCeiling covers AC-11/AC-42: an +// automatic resume at the ceiling is refused and its "resume" payload is +// recorded, not dropped. +func TestAdmitOrDeferSeam4DefersAutomaticOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam4-first", "seam4-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + + if _, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-first", "seam4-first-session", launchOriginAutomatic, map[string]interface{}{"session_id": "seam4-first-session"}); err != nil || deferred { + t.Fatalf("first resume was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-second", "seam4-second-session", launchOriginAutomatic, seam4ResumePayload("seam4-second-session", executor.ResumeOptions{AllowBranchReplacement: true})) + if err != nil { + t.Fatalf("admitOrDeferSeam4: %v", err) + } + if !deferred { + t.Fatal("second automatic resume over the ceiling was admitted, want deferred") + } + if reservation != nil { + t.Fatal("a deferred resume must not hold a reservation") + } + + record := deferredLaunchOf(t, svc, "seam4-second") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the refused resume's payload was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchResume) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchResume) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["allow_branch_replacement"] != true { + t.Fatalf("the recorded payload does not match the refused resume: %+v", record) + } +} + +// TestAdmitOrDeferSeam4AdmitsManualOverCeiling covers AC-14: a manual resume is +// always admitted, never deferred, even at the ceiling. +func TestAdmitOrDeferSeam4AdmitsManualOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam4-manual-first", "seam4-manual-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-manual-first", "seam4-manual-first-session", launchOriginAutomatic, map[string]interface{}{"session_id": "seam4-manual-first-session"}); err != nil || deferred { + t.Fatalf("first resume was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-manual-second", "seam4-manual-second-session", launchOriginManual, map[string]interface{}{"session_id": "seam4-manual-second-session"}) + if err != nil { + t.Fatalf("admitOrDeferSeam4: %v", err) + } + if deferred { + t.Fatal("a manual resume was deferred; AC-14 requires it always be admitted") + } + if reservation == nil { + t.Fatal("a manual override still consumes a reservation") + } + if record := deferredLaunchOf(t, svc, "seam4-manual-second"); record != nil { + t.Fatalf("a manual override must never write a ceiling_deferred record: %+v", record) + } +} + +// TestAdmitOrDeferSeam4ReportsWriteFailure covers AC-45: a refusal that cannot +// be persisted must not be swallowed. +func TestAdmitOrDeferSeam4ReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + ctx := context.Background() + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "resumeTaskSession"}) + + _, _, err := svc.admitOrDeferSeam4(ctx, "missing-task", "missing-session", launchOriginAutomatic, map[string]interface{}{"session_id": "missing-session"}) + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } +} + +// TestAdmitOrDeferSeam4UnsetOriginDefaultsAutomatic pins AC-13b directly +// against the controller: an unset/invalid origin string (what ResumeOptions's +// zero value converts to) must classify as automatic, not fall through as an +// unconditional manual admit. +func TestAdmitOrDeferSeam4UnsetOriginDefaultsAutomatic(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam4-unset-first", "seam4-unset-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-unset-first", "seam4-unset-first-session", launchOriginAutomatic, map[string]interface{}{}); err != nil || deferred { + t.Fatalf("first resume was not admitted: deferred=%v err=%v", deferred, err) + } + + // launchOrigin("") is neither "manual" nor "automatic". + _, deferred, err := svc.admitOrDeferSeam4(ctx, "seam4-unset-second", "seam4-unset-second-session", launchOrigin(""), map[string]interface{}{}) + if err != nil { + t.Fatalf("admitOrDeferSeam4: %v", err) + } + if !deferred { + t.Fatal("an unset origin over the ceiling must default to automatic and be deferred, not admitted as manual") + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam5.go b/apps/backend/internal/orchestrator/ceiling_seam5.go new file mode 100644 index 0000000000..229f05f947 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam5.go @@ -0,0 +1,63 @@ +package orchestrator + +import ( + "context" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/orchestrator/watcher" + "github.com/kandev/kandev/internal/task/models" +) + +// seam5DynamicRelaunchPayload builds the AC-42f "dynamic_relaunch" replay row +// from relaunchDynamicTaskAfterFailure's own frame, at the point the gate is +// consulted. +func seam5DynamicRelaunchPayload(data watcher.AgentEventData, executionProfileID string) map[string]interface{} { + return map[string]interface{}{ + metaKeySessionID: data.SessionID, + metaKeyTaskID: data.TaskID, + "agent_execution_id": data.AgentExecutionID, + "execution_profile_id": executionProfileID, + metaKeyAgentProfileID: data.AgentProfileID, + } +} + +// admitOrDeferSeam5 is Service.relaunchDynamicTaskAfterFailure's gate, consulted +// at entry, before the CREATED transition and before the Office/non-Office +// fork. It uses handOffOrAdmit rather than admit: on the automatic path the +// predecessor session is still counted, so this is AC-38's capacity hand-off, +// not an ordinary admission request; only the two LaunchDynamicRouteAction +// callers, which arrive after the predecessor has already left the counted +// population, fall through to an ordinary AC-13e-governed admission. +// +// The returned reservation is non-nil only when the relaunch is admitted or +// handed off. The caller is responsible for deferring +// reservation.releaseIfNotConsumed and for calling reservation.consume once a +// replacement launch is actually dispatched. +func (s *Service) admitOrDeferSeam5( + ctx context.Context, taskID string, origin launchOrigin, relaunchPayload map[string]interface{}, +) (reservation *sessionKeyedCeilingReservation, deferred bool, err error) { + sessionID, _ := relaunchPayload[metaKeySessionID].(string) + decision := s.sessionCeiling.handOffOrAdmit(ctx, admissionRequest{ + taskID: taskID, + sessionID: sessionID, + origin: origin, + seam: "relaunchDynamicTaskAfterFailure", + }) + if decision.admitted { + return &sessionKeyedCeilingReservation{ + controller: s.sessionCeiling, key: decision.reservationKey, + manualOverride: decision.manualOverride, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + }, false, nil + } + + if err := s.deferCeilingRefusal(ctx, taskID, sessionID, models.CeilingLaunchDynamicRelaunch, relaunchPayload, decision.reasonCode, + decision.population, decision.populationKnown, decision.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the dynamic relaunch could not be admitted or recorded", + zap.String("task_id", taskID), zap.String("session_id", sessionID), + zap.String(ceilingFieldReasonCode, ceilingReasonDeferWriteFailed), zap.Error(err)) + return nil, false, err + } + return nil, true, nil +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam5_integration_test.go b/apps/backend/internal/orchestrator/ceiling_seam5_integration_test.go new file mode 100644 index 0000000000..3bbd6c235f --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam5_integration_test.go @@ -0,0 +1,198 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/watcher" + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +// seam5EventData builds the watcher.AgentEventData shape +// relaunchDynamicTaskAfterFailure's real callers construct. +func seam5EventData(taskID, sessionID, executionID string) watcher.AgentEventData { + return watcher.AgentEventData{ + TaskID: taskID, + SessionID: sessionID, + AgentExecutionID: executionID, + AgentProfileID: "profile-1", + } +} + +// newSeam5TestService builds a RUNNING, non-Office session with a live +// executors_running row and a mock agent manager that transitions the +// session to WAITING_FOR_INPUT on launch, matching +// relaunchDynamicTaskAfterFailure's own non-Office branch +// (s.StartCreatedSession). +func newSeam5TestService(t *testing.T, taskID, sessionID, executionID string) (*Service, *sqliterepo.Repository) { + t.Helper() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateRunning) + seedExecutorRunning(t, repo, sessionID, taskID, executionID) + + ctx := context.Background() + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("get session: %v", err) + } + session.AgentProfileID = "profile-1" + if err := repo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("update session: %v", err) + } + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, taskID, v1.TaskStateInProgress) + + started := false + agentMgr := &sessionUpdatingAgentManager{ + mockAgentManager: &mockAgentManager{}, + repo: repo, + sessionID: sessionID, + taskID: taskID, + onStartCalled: &started, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.lastTurnPrompt.Store(sessionID, capturedPrompt{text: "retry the task"}) + return svc, repo +} + +// TestRelaunchDynamicTaskAfterFailure_SecondAutomaticOverCeilingIsDeferred +// pins AC-38a/AC-42f at the real seam: relaunchDynamicTaskAfterFailure +// itself, gated at entry before the CREATED transition, must refuse the +// second automatic relaunch once the ceiling's single slot is held. +func TestRelaunchDynamicTaskAfterFailure_SecondAutomaticOverCeilingIsDeferred(t *testing.T) { + ctx := context.Background() + svcA, _ := newSeam5TestService(t, "seam5-it-task-a", "seam5-it-session-a", "seam5-it-exec-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + relaunched := svcA.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-task-a", "seam5-it-session-a", "seam5-it-exec-a"), "profile-1", launchOriginAutomatic) + if !relaunched { + t.Fatal("first automatic relaunch was not admitted") + } + + // A second, independent service sharing the same controller instance + // simulates a second relaunch arriving while the first's reservation is + // still held. + svcB, repoB := newSeam5TestService(t, "seam5-it-task-b", "seam5-it-session-b", "seam5-it-exec-b") + svcB.sessionCeiling = svcA.sessionCeiling + + relaunched = svcB.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-task-b", "seam5-it-session-b", "seam5-it-exec-b"), "profile-1", launchOriginAutomatic) + if relaunched { + t.Fatal("a refused automatic relaunch must not report success") + } + + session, err := repoB.GetTaskSession(ctx, "seam5-it-session-b") + if err != nil { + t.Fatalf("get session: %v", err) + } + if session.State == models.TaskSessionStateCreated { + t.Fatal("a refused relaunch must not have performed the CREATED transition") + } + + record := deferredLaunchOf(t, svcB, "seam5-it-task-b") + if record == nil { + t.Fatal("a ceiling refusal must persist a deferred_launch record") + } + if record[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling_deferred = %v, want true", record[models.CeilingDeferredKey]) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchDynamicRelaunch) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchDynamicRelaunch) + } +} + +func TestRelaunchDynamicTaskAfterFailure_ReportsDeferredOutcome(t *testing.T) { + ctx := context.Background() + svcA, _ := newSeam5TestService(t, "seam5-outcome-task-a", "seam5-outcome-session-a", "seam5-outcome-exec-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + require.True(t, svcA.relaunchDynamicTaskAfterFailure( + ctx, + seam5EventData("seam5-outcome-task-a", "seam5-outcome-session-a", "seam5-outcome-exec-a"), + "profile-1", + launchOriginAutomatic, + )) + + svcB, _ := newSeam5TestService(t, "seam5-outcome-task-b", "seam5-outcome-session-b", "seam5-outcome-exec-b") + svcB.sessionCeiling = svcA.sessionCeiling + require.Equal(t, dynamicRelaunchDeferred, svcB.relaunchDynamicTaskAfterFailureOutcome( + ctx, + seam5EventData("seam5-outcome-task-b", "seam5-outcome-session-b", "seam5-outcome-exec-b"), + "profile-1", + launchOriginAutomatic, + )) +} + +// TestRelaunchDynamicTaskAfterFailure_ManualOverCeilingIsAdmitted pins AC-14 +// through the real relaunchDynamicTaskAfterFailure entry point: the manual +// route-action origin is always admitted, even at the ceiling. +func TestRelaunchDynamicTaskAfterFailure_ManualOverCeilingIsAdmitted(t *testing.T) { + ctx := context.Background() + svcA, _ := newSeam5TestService(t, "seam5-it-manual-a", "seam5-it-manual-session-a", "seam5-it-manual-exec-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + if !svcA.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-manual-a", "seam5-it-manual-session-a", "seam5-it-manual-exec-a"), "profile-1", launchOriginAutomatic) { + t.Fatal("first automatic relaunch was not admitted") + } + + svcB, _ := newSeam5TestService(t, "seam5-it-manual-b", "seam5-it-manual-session-b", "seam5-it-manual-exec-b") + svcB.sessionCeiling = svcA.sessionCeiling + + if !svcB.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-manual-b", "seam5-it-manual-session-b", "seam5-it-manual-exec-b"), "profile-1", launchOriginManual) { + t.Fatal("a manual relaunch must be admitted even over the ceiling") + } + + if record := deferredLaunchOf(t, svcB, "seam5-it-manual-b"); record != nil { + t.Fatalf("a manual override must never write a ceiling_deferred record: %+v", record) + } +} + +// TestRelaunchDynamicTaskAfterFailure_ManualOverCeilingIsAuditedEvenWhenTheLaunchFails +// pins AC-14/AC-53's unconditional audit contract: the record must be written +// once the reservation is admitted onto the session being relaunched, not only +// once the subsequent StartCreatedSession call has also succeeded. A launch +// failure between admission and agent start must not erase the only evidence +// a ceiling override happened. +func TestRelaunchDynamicTaskAfterFailure_ManualOverCeilingIsAuditedEvenWhenTheLaunchFails(t *testing.T) { + ctx := context.Background() + svcA, _ := newSeam5TestService(t, "seam5-it-fail-a", "seam5-it-fail-session-a", "seam5-it-fail-exec-a") + svcA.sessionCeiling = newSessionCeilingController(1, nil, nil) + + if !svcA.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-fail-a", "seam5-it-fail-session-a", "seam5-it-fail-exec-a"), "profile-1", launchOriginAutomatic) { + t.Fatal("first automatic relaunch was not admitted") + } + + // Session B deliberately has no executors_running row, so the ensuing + // StartCreatedSession call takes the full synchronous LaunchAgent path + // (not the existing-workspace fast path) where the injected failure + // actually propagates back to this function. + repoB := setupTestRepo(t) + seedTaskAndSession(t, repoB, "seam5-it-fail-b", "seam5-it-fail-session-b", models.TaskSessionStateRunning) + sessionB, err := repoB.GetTaskSession(ctx, "seam5-it-fail-session-b") + require.NoError(t, err) + sessionB.AgentProfileID = "profile-1" + require.NoError(t, repoB.UpdateTaskSession(ctx, sessionB)) + taskRepoB := newMockTaskRepo() + seedMockTaskState(taskRepoB, "seam5-it-fail-b", v1.TaskStateInProgress) + launchErr := errors.New("relaunch failed") + agentMgrB := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + return nil, launchErr + }, + } + svcB := createTestServiceWithScheduler(repoB, newMockStepGetter(), taskRepoB, agentMgrB) + svcB.lastTurnPrompt.Store("seam5-it-fail-session-b", capturedPrompt{text: "retry the task"}) + svcB.sessionCeiling = svcA.sessionCeiling + + relaunched := svcB.relaunchDynamicTaskAfterFailure(ctx, seam5EventData("seam5-it-fail-b", "seam5-it-fail-session-b", "seam5-it-fail-exec-b"), "profile-1", launchOriginManual) + require.False(t, relaunched, "the forced launch failure must surface as an unsuccessful relaunch") + + session, err := repoB.GetTaskSession(ctx, "seam5-it-fail-session-b") + require.NoError(t, err) + require.NotNil(t, session.Metadata[ceilingManualOverrideMetadataKey], + "a manual override must be audited even when the relaunch subsequently fails") +} diff --git a/apps/backend/internal/orchestrator/ceiling_seam5_test.go b/apps/backend/internal/orchestrator/ceiling_seam5_test.go new file mode 100644 index 0000000000..b1b7d8edab --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seam5_test.go @@ -0,0 +1,160 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/orchestrator/watcher" + "github.com/kandev/kandev/internal/task/models" +) + +// seam5Payload builds a minimal AC-42f payload for a fake relaunch, keyed by +// the given session id, so these unit tests can exercise admitOrDeferSeam5 +// without going through the full relaunchDynamicTaskAfterFailure body. +func seam5Payload(sessionID string) map[string]interface{} { + return seam5DynamicRelaunchPayload(watcher.AgentEventData{ + TaskID: "irrelevant-for-payload", + SessionID: sessionID, + AgentExecutionID: "exec-1", + AgentProfileID: "agent-1", + }, "profile-1") +} + +// TestAdmitOrDeferSeam5AdmitsUnderCeiling covers the ordinary case: with no +// lister wired (as in every other seam's unit test), the session is never +// reported as already counted, so handOffOrAdmit falls through to an +// ordinary admission and behaves exactly like admit() while under the +// ceiling. +func TestAdmitOrDeferSeam5AdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam5-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-admit", launchOriginAutomatic, seam5Payload("seam5-admit-session")) + if err != nil { + t.Fatalf("admitOrDeferSeam5: %v", err) + } + if deferred { + t.Fatal("relaunch was deferred while under the ceiling") + } + if reservation == nil || reservation.key != "seam5-admit-session" { + t.Fatalf("reservation not keyed by the session being relaunched: %+v", reservation) + } + + record := deferredLaunchOf(t, svc, "seam5-admit") + if record != nil { + t.Fatalf("a deferred_launch record was written for an admitted relaunch: %+v", record) + } +} + +// TestAdmitOrDeferSeam5DefersAutomaticOverCeiling covers AC-38a/AC-42f: with +// the predecessor session not counted (the two LaunchDynamicRouteAction +// callers reach this after markDynamicRouteActionRequired has already +// parked the route), an automatic relaunch at the ceiling is an ordinary +// refusal and its "dynamic_relaunch" payload is recorded, not dropped. +func TestAdmitOrDeferSeam5DefersAutomaticOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam5-first", "seam5-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + + if _, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-first", launchOriginAutomatic, seam5Payload("seam5-first-session")); err != nil || deferred { + t.Fatalf("first relaunch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-second", launchOriginAutomatic, seam5Payload("seam5-second-session")) + if err != nil { + t.Fatalf("admitOrDeferSeam5: %v", err) + } + if !deferred { + t.Fatal("second automatic relaunch over the ceiling was admitted, want deferred") + } + if reservation != nil { + t.Fatal("a deferred relaunch must not hold a reservation") + } + + record := deferredLaunchOf(t, svc, "seam5-second") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the refused relaunch's payload was not recorded: %+v", record) + } + if record[models.CeilingLaunchKindKey] != string(models.CeilingLaunchDynamicRelaunch) { + t.Fatalf("ceiling_launch_kind = %v, want %q", record[models.CeilingLaunchKindKey], models.CeilingLaunchDynamicRelaunch) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["agent_execution_id"] != "exec-1" { + t.Fatalf("the recorded payload does not match the refused relaunch: %+v", record) + } +} + +// TestAdmitOrDeferSeam5AdmitsManualOverCeiling covers AC-14/AC-38a: a manual +// relaunch (the UI's manual retry/try-next action) is always admitted, never +// deferred, even at the ceiling. +func TestAdmitOrDeferSeam5AdmitsManualOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam5-manual-first", "seam5-manual-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-manual-first", launchOriginAutomatic, seam5Payload("seam5-manual-first-session")); err != nil || deferred { + t.Fatalf("first relaunch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-manual-second", launchOriginManual, seam5Payload("seam5-manual-second-session")) + if err != nil { + t.Fatalf("admitOrDeferSeam5: %v", err) + } + if deferred { + t.Fatal("a manual relaunch was deferred; AC-14 requires it always be admitted") + } + if reservation == nil { + t.Fatal("a manual override still consumes a reservation") + } + if record := deferredLaunchOf(t, svc, "seam5-manual-second"); record != nil { + t.Fatalf("a manual override must never write a ceiling_deferred record: %+v", record) + } +} + +// TestAdmitOrDeferSeam5ReportsWriteFailure covers AC-45: a refusal that +// cannot be persisted must not be swallowed. +func TestAdmitOrDeferSeam5ReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + ctx := context.Background() + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", sessionID: "filler-session", origin: launchOriginAutomatic, seam: "relaunchDynamicTaskAfterFailure"}) + + _, _, err := svc.admitOrDeferSeam5(ctx, "missing-task", launchOriginAutomatic, seam5Payload("missing-session")) + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } +} + +// TestAdmitOrDeferSeam5UnsetOriginDefaultsAutomatic pins AC-13b: an +// unset/invalid origin must classify as automatic, not fall through as an +// unconditional manual admit. +func TestAdmitOrDeferSeam5UnsetOriginDefaultsAutomatic(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam5-unset-first", "seam5-unset-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-unset-first", launchOriginAutomatic, seam5Payload("seam5-unset-first-session")); err != nil || deferred { + t.Fatalf("first relaunch was not admitted: deferred=%v err=%v", deferred, err) + } + + // launchOrigin("") is neither "manual" nor "automatic". + _, deferred, err := svc.admitOrDeferSeam5(ctx, "seam5-unset-second", launchOrigin(""), seam5Payload("seam5-unset-second-session")) + if err != nil { + t.Fatalf("admitOrDeferSeam5: %v", err) + } + if !deferred { + t.Fatal("an unset origin over the ceiling must default to automatic and be deferred, not admitted as manual") + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_seams.go b/apps/backend/internal/orchestrator/ceiling_seams.go new file mode 100644 index 0000000000..718ddba9a6 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seams.go @@ -0,0 +1,212 @@ +package orchestrator + +import ( + "context" + "errors" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// ErrCeilingLaunchDeferred is returned by startTask (and its StartTask* +// variants) when admitOrDeferSeam1 refuses admission: a ceiling_deferred +// record has already been written from the caller's own launch parameters, +// so this launch will be replayed automatically once capacity frees up +// rather than failing outright. A caller that only distinguishes "session +// created" (non-nil execution) from "hard failure" (non-nil error) has no +// way to notice this without checking for this sentinel — the nil execution +// looks the same as it would for an ordinary success it forgot to populate. +// Callers that only care whether the caller-visible request itself is +// blocked (GitHub/automation/watcher auto-start, which hold a one-shot +// claim that a real failure must restore) should treat this sentinel the +// same as a successful dispatch, since the sweep owns retrying it; callers +// that record launch bookkeeping (Office scheduler launch counters, health, +// persisted session id) must treat it as "not launched yet" instead of a +// success. +var ErrCeilingLaunchDeferred = errors.New("orchestrator: launch deferred by session ceiling") + +// originFromAutoStart derives the launch origin from the existing autoStart +// signal, the widening AC-13a describes for a seam that has no dedicated origin +// parameter of its own. It is not trusted at the two call sites AC-13d names, +// which force automatic regardless of the autoStart value they carry. +func originFromAutoStart(autoStart bool) launchOrigin { + if autoStart { + return launchOriginAutomatic + } + return launchOriginManual +} + +// seam1Reservation tracks the launch-scoped admission reservation seam 1 takes +// before a session exists (AC-6a). Every return path releases it, by +// deferring releaseIfNotConsumed at the point of admission, until the launch +// this reservation guards actually succeeds. rebindToSession moves the +// reservation onto the created session's id partway through, but rebinding +// alone does not protect it from release — a launch that fails after the +// session exists but before it ever reaches STARTING must still free the +// slot, so release is gated on consumed, not on rebound. +type seam1Reservation struct { + controller *sessionCeilingController + key string + rebound bool + consumed bool + + // manualOverride, population, populationKnown and ceiling are the + // admission decision's own reading, carried forward so AC-14/AC-53's + // audit write and card warning can be performed once the session this + // launch creates actually exists (AC-53, AC-14b). + manualOverride bool + population int + populationKnown bool + ceiling int +} + +// rebindToSession moves the reservation onto the created session's id, as one +// operation rather than a release followed by an acquire (AC-6a). The local +// key is updated to match, so a later release (if the launch never reaches +// consume) still targets the reservation the controller is actually holding. +func (r *seam1Reservation) rebindToSession(sessionID string) { + if r == nil || r.controller == nil || r.rebound || sessionID == "" { + return + } + if r.controller.rebind(r.key, sessionID) { + r.rebound = true + r.key = sessionID + } +} + +// consume marks the reservation as belonging to a launch that actually +// succeeded, so releaseIfNotConsumed becomes a no-op. +func (r *seam1Reservation) consume() { + if r == nil { + return + } + r.consumed = true +} + +// releaseIfNotConsumed is the deferred cleanup for every return path between +// admission and the launch actually succeeding, including failures that +// happen after rebindToSession has already moved the reservation onto the +// created session's id. +func (r *seam1Reservation) releaseIfNotConsumed() { + if r == nil || r.controller == nil || r.consumed { + return + } + r.controller.release(r.key) +} + +// ceilingCredentialEnvKeys are short-lived Office runtime credentials +// (internal/office/agents/auth.go mints them with a 4-hour expiry) that must +// never be persisted verbatim in task metadata: a launch can sit deferred +// longer than that, and replaying it with an expired bearer token leaves the +// agent unable to call the Kandev API. seam1StartPayload strips them before +// persisting; CeilingLaunchCredentialReminter re-mints fresh ones immediately +// before replay from the durable identity fields (agent/workspace/run id) +// that remain in the payload. +var ceilingCredentialEnvKeys = []string{"KANDEV_API_KEY", "KANDEV_RUN_TOKEN"} + +// redactedCeilingLaunchEnv returns a copy of env with ceilingCredentialEnvKeys +// removed, leaving the original map (still used for the live launch attempt +// this payload is only a record of) untouched. +func redactedCeilingLaunchEnv(env map[string]string) map[string]string { + if len(env) == 0 { + return env + } + redacted := make(map[string]string, len(env)) + for k, v := range env { + redacted[k] = v + } + for _, k := range ceilingCredentialEnvKeys { + delete(redacted, k) + } + return redacted +} + +// CeilingLaunchCredentialReminter refreshes the short-lived Office runtime +// credentials a ceiling-deferred "start" replay needs, immediately before +// replay, using the durable identity fields (agent/workspace/run id) that +// redactedCeilingLaunchEnv left in the persisted env. Registered via +// SetCeilingLaunchCredentialReminter; nil is a valid, common case (non-Office +// launches never carry these keys, so there is nothing to re-mint). +type CeilingLaunchCredentialReminter interface { + RemintCeilingLaunchCredentials(ctx context.Context, taskID string, env map[string]string) (map[string]string, error) +} + +// SetCeilingLaunchCredentialReminter wires the Office-side credential +// re-minter. See CeilingLaunchCredentialReminter. +func (s *Service) SetCeilingLaunchCredentialReminter(r CeilingLaunchCredentialReminter) { + s.ceilingCredentialReminter = r +} + +// seam1StartPayload builds the AC-42 "start" replay row from startTask's own +// frame, at the point the gate is consulted — before any workflow-step profile +// resolution, so the recorded profile is the one the caller actually chose. +func seam1StartPayload( + agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID string, + planMode, autoStart bool, + attachments []v1.MessageAttachment, + opts startTaskOptions, +) map[string]interface{} { + payload := map[string]interface{}{ + metaKeyAgentProfileID: agentProfileID, + "executor_id": executorID, + metaKeyExecutorProfile: executorProfileID, + "priority": priority, + metaKeyPrompt: prompt, + metaKeyWorkflowStepID: workflowStepID, + metaKeyPlanMode: planMode, + metaKeyAttachments: attachments, + "env": redactedCeilingLaunchEnv(opts.Env), + "route": opts.Route, + "profile_explicit": opts.ProfileExplicit, + "auto_start": autoStart, + "additional_skill_slugs": opts.AdditionalSkillSlugs, + "entry_options": opts.EntryOptions, + "workflow_entry_id": opts.WorkflowEntryID, + "origin": string(opts.Origin), + } + if opts.SpawnOrigin != nil { + payload["spawn_origin"] = map[string]interface{}{ + metaKeyTaskID: opts.SpawnOrigin.TaskID, + metaKeySessionID: opts.SpawnOrigin.SessionID, + "session_name": opts.SpawnOrigin.SessionName, + } + } + return payload +} + +// admitOrDeferSeam1 is Service.startTask's AC-4a gate: consulted before its own +// claimDeferredLaunchForStart, keyed by no session id yet (AC-6a). On refusal it +// persists the ceiling_deferred record from the caller-supplied AC-42 "start" +// payload and reports the decision so the caller returns without creating a +// session, materializing a workspace, or claiming the launch intent it holds. +// +// The returned reservation is non-nil only when the launch is admitted. The +// caller is responsible for deferring reservation.releaseIfNotConsumed, +// calling reservation.rebindToSession once the session exists, and calling +// reservation.consume once the launch actually succeeds. +func (s *Service) admitOrDeferSeam1( + ctx context.Context, taskID string, origin launchOrigin, startPayload map[string]interface{}, +) (reservation *seam1Reservation, deferred bool, err error) { + decision := s.sessionCeiling.admit(ctx, admissionRequest{ + taskID: taskID, + origin: origin, + seam: "startTask", + }) + if decision.admitted { + return &seam1Reservation{ + controller: s.sessionCeiling, key: decision.reservationKey, + manualOverride: decision.manualOverride, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + }, false, nil + } + + if err := s.deferCeilingRefusal(ctx, taskID, "", models.CeilingLaunchStart, startPayload, decision.reasonCode, + decision.population, decision.populationKnown, decision.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the launch could not be admitted or recorded", + zap.String("task_id", taskID), zap.String(ceilingFieldReasonCode, ceilingReasonDeferWriteFailed), zap.Error(err)) + return nil, false, err + } + return nil, true, nil +} diff --git a/apps/backend/internal/orchestrator/ceiling_seams_test.go b/apps/backend/internal/orchestrator/ceiling_seams_test.go new file mode 100644 index 0000000000..3c2376affd --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_seams_test.go @@ -0,0 +1,223 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +func newServiceWithCeiling(t *testing.T, ceiling int) (*Service, *tasksqlite.Repository) { + t.Helper() + svc, repo := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(ceiling, nil, nil) + return svc, repo +} + +// TestAdmitOrDeferSeam1AdmitsUnderCeiling covers the ordinary case: population +// below the ceiling, admitted, no record written. +func TestAdmitOrDeferSeam1AdmitsUnderCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 5) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "seam1-admit", Title: "T"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + reservation, deferred, err := svc.admitOrDeferSeam1(ctx, "seam1-admit", launchOriginAutomatic, map[string]interface{}{"prompt": "p"}) + if err != nil { + t.Fatalf("admitOrDeferSeam1: %v", err) + } + if deferred { + t.Fatal("launch was deferred while under the ceiling") + } + if reservation == nil || reservation.key == "" { + t.Fatal("no reservation was returned for an admitted launch") + } + + record := deferredLaunchOf(t, svc, "seam1-admit") + if record != nil { + t.Fatalf("a deferred_launch record was written for an admitted launch: %+v", record) + } +} + +// TestAdmitOrDeferSeam1DefersAutomaticOverCeiling covers AC-11: an automatic +// launch at the ceiling is refused and its payload is recorded, not dropped. +func TestAdmitOrDeferSeam1DefersAutomaticOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam1-first", "seam1-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + + if _, deferred, err := svc.admitOrDeferSeam1(ctx, "seam1-first", launchOriginAutomatic, map[string]interface{}{"prompt": "first"}); err != nil || deferred { + t.Fatalf("first launch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam1(ctx, "seam1-second", launchOriginAutomatic, map[string]interface{}{"prompt": "second"}) + if err != nil { + t.Fatalf("admitOrDeferSeam1: %v", err) + } + if !deferred { + t.Fatal("second automatic launch over the ceiling was admitted, want deferred") + } + if reservation != nil { + t.Fatal("a deferred launch must not hold a reservation") + } + + record := deferredLaunchOf(t, svc, "seam1-second") + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("the refused launch's payload was not recorded: %+v", record) + } + nested, _ := record[models.CeilingLaunchPayloadKey].(map[string]interface{}) + if nested["prompt"] != "second" { + t.Fatalf("the recorded payload does not match the refused launch: %+v", record) + } +} + +// TestAdmitOrDeferSeam1AdmitsManualOverCeiling covers AC-14: a manual launch is +// always admitted, never deferred, even at the ceiling. +func TestAdmitOrDeferSeam1AdmitsManualOverCeiling(t *testing.T) { + svc, repo := newServiceWithCeiling(t, 1) + ctx := context.Background() + for _, id := range []string{"seam1-manual-first", "seam1-manual-second"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: "T"}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + if _, deferred, err := svc.admitOrDeferSeam1(ctx, "seam1-manual-first", launchOriginAutomatic, map[string]interface{}{"prompt": "first"}); err != nil || deferred { + t.Fatalf("first launch was not admitted: deferred=%v err=%v", deferred, err) + } + + reservation, deferred, err := svc.admitOrDeferSeam1(ctx, "seam1-manual-second", launchOriginManual, map[string]interface{}{"prompt": "second"}) + if err != nil { + t.Fatalf("admitOrDeferSeam1: %v", err) + } + if deferred { + t.Fatal("a manual launch was deferred; AC-14 requires it always be admitted") + } + if reservation == nil { + t.Fatal("a manual override still consumes a reservation") + } + if record := deferredLaunchOf(t, svc, "seam1-manual-second"); record != nil { + t.Fatalf("a manual override must never write a ceiling_deferred record: %+v", record) + } +} + +// TestAdmitOrDeferSeam1ReportsWriteFailure covers AC-45: a refusal that cannot +// be persisted must not be swallowed. +func TestAdmitOrDeferSeam1ReportsWriteFailure(t *testing.T) { + svc, _ := newServiceWithCeiling(t, 1) + ctx := context.Background() + // Fill the one slot with a launch-scoped reservation so the next automatic + // request is refused. + svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "filler", origin: launchOriginAutomatic, seam: "startTask"}) + + // "missing-task" was never created, so the CAS read fails and the refusal + // cannot be persisted. + _, _, err := svc.admitOrDeferSeam1(ctx, "missing-task", launchOriginAutomatic, map[string]interface{}{"prompt": "p"}) + if err == nil { + t.Fatal("expected an error when the refusal could not be persisted") + } +} + +// TestSeam1ReservationReleasesAfterRebindIfNeverConsumed pins the corrected +// rebind/release contract: rebinding moves the reservation onto the session's +// id, but does not by itself protect it from release. A launch that fails +// after the session was created (so rebindToSession already ran) but before +// it ever reached STARTING must still free the slot via the deferred +// releaseIfNotConsumed — otherwise the reservation would leak under the +// session id until the stale-reservation sweep eventually reclaims it. +func TestSeam1ReservationReleasesAfterRebindIfNeverConsumed(t *testing.T) { + controller := newSessionCeilingController(5, nil, nil) + decision := controller.admit(context.Background(), admissionRequest{taskID: "t", origin: launchOriginAutomatic, seam: "startTask"}) + if !decision.admitted { + t.Fatal("expected admission") + } + reservation := &seam1Reservation{controller: controller, key: decision.reservationKey} + reservation.rebindToSession("session-1") + // Simulate a launch failure between rebind and STARTING: the reservation + // was never consumed, so the deferred cleanup must still release it. + reservation.releaseIfNotConsumed() + + population, err := controller.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 0 { + t.Fatalf("population = %d, want 0 (a rebound-but-unconsumed reservation must still release)", population) + } +} + +// TestSeam1ReservationConsumeSurvivesRebind covers the success path: once the +// launch actually succeeds and the reservation is consumed, the deferred +// releaseIfNotConsumed becomes a no-op and the population keeps counting the +// session under its rebound key. +func TestSeam1ReservationConsumeSurvivesRebind(t *testing.T) { + controller := newSessionCeilingController(5, nil, nil) + decision := controller.admit(context.Background(), admissionRequest{taskID: "t", origin: launchOriginAutomatic, seam: "startTask"}) + if !decision.admitted { + t.Fatal("expected admission") + } + reservation := &seam1Reservation{controller: controller, key: decision.reservationKey} + reservation.rebindToSession("session-1") + reservation.consume() + reservation.releaseIfNotConsumed() + + population, err := controller.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 1 { + t.Fatalf("population = %d, want 1 (a consumed reservation must survive the deferred release)", population) + } +} + +// TestSeam1ReservationRebindCollisionReleasesTheLoserNotTheWinner pins RV3-A: +// two independently-admitted seam 1 reservations can converge on the same +// session id (Office identity-owned sessions: EnsureSessionForAgentWithCreation +// converges concurrent callers for the same task/agent onto one session row). +// The second rebind to arrive must not silently overwrite the first +// reservation's map slot — its own later cleanup must be a no-op, not a +// deletion of the still-in-flight winner's reservation. +func TestSeam1ReservationRebindCollisionReleasesTheLoserNotTheWinner(t *testing.T) { + controller := newSessionCeilingController(2, nil, nil) + decisionA := controller.admit(context.Background(), admissionRequest{taskID: "task-a", origin: launchOriginAutomatic, seam: "startTask"}) + decisionB := controller.admit(context.Background(), admissionRequest{taskID: "task-b", origin: launchOriginAutomatic, seam: "startTask"}) + if !decisionA.admitted || !decisionB.admitted { + t.Fatal("expected both launches to be admitted under a ceiling of 2") + } + + winner := &seam1Reservation{controller: controller, key: decisionA.reservationKey} + loser := &seam1Reservation{controller: controller, key: decisionB.reservationKey} + + // Both launches converge on the same session, exactly as two concurrent + // startTask calls do via EnsureSessionForAgentWithCreation. + winner.rebindToSession("session-shared") + loser.rebindToSession("session-shared") + + // The loser's launch fails before the winner's ever resolves. + loser.releaseIfNotConsumed() + + population, err := controller.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 1 { + t.Fatalf("population = %d, want 1 (the winner's still in-flight reservation must survive "+ + "the loser's collision cleanup)", population) + } + + // The winner's own eventual failure must still free its slot: the fix must + // not leave it stranded forever. + winner.releaseIfNotConsumed() + population, err = controller.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + if population != 0 { + t.Fatalf("population = %d, want 0 (the winner's own release must still free its slot)", population) + } +} diff --git a/apps/backend/internal/orchestrator/ceiling_session_keyed_reservation.go b/apps/backend/internal/orchestrator/ceiling_session_keyed_reservation.go new file mode 100644 index 0000000000..f108753c9e --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_session_keyed_reservation.go @@ -0,0 +1,93 @@ +package orchestrator + +import ( + "context" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" +) + +// sessionKeyedCeilingReservation tracks an admission reservation taken for a +// session that already exists, shared by seam 2 (AC-4d/AC-4e) and seam 4 +// (AC-4 family) — unlike seam 1's launch-scoped reservation, this one is keyed +// by the session id from the moment it is taken, because both seams are +// called with a session id already in hand. +type sessionKeyedCeilingReservation struct { + controller *sessionCeilingController + key string + consumed bool + + // manualOverride, population, populationKnown and ceiling are the + // admission decision's own reading, carried forward so AC-14/AC-53's + // audit write and card warning can be performed once this reservation is + // consumed by a session that already exists. + manualOverride bool + population int + populationKnown bool + ceiling int +} + +// rekeyToSession moves the reservation from the session it was gated under +// onto the replacement session that actually launches, as one operation so the +// population never momentarily drops. Only seam 2's on_turn_start redirect +// needs this; seam 4 never changes session id mid-resume and simply never +// calls it. +func (r *sessionKeyedCeilingReservation) rekeyToSession(ctx context.Context, sessionID string) { + if r == nil || r.controller == nil || r.consumed || sessionID == "" || sessionID == r.key { + return + } + if r.controller.rekey(ctx, r.key, sessionID) { + r.key = sessionID + } +} + +// consume marks the reservation as belonging to a session that is now actually +// running, so releaseIfNotConsumed becomes a no-op. +func (r *sessionKeyedCeilingReservation) consume() { + if r == nil { + return + } + r.consumed = true +} + +// releaseIfNotConsumed is the deferred cleanup for every return path between +// admission and the launch actually succeeding. +func (r *sessionKeyedCeilingReservation) releaseIfNotConsumed() { + if r == nil || r.controller == nil || r.consumed { + return + } + r.controller.release(r.key) +} + +// admitOrDeferSessionKeyedLaunch is the shared admission/defer sequence for +// every seam whose session already exists at admission time (seams 2 and 4): +// consult the ceiling for the caller's session id, and on refusal persist a +// ceiling_deferred record of the given kind from the caller's own payload. +// failureContext names the launch kind in the error log ("the resume could +// not be admitted or recorded" vs. "the launch could not be admitted or +// recorded") so the two seams keep their own wording. +func (s *Service) admitOrDeferSessionKeyedLaunch( + ctx context.Context, taskID, sessionID string, origin launchOrigin, seam string, + kind models.CeilingLaunchKind, payload map[string]interface{}, failureContext string, +) (*sessionKeyedCeilingReservation, bool, error) { + decision := s.sessionCeiling.admit(ctx, admissionRequest{ + taskID: taskID, sessionID: sessionID, origin: origin, seam: seam, + }) + if decision.admitted { + return &sessionKeyedCeilingReservation{ + controller: s.sessionCeiling, key: decision.reservationKey, + manualOverride: decision.manualOverride, population: decision.population, + populationKnown: decision.populationKnown, ceiling: decision.ceiling, + }, false, nil + } + + if err := s.deferCeilingRefusal(ctx, taskID, sessionID, kind, payload, decision.reasonCode, + decision.population, decision.populationKnown, decision.ceiling); err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; "+failureContext, + zap.String("task_id", taskID), zap.String("session_id", sessionID), + zap.String(ceilingFieldReasonCode, ceilingReasonDeferWriteFailed), zap.Error(err)) + return nil, false, err + } + return nil, true, nil +} diff --git a/apps/backend/internal/orchestrator/ceiling_streaming_release_test.go b/apps/backend/internal/orchestrator/ceiling_streaming_release_test.go new file mode 100644 index 0000000000..fe2d7efe8b --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_streaming_release_test.go @@ -0,0 +1,72 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestPersistTaskSessionState_ReleasesOnlyWhenLeavingThePopulation pins AC-51's +// first persistence funnel: a transition out of the AC-1 population releases +// the reservation, and a transition that stays inside (or enters) it does not. +func TestPersistTaskSessionState_ReleasesOnlyWhenLeavingThePopulation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "streaming-task-leave", "streaming-session-leave", models.TaskSessionStateRunning) + seedTaskAndSession(t, repo, "streaming-task-stay", "streaming-session-stay", models.TaskSessionStateStarting) + + svc := &Service{repo: repo, logger: testLogger(), sessionCeiling: newSessionCeilingController(unlimitedSessionCeiling, nil, nil), ceilingSweeper: newCeilingSweeper()} + + held := func(sessionID string) bool { + svc.sessionCeiling.mu.Lock() + defer svc.sessionCeiling.mu.Unlock() + _, ok := svc.sessionCeiling.reservations[sessionID] + return ok + } + reserve := func(sessionID string) { + decision := svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "t", sessionID: sessionID, origin: launchOriginAutomatic, seam: "test"}) + require.True(t, decision.admitted) + } + + // RUNNING -> FAILED leaves the population. + reserve("streaming-session-leave") + leaveSession, err := repo.GetTaskSession(ctx, "streaming-session-leave") + require.NoError(t, err) + _, _, ok := svc.persistTaskSessionState(ctx, "streaming-session-leave", leaveSession, models.TaskSessionStateFailed, "boom") + require.True(t, ok) + require.False(t, held("streaming-session-leave"), "leaving the population must release the reservation") + + // STARTING -> RUNNING stays inside the population. + reserve("streaming-session-stay") + staySession, err := repo.GetTaskSession(ctx, "streaming-session-stay") + require.NoError(t, err) + _, _, ok = svc.persistTaskSessionState(ctx, "streaming-session-stay", staySession, models.TaskSessionStateRunning, "") + require.True(t, ok) + require.True(t, held("streaming-session-stay"), "staying inside the population must not release the reservation") +} + +// TestPersistStrictTaskSessionState_ReleasesOnlyOnASuccessfulLeavingChange pins +// AC-51's second, independent persistence funnel used by transitionTaskSessionState. +func TestPersistStrictTaskSessionState_ReleasesOnlyOnASuccessfulLeavingChange(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "strict-task-leave", "strict-session-leave", models.TaskSessionStateRunning) + + svc := &Service{repo: repo, logger: testLogger(), sessionCeiling: newSessionCeilingController(unlimitedSessionCeiling, nil, nil), ceilingSweeper: newCeilingSweeper()} + decision := svc.sessionCeiling.admit(ctx, admissionRequest{taskID: "t", sessionID: "strict-session-leave", origin: launchOriginAutomatic, seam: "test"}) + require.True(t, decision.admitted) + + session, err := repo.GetTaskSession(ctx, "strict-session-leave") + require.NoError(t, err) + changed, _, _, err := svc.persistStrictTaskSessionState(ctx, "strict-session-leave", session, models.TaskSessionStateCompleted, "") + require.NoError(t, err) + require.True(t, changed) + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations["strict-session-leave"] + svc.sessionCeiling.mu.Unlock() + require.False(t, held, "a successful strict transition leaving the population must release the reservation") +} diff --git a/apps/backend/internal/orchestrator/ceiling_surface.go b/apps/backend/internal/orchestrator/ceiling_surface.go new file mode 100644 index 0000000000..e6f68dd551 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_surface.go @@ -0,0 +1,139 @@ +package orchestrator + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// ceilingSurfaceRetryBudget bounds AC-49g1's carrier-write retries: at most +// this many consecutive failures before the sweep stops attempting and logs +// once at ERROR, leaving the record deferred and retryable. +const ceilingSurfaceRetryBudget = 5 + +// attemptCeilingSurfaceWrite is AC-49's single surfacing actor for the +// ordinary refusal path (seams 2-5; seam 1 has no session and is excluded). +// It writes the card note for a task's currently-stored ceiling deferral, +// unless one was already written for the current reason (AC-49e) or the +// write has already failed ceilingSurfaceRetryBudget consecutive times for +// this reason (AC-49g1). It is called both inline by deferCeilingRefusal, +// right after a fresh or reason-changed write, and by the sweep's drain pass +// for a launch still deferred this tick (AC-49g). +// +// It always re-reads the current record rather than trusting a caller-held +// snapshot, so a write racing with this one is never overwritten blind. +func (s *Service) attemptCeilingSurfaceWrite(ctx context.Context, taskID string) { + for attempt := 0; attempt < deferredLaunchCASRetryBudget; attempt++ { + record, prior, err := s.repo.GetTaskDeferredLaunch(ctx, taskID) + if err != nil { + s.logger.Zap().Warn("could not read deferred launch to surface a ceiling refusal", + zap.String("task_id", taskID), zap.Error(err)) + return + } + deferral, readErr := models.ReadCeilingDeferral(record) + if readErr != nil { + return + } + sessionID := sessionIDFromCeilingPayload(deferral) + if sessionID == "" { + return + } + if _, written := record[models.CeilingSurfaceWrittenAtKey]; written { + return + } + if ceilingRecordAttemptCount(record) >= ceilingSurfaceRetryBudget { + return + } + + if s.writeCeilingSurfaceNoteAndStamp(ctx, taskID, sessionID, deferral, record, prior) { + return + } + // A lost compare-and-set race: re-read and try again, up to the CAS + // retry budget, mirroring deferCeilingRefusal's own loop. + } +} + +// ceilingRecordAttemptCount reads the AC-49g1 failure counter off a raw +// deferred_launch record, defaulting to zero for an absent or malformed value. +func ceilingRecordAttemptCount(record map[string]interface{}) int { + raw, ok := record[models.CeilingSurfaceAttemptCountKey] + if !ok { + return 0 + } + return models.CeilingRecordInt(raw) +} + +// writeCeilingSurfaceNoteAndStamp attempts the card note write, then stores +// either the AC-49e success stamp or the AC-49g1 failure counter under the +// same compare-and-set the caller read prior from. It reports whether the +// write landed (stored or not, but not a lost compare), so the caller knows +// whether to retry the read-modify-write cycle. +func (s *Service) writeCeilingSurfaceNoteAndStamp( + ctx context.Context, taskID, sessionID string, deferral models.CeilingDeferral, + record map[string]interface{}, prior interface{}, +) bool { + writeErr := s.writeCeilingSurfaceNote(ctx, taskID, sessionID, deferral) + + updated := make(map[string]interface{}, len(record)+1) + for key, value := range record { + updated[key] = value + } + if writeErr == nil { + updated[models.CeilingSurfaceWrittenAtKey] = time.Now().UTC().Format(time.RFC3339) + delete(updated, models.CeilingSurfaceAttemptCountKey) + } else { + count := ceilingRecordAttemptCount(record) + 1 + updated[models.CeilingSurfaceAttemptCountKey] = count + s.logger.Zap().Warn("could not write the ceiling refusal card note; a later sweep will retry", + zap.String("task_id", taskID), zap.String("session_id", sessionID), + zap.String(ceilingFieldReasonCode, ceilingReasonSurfaceWriteFailed), zap.Error(writeErr)) + if count >= ceilingSurfaceRetryBudget { + s.logger.Zap().Error("ceiling refusal card note failed repeatedly; giving up until the reason changes", + zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Int("attempts", count)) + } + } + + stored, lostCompare, err := s.repo.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, prior, updated) + if err != nil { + s.logger.Zap().Warn("could not persist the ceiling card-surface bookkeeping", + zap.String("task_id", taskID), zap.Error(err)) + return true + } + if lostCompare { + return false + } + _ = stored + return true +} + +// writeCeilingSurfaceNote is AC-49's carrier: one session-anchored status +// message row, written through the existing CreateSessionMessage entry point +// createRecoveryStatusMessage already uses for a different variant. It +// carries no recovery actions (AC-49d): a ceiling deferral clears itself on +// its own. +func (s *Service) writeCeilingSurfaceNote(ctx context.Context, taskID, sessionID string, deferral models.CeilingDeferral) error { + if s.messageCreator == nil { + return fmt.Errorf("no message creator is configured") + } + metadata := map[string]interface{}{ + metaKeyVariant: metaVariantCeiling, + ceilingFieldReasonCode: deferral.ReasonCode, + ceilingFieldCeiling: deferral.Ceiling, + metaKeyTaskID: taskID, + metaKeySessionID: sessionID, + } + content := fmt.Sprintf("This session's launch is queued: the instance is at its session ceiling (%s).", deferral.ReasonCode) + if deferral.PopulationKnown { + metadata["population"] = deferral.Population + content = fmt.Sprintf("This session's launch is queued: %d of %d concurrent sessions are in use.", + deferral.Population, deferral.Ceiling) + } + return s.messageCreator.CreateSessionMessage( + ctx, taskID, content, sessionID, string(v1.MessageTypeStatus), "", metadata, false, + ) +} diff --git a/apps/backend/internal/orchestrator/ceiling_surface_test.go b/apps/backend/internal/orchestrator/ceiling_surface_test.go new file mode 100644 index 0000000000..5dc6d1c783 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_surface_test.go @@ -0,0 +1,155 @@ +package orchestrator + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestDeferCeilingRefusal_WritesCardNoteOnFirstRefusal pins AC-49: a +// session-bearing refusal writes the card note inline and stamps +// ceiling_surface_written_at so a later attempt does not repeat it. +func TestDeferCeilingRefusal_WritesCardNoteOnFirstRefusal(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "surface-task", "surface-session", models.TaskSessionStateRunning) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + require.NoError(t, svc.deferCeilingRefusal( + ctx, "surface-task", "surface-session", models.CeilingLaunchResume, + map[string]interface{}{metaKeySessionID: "surface-session"}, ceilingReasonRefused, 3, true, 3, + )) + + require.Len(t, messages.sessionMessages, 1) + note := messages.sessionMessages[0] + require.Equal(t, "surface-session", note.sessionID) + require.Equal(t, metaVariantCeiling, note.metadata[metaKeyVariant]) + require.Equal(t, ceilingReasonRefused, note.metadata[ceilingFieldReasonCode]) + + record := deferredLaunchOf(t, svc, "surface-task") + require.NotEmpty(t, record[models.CeilingSurfaceWrittenAtKey], "the success stamp must be recorded") +} + +// TestDeferCeilingRefusal_SeamOneNeverWritesACardNote pins AC-49's exclusion +// of seam 1: a "start" kind refusal carries no session id, so it must never +// reach the carrier at all. +func TestDeferCeilingRefusal_SeamOneNeverWritesACardNote(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + require.NoError(t, repo.CreateTask(ctx, &models.Task{ID: "surface-seam1-task", Title: "T"})) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + require.NoError(t, svc.deferCeilingRefusal( + ctx, "surface-seam1-task", "", models.CeilingLaunchStart, + map[string]interface{}{"prompt": "p"}, ceilingReasonRefused, 3, true, 3, + )) + + require.Empty(t, messages.sessionMessages) + record := deferredLaunchOf(t, svc, "surface-seam1-task") + require.Nil(t, record[models.CeilingSurfaceWrittenAtKey]) +} + +// TestDeferCeilingRefusal_SameReasonDoesNotRewriteTheNote pins AC-49e: +// suppression is keyed on a note having been successfully written, so a +// repeat refusal for the same unchanged reason must not write a second one. +func TestDeferCeilingRefusal_SameReasonDoesNotRewriteTheNote(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "surface-dup-task", "surface-dup-session", models.TaskSessionStateRunning) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + payload := map[string]interface{}{metaKeySessionID: "surface-dup-session"} + require.NoError(t, svc.deferCeilingRefusal(ctx, "surface-dup-task", "surface-dup-session", models.CeilingLaunchResume, payload, ceilingReasonRefused, 3, true, 3)) + require.NoError(t, svc.deferCeilingRefusal(ctx, "surface-dup-task", "surface-dup-session", models.CeilingLaunchResume, payload, ceilingReasonRefused, 3, true, 3)) + + require.Len(t, messages.sessionMessages, 1, "an unchanged reason must not write a second note") +} + +// TestDeferCeilingRefusal_ReasonChangeRewritesTheRecordAndTheNote pins AC-49f: +// the same pending launch refused again for a different reason updates the +// stored reason code and gets its own note. +func TestDeferCeilingRefusal_ReasonChangeRewritesTheRecordAndTheNote(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "surface-change-task", "surface-change-session", models.TaskSessionStateRunning) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + payload := map[string]interface{}{metaKeySessionID: "surface-change-session"} + require.NoError(t, svc.deferCeilingRefusal(ctx, "surface-change-task", "surface-change-session", models.CeilingLaunchResume, payload, ceilingReasonUnknownPopulation, 0, false, 3)) + require.NoError(t, svc.deferCeilingRefusal(ctx, "surface-change-task", "surface-change-session", models.CeilingLaunchResume, payload, ceilingReasonRefused, 3, true, 3)) + + require.Len(t, messages.sessionMessages, 2, "a reason-code change must write its own note") + require.Equal(t, ceilingReasonUnknownPopulation, messages.sessionMessages[0].metadata[ceilingFieldReasonCode]) + require.Equal(t, ceilingReasonRefused, messages.sessionMessages[1].metadata[ceilingFieldReasonCode]) + + record := deferredLaunchOf(t, svc, "surface-change-task") + require.Equal(t, ceilingReasonRefused, record[models.CeilingReasonCodeKey]) + require.NotEmpty(t, record[models.CeilingSurfaceWrittenAtKey]) +} + +// TestAttemptCeilingSurfaceWrite_RetriesUpToBudgetThenStops pins +// AC-49g/AC-49g1: a carrier write that keeps failing is retried on each +// sweep pass, bounded at five consecutive failures, after which no further +// attempt is made. +func TestAttemptCeilingSurfaceWrite_RetriesUpToBudgetThenStops(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "surface-fail-task", "surface-fail-session", models.TaskSessionStateRunning) + failing := &mockMessageCreator{sessionMessageErr: errors.New("carrier down")} + svc.messageCreator = failing + + require.NoError(t, svc.deferCeilingRefusal( + ctx, "surface-fail-task", "surface-fail-session", models.CeilingLaunchResume, + map[string]interface{}{metaKeySessionID: "surface-fail-session"}, ceilingReasonRefused, 3, true, 3, + )) + require.Equal(t, 1, failing.sessionMessageAttempts) + + // Four more sweep-driven retries reach the five-attempt budget. + for i := 0; i < 4; i++ { + svc.attemptCeilingSurfaceWrite(ctx, "surface-fail-task") + } + require.Equal(t, 5, failing.sessionMessageAttempts) + + // The budget is exhausted: a further tick must not attempt again. + svc.attemptCeilingSurfaceWrite(ctx, "surface-fail-task") + require.Equal(t, 5, failing.sessionMessageAttempts, "must stop attempting once the budget is exhausted") + + record := deferredLaunchOf(t, svc, "surface-fail-task") + require.Nil(t, record[models.CeilingSurfaceWrittenAtKey]) + require.EqualValues(t, 5, record[models.CeilingSurfaceAttemptCountKey]) +} + +// TestAttemptCeilingSurfaceWrite_SucceedsAfterEarlierFailuresStampsWritten +// covers the recovery path: once the carrier starts working again, the next +// attempt succeeds, stamps ceiling_surface_written_at and clears the failure +// counter. +func TestAttemptCeilingSurfaceWrite_SucceedsAfterEarlierFailuresStampsWritten(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "surface-recover-task", "surface-recover-session", models.TaskSessionStateRunning) + messages := &mockMessageCreator{sessionMessageErr: errors.New("carrier down")} + svc.messageCreator = messages + + require.NoError(t, svc.deferCeilingRefusal( + ctx, "surface-recover-task", "surface-recover-session", models.CeilingLaunchResume, + map[string]interface{}{metaKeySessionID: "surface-recover-session"}, ceilingReasonRefused, 3, true, 3, + )) + record := deferredLaunchOf(t, svc, "surface-recover-task") + require.EqualValues(t, 1, record[models.CeilingSurfaceAttemptCountKey]) + + messages.sessionMessageErr = nil + svc.attemptCeilingSurfaceWrite(ctx, "surface-recover-task") + + require.Len(t, messages.sessionMessages, 1) + record = deferredLaunchOf(t, svc, "surface-recover-task") + require.NotEmpty(t, record[models.CeilingSurfaceWrittenAtKey]) + require.Nil(t, record[models.CeilingSurfaceAttemptCountKey]) +} diff --git a/apps/backend/internal/orchestrator/ceiling_sweep.go b/apps/backend/internal/orchestrator/ceiling_sweep.go new file mode 100644 index 0000000000..5f16546ff3 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_sweep.go @@ -0,0 +1,159 @@ +package orchestrator + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" +) + +// ceilingSweepInterval is AC-17a's periodic retry interval, no longer than +// 60 seconds. It is a code constant rather than an env var, for the same +// reason idleReaperInterval is: this card's one env var (AC-18a) is the +// ceiling value itself, not its plumbing. +const ceilingSweepInterval = 20 * time.Second + +// ceilingSweeper owns the AC-50 background goroutine: the AC-17a periodic +// sweep and AC-15's retry-on-release driver are one component, modelled on +// idleSessionReaper ("a single owner of one background goroutine on +// Service"). All methods are nil-safe: a Service with sweeper == nil treats +// the sweep as off, which is what keeps existing tests unchanged. +type ceilingSweeper struct { + mu sync.Mutex + cancel context.CancelFunc + workers sync.WaitGroup + started bool + interval time.Duration + // signal carries release-driven wake-ups (AC-15). Buffered to exactly + // one: a signal received while a pass is already running or already + // pending coalesces into that one further pass rather than queuing + // another (AC-15e). + signal chan struct{} +} + +func newCeilingSweeper() *ceilingSweeper { + return &ceilingSweeper{ + interval: ceilingSweepInterval, + signal: make(chan struct{}, 1), + } +} + +// start launches the sweeper loop with the given tick callback. Idempotent: +// a second call on a running sweeper is a no-op. +func (r *ceilingSweeper) start(parent context.Context, tick func(ctx context.Context)) bool { + if r == nil || tick == nil { + return false + } + if parent == nil { + parent = context.Background() + } + r.mu.Lock() + defer r.mu.Unlock() + if r.started { + return false + } + loopCtx, cancel := context.WithCancel(parent) + r.cancel = cancel + r.started = true + r.workers.Add(1) + go func() { + defer r.workers.Done() + r.runLoop(loopCtx, tick) + }() + return true +} + +// stop signals the sweeper to exit and waits for it. Safe to call on a +// never-started / already-stopped sweeper. +func (r *ceilingSweeper) stop() { + if r == nil { + return + } + r.mu.Lock() + if r.cancel != nil { + r.cancel() + } + r.mu.Unlock() + r.workers.Wait() + r.mu.Lock() + r.cancel = nil + r.started = false + r.mu.Unlock() +} + +// signalNow requests a drain pass promptly (AC-15), coalescing with any pass +// already queued or currently running into exactly one further pass. +func (r *ceilingSweeper) signalNow() { + if r == nil { + return + } + select { + case r.signal <- struct{}{}: + default: + } +} + +// runLoop selects on the ticker, the signal channel and the loop context. +// The tick runs synchronously in this one goroutine, so two ticks — whether +// both timer-driven, both signal-driven, or one of each — can never overlap +// (AC-50a). +func (r *ceilingSweeper) runLoop(ctx context.Context, tick func(ctx context.Context)) { + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + tick(ctx) + case <-r.signal: + tick(ctx) + } + } +} + +// Service-level hooks, mirroring idleSessionReaper's. + +// startCeilingSweeper wires the sweeper into Service.Start. +func (s *Service) startCeilingSweeper(ctx context.Context) { + if s.ceilingSweeper == nil { + return + } + if !s.ceilingSweeper.start(ctx, s.ceilingSweepTick) { + return + } + s.logger.Info("session ceiling sweeper started", + zap.Duration("interval", s.ceilingSweeper.interval)) +} + +// stopCeilingSweeper joins the sweeper. Must be called before Service.Stop +// returns to honour the goroutine-ownership invariant. +func (s *Service) stopCeilingSweeper() { + if s.ceilingSweeper == nil { + return + } + s.ceilingSweeper.stop() + s.logger.Info("session ceiling sweeper stopped") +} + +// signalCeilingSweep requests a prompt drain pass (AC-15). Nil-safe: most +// unit tests build a Service with no sweeper, and a release signalled before +// Start (or after Stop) simply has nothing listening, which is harmless +// because nothing is waiting on it either. +func (s *Service) signalCeilingSweep() { + if s.ceilingSweeper == nil { + return + } + s.ceilingSweeper.signalNow() +} + +// ceilingSweepTick is the AC-50 tick body. It is both AC-7's periodic +// backstop and AC-15/AC-17a's retry driver, because AC-50 states they are +// one component sharing one goroutine. +func (s *Service) ceilingSweepTick(ctx context.Context) { + if s.sessionCeiling != nil { + s.sessionCeiling.expireStaleReservations() + } + s.drainDeferredCeilingLaunches(ctx) +} diff --git a/apps/backend/internal/orchestrator/ceiling_sweep_test.go b/apps/backend/internal/orchestrator/ceiling_sweep_test.go new file mode 100644 index 0000000000..b2c8a91102 --- /dev/null +++ b/apps/backend/internal/orchestrator/ceiling_sweep_test.go @@ -0,0 +1,117 @@ +package orchestrator + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCeilingSweeper_TicksOnIntervalAndStops(t *testing.T) { + r := newCeilingSweeper() + r.interval = 5 * time.Millisecond + var ticks atomic.Int32 + require.True(t, r.start(context.Background(), func(context.Context) { ticks.Add(1) })) + + deadline := time.Now().Add(500 * time.Millisecond) + for ticks.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + require.GreaterOrEqual(t, ticks.Load(), int32(2)) + + r.stop() + afterStop := ticks.Load() + time.Sleep(30 * time.Millisecond) + require.Equal(t, afterStop, ticks.Load(), "no further ticks after stop") +} + +func TestCeilingSweeper_SignalNowTriggersAPromptTick(t *testing.T) { + // A long interval means only a signal, not the ticker, can produce a tick + // within the test's deadline (AC-15). + r := newCeilingSweeper() + r.interval = time.Hour + var ticks atomic.Int32 + require.True(t, r.start(context.Background(), func(context.Context) { ticks.Add(1) })) + defer r.stop() + + r.signalNow() + deadline := time.Now().Add(500 * time.Millisecond) + for ticks.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + require.Equal(t, int32(1), ticks.Load()) +} + +func TestCeilingSweeper_SignalCoalescesWhilePassIsInFlight(t *testing.T) { + // AC-15e: a signal received while a pass is already running or pending + // coalesces into exactly one further pass, not one per signal. + r := newCeilingSweeper() + r.interval = time.Hour + tickStarted := make(chan struct{}, 8) + release := make(chan struct{}) + var ticks atomic.Int32 + require.True(t, r.start(context.Background(), func(context.Context) { + ticks.Add(1) + tickStarted <- struct{}{} + <-release + })) + defer func() { + close(release) + r.stop() + }() + + r.signalNow() + <-tickStarted // first tick is now blocked inside release + + // Every one of these must coalesce into at most one further tick. + r.signalNow() + r.signalNow() + r.signalNow() + + release <- struct{}{} // let the first tick finish + <-tickStarted // the one coalesced pass + require.Equal(t, int32(2), ticks.Load()) + + // No third tick should follow from the extra signals. + select { + case <-tickStarted: + t.Fatal("a third tick fired; signals must coalesce into one further pass") + case <-time.After(50 * time.Millisecond): + } +} + +func TestService_SignalCeilingSweep_NilSweeperIsANoOp(t *testing.T) { + svc := &Service{} + require.NotPanics(t, svc.signalCeilingSweep) +} + +func TestService_StartStopCeilingSweeper_NilSweeperIsANoOp(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + require.NotPanics(t, func() { svc.startCeilingSweeper(context.Background()) }) + require.NotPanics(t, svc.stopCeilingSweeper) +} + +func TestCeilingSweepTick_ExpiresStaleReservationsAndDrains(t *testing.T) { + svc, _ := newServiceWithRealRepo(t) + svc.sessionCeiling = newSessionCeilingController(1, nil, nil) + fixedStart := time.Now() + svc.sessionCeiling.now = func() time.Time { return fixedStart } + + decision := svc.sessionCeiling.admit(context.Background(), admissionRequest{ + taskID: "stale-task", sessionID: "", origin: launchOriginAutomatic, seam: "test", + }) + require.True(t, decision.admitted) + require.NotEmpty(t, decision.reservationKey) + + // Advance the controller's clock well past the launch budget, so the tick's + // AC-7 backstop finds this reservation stale. + svc.sessionCeiling.now = func() time.Time { return fixedStart.Add(24 * time.Hour) } + svc.ceilingSweepTick(context.Background()) + + svc.sessionCeiling.mu.Lock() + _, held := svc.sessionCeiling.reservations[decision.reservationKey] + svc.sessionCeiling.mu.Unlock() + require.False(t, held, "a tick must run the AC-7 backstop and expire a stale reservation") +} diff --git a/apps/backend/internal/orchestrator/ci_automation_dispatch.go b/apps/backend/internal/orchestrator/ci_automation_dispatch.go index 4949360f86..47fc9829d9 100644 --- a/apps/backend/internal/orchestrator/ci_automation_dispatch.go +++ b/apps/backend/internal/orchestrator/ci_automation_dispatch.go @@ -113,7 +113,7 @@ func (s *Service) dispatchCIAutomationPromptToIdleSession( return ciAutomationDispatchResult{}, err } } - promptResult, promptErr := s.promptTask(ctx, session.TaskID, session.ID, params.ChatPrompt, "", false, nil, true, promptTaskOptions{ + promptResult, promptErr := s.promptTask(ctx, session.TaskID, session.ID, params.ChatPrompt, "", false, nil, true, launchOriginAutomatic, promptTaskOptions{ expectedSessionIdentity: &identity, onAccepted: func(acceptedTurnID string) { if params.OnAccepted != nil { diff --git a/apps/backend/internal/orchestrator/clarification_turn_authority_test.go b/apps/backend/internal/orchestrator/clarification_turn_authority_test.go index d4ef1a864b..16078676a4 100644 --- a/apps/backend/internal/orchestrator/clarification_turn_authority_test.go +++ b/apps/backend/internal/orchestrator/clarification_turn_authority_test.go @@ -144,6 +144,7 @@ func TestPromptAdmissionRejectsSupersededClarificationTurn(t *testing.T) { false, nil, false, + launchOriginAutomatic, promptTaskOptions{expectedCurrentTurnID: "turn-clarification"}, ) if err == nil { diff --git a/apps/backend/internal/orchestrator/deferred_launch_consume.go b/apps/backend/internal/orchestrator/deferred_launch_consume.go index c4f5a123ea..591ba5851e 100644 --- a/apps/backend/internal/orchestrator/deferred_launch_consume.go +++ b/apps/backend/internal/orchestrator/deferred_launch_consume.go @@ -4,8 +4,6 @@ import ( "context" "go.uber.org/zap" - - "github.com/kandev/kandev/internal/task/models" ) // deferredLaunchClaim is a direct start's hold on a task's pending launch @@ -31,9 +29,9 @@ import ( // // So a direct start reserves the intent first and holds it across the launch, // which is exactly the protocol the gate already follows (claimDeferredLaunch -// before LaunchSession, restoreDeferredLaunch on failure). One atomic -// RemoveTaskMetadataKey elects a single starter, whichever path gets there -// first, and the loser finds nothing to launch. +// before LaunchSession, restoreDeferredLaunch on failure). One locked +// read-and-remove elects a single starter, whichever path gets there first, and +// the loser finds nothing to launch. // // # Why "started by any means" and not "has any non-cancelled session" // @@ -50,13 +48,17 @@ import ( type deferredLaunchClaim struct { svc *Service taskID string - // intent is the record as it was read at claim time, kept so a failed - // launch can put it back. It is read before the atomic remove rather than - // returned by it, matching restoreDeferredLaunch: a prompt edit landing in - // that gap would be restored at its previous value, which costs one stale - // brief on a launch that already failed. - intent interface{} - held bool + // wip holds exactly the keys this claim removed, returned by the same + // locked operation that removed them, so a failed launch puts back what it + // took and nothing else. + // + // The deferred_launch record is shared: other writers store their own keys + // on it, and one of them can land between this claim and its release. So + // the claim takes only its own keys and the release merges them back rather + // than restoring a whole-record snapshot, which would erase whatever + // arrived in the meantime. + wip map[string]interface{} + held bool } // claimDeferredLaunchForStart reserves a task's pending launch intent for a @@ -65,15 +67,7 @@ type deferredLaunchClaim struct { // launch and this one is simply a second session the user asked for. func (s *Service) claimDeferredLaunchForStart(ctx context.Context, taskID string) *deferredLaunchClaim { claim := &deferredLaunchClaim{svc: s, taskID: taskID} - task, err := s.repo.GetTask(ctx, taskID) - if err != nil || task == nil || task.Metadata == nil { - return claim - } - intent, ok := task.Metadata[models.MetaKeyDeferredLaunch] - if !ok { - return claim - } - claimed, err := s.repo.RemoveTaskMetadataKey(ctx, taskID, models.MetaKeyDeferredLaunch) + wip, claimed, err := s.repo.TakeTaskDeferredLaunchWIPKeys(ctx, taskID) if err != nil { s.logger.Warn("failed to claim deferred launch intent on start", zap.String("task_id", taskID), zap.Error(err)) @@ -82,7 +76,7 @@ func (s *Service) claimDeferredLaunchForStart(ctx context.Context, taskID string if !claimed { return claim } - claim.intent = intent + claim.wip = wip claim.held = true return claim } @@ -115,7 +109,7 @@ func (c *deferredLaunchClaim) releaseIfHeld(ctx context.Context) { return } c.held = false - if err := c.svc.repo.SetTaskMetadataKey(ctx, c.taskID, models.MetaKeyDeferredLaunch, c.intent); err != nil { + if err := c.svc.repo.RestoreTaskDeferredLaunchWIPKeys(ctx, c.taskID, c.wip); err != nil { c.svc.logger.Warn("failed to restore deferred launch intent after a failed start", zap.String("task_id", c.taskID), zap.Error(err)) } diff --git a/apps/backend/internal/orchestrator/deferred_launch_consume_ceiling_test.go b/apps/backend/internal/orchestrator/deferred_launch_consume_ceiling_test.go new file mode 100644 index 0000000000..4df3e5abbe --- /dev/null +++ b/apps/backend/internal/orchestrator/deferred_launch_consume_ceiling_test.go @@ -0,0 +1,149 @@ +package orchestrator + +import ( + "context" + "path/filepath" + "testing" + + "github.com/jmoiron/sqlx" + "go.uber.org/zap" + + commonlogger "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" + tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +func newServiceWithRealRepo(t *testing.T) (*Service, *tasksqlite.Repository) { + t.Helper() + conn, err := db.OpenSQLite(filepath.Join(t.TempDir(), "deferred-launch.db")) + if err != nil { + t.Fatalf("OpenSQLite: %v", err) + } + sqlxDB := sqlx.NewDb(conn, "sqlite3") + t.Cleanup(func() { _ = sqlxDB.Close() }) + repo, err := tasksqlite.NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + t.Fatalf("NewWithDB: %v", err) + } + log, err := commonlogger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatalf("NewFromZap: %v", err) + } + return &Service{repo: repo, logger: log}, repo +} + +func deferredLaunchOf(t *testing.T, s *Service, taskID string) map[string]interface{} { + t.Helper() + task, err := s.repo.GetTask(context.Background(), taskID) + if err != nil { + t.Fatalf("GetTask: %v", err) + } + if task.Metadata == nil { + return nil + } + record, _ := task.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + return record +} + +// TestClaimForStartLeavesACeilingRecordInPlace pins the claim half: a direct +// start takes the launch intent it owns and must not delete the ceiling +// deferral sharing the record. Deleting it is a silently dropped launch that the +// sweep can no longer find. +func TestClaimForStartLeavesACeilingRecordInPlace(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "claim-mixed", Title: "Mixed", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + claim := svc.claimDeferredLaunchForStart(ctx, "claim-mixed") + if !claim.held { + t.Fatal("claim was not held; the WIP intent should have been claimable") + } + + after := deferredLaunchOf(t, svc, "claim-mixed") + if after == nil { + t.Fatal("the whole deferred_launch record was deleted by the claim") + } + if after[models.CeilingDeferredKey] != true || after[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("ceiling keys did not survive the claim: %+v", after) + } + if _, still := after[models.DeferredLaunchStartWhenUnblockedKey]; still { + t.Fatalf("the claim did not take its own key: %+v", after) + } +} + +// TestReleaseIfHeldMergesRatherThanOverwriting pins the release half: a ceiling +// record written while the launch was in flight must survive the restore. The +// pre-claim snapshot predates it, so writing that snapshot back would erase it. +func TestReleaseIfHeldMergesRatherThanOverwriting(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "claim-release", Title: "Release", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + models.DeferredLaunchUserIDKey: "user-7", + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + claim := svc.claimDeferredLaunchForStart(ctx, "claim-release") + if !claim.held { + t.Fatal("claim was not held") + } + if record := deferredLaunchOf(t, svc, "claim-release"); record != nil { + t.Fatalf("a WIP-only record should have been removed entirely, got %+v", record) + } + + // A ceiling refusal lands while the launch is in flight. + if err := repo.SetTaskMetadataKey(ctx, "claim-release", models.MetaKeyDeferredLaunch, map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }); err != nil { + t.Fatalf("SetTaskMetadataKey: %v", err) + } + + claim.releaseIfHeld(ctx) + + after := deferredLaunchOf(t, svc, "claim-release") + if after[models.CeilingDeferredKey] != true || after[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("release overwrote the ceiling record: %+v", after) + } + if after[models.DeferredLaunchStartWhenUnblockedKey] != true || after[models.DeferredLaunchUserIDKey] != "user-7" { + t.Fatalf("release did not put the claimed keys back: %+v", after) + } +} + +// TestClaimForStartDoesNotClaimACeilingOnlyRecord keeps the two meanings +// separate in the other direction: a ceiling deferral is replayed by its own +// sweep, so a direct start has nothing to claim and must leave it untouched. +func TestClaimForStartDoesNotClaimACeilingOnlyRecord(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "claim-ceiling-only", Title: "CeilingOnly", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.CeilingDeferredKey: true, + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + claim := svc.claimDeferredLaunchForStart(ctx, "claim-ceiling-only") + if claim.held { + t.Fatal("a ceiling-only record must not be claimed by the direct-start path") + } + if after := deferredLaunchOf(t, svc, "claim-ceiling-only"); after[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling record disturbed: %+v", after) + } +} diff --git a/apps/backend/internal/orchestrator/deferred_launch_orchestrator_write_protection_test.go b/apps/backend/internal/orchestrator/deferred_launch_orchestrator_write_protection_test.go new file mode 100644 index 0000000000..629e23ca91 --- /dev/null +++ b/apps/backend/internal/orchestrator/deferred_launch_orchestrator_write_protection_test.go @@ -0,0 +1,82 @@ +package orchestrator + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// deferredLaunchRaceRepo wraps the real repository and, on the first GetTask +// call for a watched task, writes a fresh deferred_launch record through the +// repository's own CAS primitive before returning — modeling the session +// ceiling's admission controller mutating deferred_launch in the window +// between an orchestrator handler's read and its write-back. +type deferredLaunchRaceRepo struct { + *sqliterepo.Repository + t *testing.T + watchTaskID string + fired bool +} + +func (r *deferredLaunchRaceRepo) GetTask(ctx context.Context, id string) (*models.Task, error) { + task, err := r.Repository.GetTask(ctx, id) + if id == r.watchTaskID && !r.fired { + r.fired = true + _, prior, priorErr := r.GetTaskDeferredLaunch(ctx, id) + require.NoError(r.t, priorErr) + stored, lostCompare, setErr := r.SetTaskDeferredLaunchIfUnchanged(ctx, id, prior, + map[string]interface{}{"prompt": "original", "ceiling_deferred": true}) + require.NoError(r.t, setErr) + require.False(r.t, lostCompare) + require.True(r.t, stored) + } + return task, err +} + +// TestMarkTaskCompletedForTerminalStep_PreservesConcurrentDeferredLaunchWrite +// pins R2R-B at a real orchestrator entry point (markTaskCompletedForTerminalStep, +// one of the nine call sites that used to write through plain UpdateTask): +// the ceiling admission controller's own CAS write to deferred_launch, landing +// between this handler's GetTask and its own write-back, must survive. Before +// the fix, UpdateTask wrote the handler's stale in-memory metadata verbatim +// and silently erased the concurrently-written ceiling_deferred record. +func TestMarkTaskCompletedForTerminalStep_PreservesConcurrentDeferredLaunchWrite(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "term-deferred-race", "term-deferred-race-session", models.TaskSessionStateCompleted) + + task, err := repo.GetTask(ctx, "term-deferred-race") + require.NoError(t, err) + task.WorkflowStepID = "term-step" + task.UpdatedAt = time.Now().UTC() + require.NoError(t, repo.UpdateTask(ctx, task)) + + stored, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "term-deferred-race", + sqliterepo.AbsentDeferredLaunch(), map[string]interface{}{"prompt": "original"}) + require.NoError(t, err) + require.False(t, lostCompare) + require.True(t, stored) + + raceRepo := &deferredLaunchRaceRepo{Repository: repo, t: t, watchTaskID: "term-deferred-race"} + svc := &Service{repo: raceRepo, logger: testLogger()} + + svc.markTaskCompletedForTerminalStep(ctx, "term-deferred-race", "term-step") + + require.True(t, raceRepo.fired, "the concurrent race must have actually been injected") + + current, err := repo.GetTask(ctx, "term-deferred-race") + require.NoError(t, err) + require.Equal(t, v1.TaskStateCompleted, current.State, "the terminal-step completion must still apply") + + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "deferred_launch missing or wrong shape: %#v", current.Metadata[models.MetaKeyDeferredLaunch]) + require.Equal(t, true, deferred["ceiling_deferred"], + "a plain UpdateTask through this orchestrator path must not clobber a concurrent ceiling CAS write") +} diff --git a/apps/backend/internal/orchestrator/dynamic_launch.go b/apps/backend/internal/orchestrator/dynamic_launch.go index 8a62c40167..c254af763d 100644 --- a/apps/backend/internal/orchestrator/dynamic_launch.go +++ b/apps/backend/internal/orchestrator/dynamic_launch.go @@ -299,6 +299,14 @@ func (s *Service) handleAgentProcessStarted( ctx context.Context, _, sessionID, agentExecutionID string, ) { + if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) { + return + } + // AC-52's acceptance edge, composed first and unconditionally on + // sessionID alone (AC-56a): profileExecutionResolver is a dynamic-routing + // precondition, not a launch one, so an instance without it configured + // must still confirm the reservation on every ordinary launch. + s.confirmCeilingReservation(sessionID) if s.profileExecutionResolver == nil || sessionID == "" { return } @@ -329,6 +337,12 @@ func (s *Service) handleAgentProcessStartFailed( _, sessionID, agentExecutionID string, _ error, ) { + if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) { + return + } + // AC-52's failure edge, composed first and unconditionally on sessionID + // alone (AC-56a) — see handleAgentProcessStarted's acceptance edge. + s.releaseCeilingReservation(sessionID) if s.profileExecutionResolver == nil || sessionID == "" { return } @@ -838,7 +852,10 @@ func (s *Service) launchDynamicSuccessorDetached( // runDetachedDynamicSuccessorLaunch serializes with the session's cancel guard // like every other session-backed failure decision, then launches the // successor. A launch that cannot complete falls back to the ordinary -// recoverable-failure surface instead of leaving the session RUNNING. +// recoverable-failure surface instead of leaving the session RUNNING. A +// ceiling deferral is different: the durable replay record owns the retry, so +// the automation run must remain open until that replay either succeeds or +// fails for a non-ceiling reason. func (s *Service) runDetachedDynamicSuccessorLaunch( ctx context.Context, data watcher.AgentEventData, @@ -879,7 +896,14 @@ func (s *Service) runDetachedDynamicSuccessorLaunch( zap.Error(ctx.Err())) return } - if s.relaunchDynamicTaskAfterFailure(ctx, data, executionProfileID) { + switch s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, launchOriginAutomatic) { + case dynamicRelaunchSucceeded: + return + case dynamicRelaunchDeferred: + s.logger.Debug("dynamic successor launch deferred by session ceiling", + zap.String("task_id", data.TaskID), + zap.String("session_id", data.SessionID), + zap.String("execution_profile_id", executionProfileID)) return } if errors.Is(ctx.Err(), context.Canceled) { @@ -943,6 +967,17 @@ func (s *Service) stopDynamicSuccessorWorkers() { // and final launch error surface so the UI never has to issue a second launch // request. func (s *Service) LaunchDynamicRouteAction(ctx context.Context, sessionID string) error { + return s.launchDynamicRouteAction(ctx, sessionID, launchOriginManual) +} + +// LaunchDynamicRouteActionForRecovery is the unattended counterpart used by +// the policy recovery timer sweep: same successor-launch transaction, but the +// session was never routed by a human action in this call. +func (s *Service) LaunchDynamicRouteActionForRecovery(ctx context.Context, sessionID string) error { + return s.launchDynamicRouteAction(ctx, sessionID, launchOriginAutomatic) +} + +func (s *Service) launchDynamicRouteAction(ctx context.Context, sessionID string, origin launchOrigin) error { if s.profileExecutionResolver == nil || sessionID == "" { return errors.New("dynamic route action launch is not configured") } @@ -962,14 +997,15 @@ func (s *Service) LaunchDynamicRouteAction(ctx context.Context, sessionID string succeeded := false defer func() { if !succeeded { - s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, "manual dynamic route action failed") + s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, string(origin)+" dynamic route action failed") } }() task, err := s.scheduler.GetTask(ctx, session.TaskID) if err != nil { return err } - input, err := s.buildDynamicContinuation(ctx, task, sessionID, "", "manual dynamic route action") + reason := string(origin) + " dynamic route action" + input, err := s.buildDynamicContinuation(ctx, task, sessionID, "", reason) if err != nil { return err } @@ -994,9 +1030,9 @@ func (s *Service) LaunchDynamicRouteAction(ctx context.Context, sessionID string AgentExecutionID: session.AgentExecutionID, AgentProfileID: session.AgentProfileID, ExecutionProfileID: session.ExecutionProfileID, - ErrorMessage: "manual dynamic route action", + ErrorMessage: reason, } - if !s.relaunchDynamicTaskAfterFailure(ctx, data, session.ExecutionProfileID) { + if !s.relaunchDynamicTaskAfterFailure(ctx, data, session.ExecutionProfileID, origin) { return errors.New("dynamic route action successor launch failed") } succeeded = true @@ -1021,13 +1057,46 @@ func (s *Service) dynamicFailureSession( return session, true } +type dynamicRelaunchOutcome uint8 + +const ( + dynamicRelaunchFailed dynamicRelaunchOutcome = iota + dynamicRelaunchDeferred + dynamicRelaunchSucceeded +) + +// relaunchDynamicTaskAfterFailure preserves the historical boolean API for +// synchronous route actions. Callers that own terminalization use the richer +// outcome so a durable ceiling deferral is not mistaken for a failed launch. func (s *Service) relaunchDynamicTaskAfterFailure( ctx context.Context, data watcher.AgentEventData, executionProfileID string, -) (succeeded bool) { + origin launchOrigin, +) bool { + return s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, origin) == dynamicRelaunchSucceeded +} + +func (s *Service) relaunchDynamicTaskAfterFailureOutcome( + ctx context.Context, + data watcher.AgentEventData, + executionProfileID string, + origin launchOrigin, +) (outcome dynamicRelaunchOutcome) { + seam5Res, deferredLaunch, err := s.admitOrDeferSeam5(ctx, data.TaskID, origin, seam5DynamicRelaunchPayload(data, executionProfileID)) + if err != nil { + s.logger.Zap().Error("could not persist a ceiling deferral; the dynamic relaunch could not be admitted or recorded", + zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.Error(err)) + return dynamicRelaunchFailed + } + if deferredLaunch { + return dynamicRelaunchDeferred + } + defer seam5Res.releaseIfNotConsumed() + s.recordManualOverrideIfAdmitted(ctx, data.TaskID, data.SessionID, seam5Res.manualOverride, seam5Res.population, seam5Res.populationKnown, seam5Res.ceiling) + defer func() { - if succeeded || s.profileExecutionResolver == nil || data.SessionID == "" { + if outcome == dynamicRelaunchSucceeded || s.profileExecutionResolver == nil || data.SessionID == "" { return } // This helper is used by both the synchronous route-failure path and @@ -1048,32 +1117,61 @@ func (s *Service) relaunchDynamicTaskAfterFailure( ) }() + task, session, prompt, ok := s.prepareDynamicRelaunchAfterFailure(ctx, data) + if !ok { + return dynamicRelaunchFailed + } + return s.launchPreparedDynamicRelaunch(ctx, data, task, session, prompt, executionProfileID, seam5Res) +} + +func (s *Service) prepareDynamicRelaunchAfterFailure( + ctx context.Context, + data watcher.AgentEventData, +) (*v1.Task, *models.TaskSession, capturedPrompt, bool) { prompt, ok := s.dynamicRelaunchPrompt(ctx, data.SessionID) if !ok { - return false + return nil, nil, capturedPrompt{}, false } task, err := s.scheduler.GetTask(ctx, data.TaskID) if err != nil { - return false + return nil, nil, capturedPrompt{}, false } session, err := s.repo.GetTaskSession(ctx, data.SessionID) if err != nil || session == nil { - return false + return nil, nil, capturedPrompt{}, false } - if err := s.executor.StopExecution(ctx, data.AgentExecutionID, "dynamic route fallback", true); err != nil { - if errors.Is(err, agentruntime.ErrNotFound) { - s.logger.Debug("dynamic fallback predecessor is already absent", zap.Error(err)) - } else { - s.logger.Debug("failed to stop dynamic fallback predecessor", zap.Error(err)) - return false - } + if !s.stopDynamicRelaunchPredecessor(ctx, data.AgentExecutionID) { + return nil, nil, capturedPrompt{}, false } + if !s.resetDynamicRelaunchSession(ctx, data.SessionID) { + return nil, nil, capturedPrompt{}, false + } + s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "") + s.completeTurnForSession(ctx, data.SessionID) + s.retireExecutionActivityAndPublish(ctx, data.TaskID, data.SessionID, data.AgentExecutionID) + return task, session, prompt, true +} + +func (s *Service) stopDynamicRelaunchPredecessor(ctx context.Context, agentExecutionID string) bool { + err := s.executor.StopExecution(ctx, agentExecutionID, "dynamic route fallback", true) + if err == nil { + return true + } + if errors.Is(err, agentruntime.ErrNotFound) { + s.logger.Debug("dynamic fallback predecessor is already absent", zap.Error(err)) + return true + } + s.logger.Debug("failed to stop dynamic fallback predecessor", zap.Error(err)) + return false +} + +func (s *Service) resetDynamicRelaunchSession(ctx context.Context, sessionID string) bool { // Reload immediately before the reset: StopExecution is I/O and a // coordinator stop can commit a terminal state while it runs. A stale // pre-stop snapshot would let this write resurrect a session the user // already cancelled, so refuse on a reloaded terminal state and CAS the // write on that same reload to catch a cancellation landing after it. - preResetState, err := s.repo.GetTaskSession(ctx, data.SessionID) + preResetState, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil || preResetState == nil { return false } @@ -1081,14 +1179,20 @@ func (s *Service) relaunchDynamicTaskAfterFailure( return false } changed, _, err := s.repo.UpdateTaskSessionStateIfCurrent( - ctx, data.SessionID, preResetState.State, models.TaskSessionStateCreated, "", + ctx, sessionID, preResetState.State, models.TaskSessionStateCreated, "", ) - if err != nil || !changed { - return false - } - s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "") - s.completeTurnForSession(ctx, data.SessionID) - s.retireExecutionActivityAndPublish(ctx, data.TaskID, data.SessionID, data.AgentExecutionID) + return err == nil && changed +} + +func (s *Service) launchPreparedDynamicRelaunch( + ctx context.Context, + data watcher.AgentEventData, + task *v1.Task, + session *models.TaskSession, + prompt capturedPrompt, + executionProfileID string, + seam5Res *sessionKeyedCeilingReservation, +) dynamicRelaunchOutcome { officeAgentProfileID := data.AgentProfileID if officeAgentProfileID == "" { officeAgentProfileID = session.AgentProfileID @@ -1099,9 +1203,13 @@ func (s *Service) relaunchDynamicTaskAfterFailure( ctx, data.TaskID, data.SessionID, session.AgentProfileID, prompt.text, true, prompt.planMode, true, prompt.attachments, nil, ) - return err == nil + if err != nil { + return dynamicRelaunchFailed + } + seam5Res.consume() + return dynamicRelaunchSucceeded } - _, err = s.launchPreparedSessionWithDynamicFallback(ctx, task, data.SessionID, executor.LaunchOptions{ + _, err := s.launchPreparedSessionWithDynamicFallback(ctx, task, data.SessionID, executor.LaunchOptions{ AgentProfileID: executionProfileID, OfficeAgentProfileID: officeAgentProfileID, ExecutorID: "", @@ -1109,7 +1217,11 @@ func (s *Service) relaunchDynamicTaskAfterFailure( StartAgent: true, McpMode: executor.McpModeOffice, }) - return err == nil + if err != nil { + return dynamicRelaunchFailed + } + seam5Res.consume() + return dynamicRelaunchSucceeded } // dynamicRelaunchPrompt prefers the in-memory prompt cache for an automatic diff --git a/apps/backend/internal/orchestrator/dynamic_launch_identity_test.go b/apps/backend/internal/orchestrator/dynamic_launch_identity_test.go index 6ecbbd3b8e..deda337bb2 100644 --- a/apps/backend/internal/orchestrator/dynamic_launch_identity_test.go +++ b/apps/backend/internal/orchestrator/dynamic_launch_identity_test.go @@ -78,7 +78,7 @@ func TestRelaunchDynamicTaskAfterFailurePreservesActingOfficeIdentity(t *testing SessionID: sessionID, AgentExecutionID: executionID, AgentProfileID: "reviewer", - }, "fallback-profile")) + }, "fallback-profile", launchOriginAutomatic)) require.NotNil(t, launchRequest) require.Equal(t, "reviewer", launchRequest.OfficeAgentProfileID, "dynamic successor must keep the acting Office identity from the failure event") diff --git a/apps/backend/internal/orchestrator/dynamic_launch_test.go b/apps/backend/internal/orchestrator/dynamic_launch_test.go index 78aac97d93..dae6046493 100644 --- a/apps/backend/internal/orchestrator/dynamic_launch_test.go +++ b/apps/backend/internal/orchestrator/dynamic_launch_test.go @@ -40,6 +40,7 @@ func TestRelaunchDynamicTaskAfterFailure_DoesNotLaunchSuccessorWhenStopFails(t * AgentExecutionID: executionID, }, "fallback-profile", + launchOriginAutomatic, ) if relaunched { @@ -108,6 +109,7 @@ func TestRelaunchDynamicTaskAfterFailure_DoesNotResurrectCancelledSession(t *tes AgentExecutionID: executionID, }, "fallback-profile", + launchOriginAutomatic, ) if relaunched { diff --git a/apps/backend/internal/orchestrator/dynamic_policy_recovery.go b/apps/backend/internal/orchestrator/dynamic_policy_recovery.go index 3010608f5f..252b17417d 100644 --- a/apps/backend/internal/orchestrator/dynamic_policy_recovery.go +++ b/apps/backend/internal/orchestrator/dynamic_policy_recovery.go @@ -189,7 +189,7 @@ func (s *Service) runDynamicPolicyRecovery(ctx context.Context, sessionID string lock.Unlock() release() locked = false - if err := s.LaunchDynamicRouteAction(ctx, sessionID); err != nil { + if err := s.LaunchDynamicRouteActionForRecovery(ctx, sessionID); err != nil { s.markDynamicPolicyRecoveryActionRequired(ctx, sessionID, generation, err) } } diff --git a/apps/backend/internal/orchestrator/ensure_session_running_cold_resume_test.go b/apps/backend/internal/orchestrator/ensure_session_running_cold_resume_test.go index bd24da13d9..2ea05b56a7 100644 --- a/apps/backend/internal/orchestrator/ensure_session_running_cold_resume_test.go +++ b/apps/backend/internal/orchestrator/ensure_session_running_cold_resume_test.go @@ -12,6 +12,40 @@ import ( v1 "github.com/kandev/kandev/pkg/api/v1" ) +func TestColdResumeSession_CeilingRefusalKeepsStartupAttemptOwnership(t *testing.T) { + ctx := context.Background() + svc, repo := newServiceWithCeiling(t, 1) + seedTaskAndSession(t, repo, "task-cold-refused", "session-cold-refused", models.TaskSessionStateWaitingForInput) + session, err := repo.GetTaskSession(ctx, "session-cold-refused") + if err != nil { + t.Fatalf("load session: %v", err) + } + + decision := svc.sessionCeiling.admit(ctx, admissionRequest{ + taskID: "task-held", sessionID: "session-held", origin: launchOriginAutomatic, seam: "test", + }) + if !decision.admitted { + t.Fatal("failed to occupy the only session-ceiling slot") + } + + startupAttempt, owner, err := svc.beginResumeAttempt(ctx, session.TaskID, session.ID) + if err != nil || !owner || startupAttempt == nil { + t.Fatalf("begin resume attempt: attempt=%v owner=%t err=%v", startupAttempt, owner, err) + } + t.Cleanup(func() { startupAttempt.finish(svc.resumeAttemptStore()) }) + + got, err := svc.coldResumeSession(ctx, session.ID, session, false, startupAttempt, launchOriginAutomatic) + if _, ok := isSeam3Refusal(err); !ok { + t.Fatalf("cold resume error = %v, want a seam-3 refusal", err) + } + if got != startupAttempt { + t.Fatalf("cold resume returned attempt %p, want the registered attempt %p", got, startupAttempt) + } + if current, ok := svc.resumeAttemptStore().current(session.ID); !ok || current != startupAttempt { + t.Fatal("the startup attempt was lost after a ceiling refusal") + } +} + // TestEnsureSessionRunning_ColdResumeReapsAndRetriesOnPromptReadyTimeout pins // down a self-healing gap specific to the "kandev restart" cold-resume path: // when the backend restarts, the in-memory execution store is empty even @@ -150,7 +184,7 @@ func TestEnsureSessionRunning_ColdResumeReapsAndRetriesOnPromptReadyTimeout(t *t t.Fatalf("reload: %v", err) } - if err := svc.ensureSessionRunning(ctx, "session1", session); err != nil { + if err := svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual); err != nil { t.Fatalf("expected cold resume to reap the wedged launch and retry, got: %v", err) } if got := launchCalls.Load(); got != 2 { diff --git a/apps/backend/internal/orchestrator/event_handlers_agent.go b/apps/backend/internal/orchestrator/event_handlers_agent.go index 88aefb02e6..59d6d0e33b 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent.go @@ -1470,6 +1470,7 @@ func (s *Service) executeQueuedMessageWithReservation( } _, err := s.promptTask(promptCtx, queuedMsg.TaskID, queuedMsg.SessionID, promptContent, queuedMsg.Model, queuedMsg.PlanMode, attachments, false, + launchOriginAutomatic, promptTaskOptions{ claimEntryID: claimEntryID, lifecyclePrompt: lifecyclePrompt, @@ -1723,13 +1724,14 @@ func (s *Service) handleQueuedMessageExecutionError( zap.Error(err)) manualRecovery := isManualRecoveryPromptError(err) + _, seam3Refusal := isSeam3Refusal(err) passthroughAttachmentRecovery := !lifecyclePrompt && len(queuedMsg.Attachments) > 0 && s.agentManager != nil && s.agentManager.IsPassthroughSession(ctx, queuedMsg.SessionID) if passthroughAttachmentRecovery || lifecyclePrompt || queuedMsg.IsDurablePlanComment() || errors.Is(err, errLifecyclePromptClaim) || errors.Is(err, errLifecyclePromptMessagePersistence) || - isSessionBusyError(err) || isTransientPromptError(err) || manualRecovery || + isSessionBusyError(err) || isTransientPromptError(err) || manualRecovery || seam3Refusal || errors.Is(err, lifecycle.ErrCancelEscalated) || isSessionResetInProgressError(err) || errors.Is(err, ErrSessionRuntimeUnavailable) { if userMessageRecorded { diff --git a/apps/backend/internal/orchestrator/event_handlers_automation.go b/apps/backend/internal/orchestrator/event_handlers_automation.go index 52c212a0df..0af92447d5 100644 --- a/apps/backend/internal/orchestrator/event_handlers_automation.go +++ b/apps/backend/internal/orchestrator/event_handlers_automation.go @@ -797,7 +797,7 @@ func (s *Service) promptAutomationContinuation( session *models.TaskSession, prompt string, ) (automation.RunDispatch, error) { - result, err := s.PromptTask(ctx, task.ID, session.ID, prompt, "", false, nil, true) + result, err := s.promptTask(ctx, task.ID, session.ID, prompt, "", false, nil, true, launchOriginAutomatic, promptTaskOptions{}) if err != nil { return automation.RunDispatch{}, err } diff --git a/apps/backend/internal/orchestrator/event_handlers_children_completed.go b/apps/backend/internal/orchestrator/event_handlers_children_completed.go index 7c3c2098c3..333c89815f 100644 --- a/apps/backend/internal/orchestrator/event_handlers_children_completed.go +++ b/apps/backend/internal/orchestrator/event_handlers_children_completed.go @@ -86,7 +86,7 @@ func (s *Service) markTaskCompletedForTerminalStep(ctx context.Context, taskID, oldState := task.State task.State = v1.TaskStateCompleted task.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTask(ctx, task); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { s.taskRuntimeStateMu.Unlock() s.logger.Warn("terminal step completion: failed to mark task completed", zap.String("task_id", taskID), diff --git a/apps/backend/internal/orchestrator/event_handlers_clarification.go b/apps/backend/internal/orchestrator/event_handlers_clarification.go index bac68eca9f..115da4c24d 100644 --- a/apps/backend/internal/orchestrator/event_handlers_clarification.go +++ b/apps/backend/internal/orchestrator/event_handlers_clarification.go @@ -242,7 +242,7 @@ func (s *Service) resumeDetachedClarificationWithPrompt( s.writeTaskInProgressForRuntime(ctx, data.TaskID, data.SessionID) if _, err := s.promptTask( - ctx, data.TaskID, data.SessionID, prompt, "", false, nil, dispatchOnly, options, + ctx, data.TaskID, data.SessionID, prompt, "", false, nil, dispatchOnly, launchOriginManual, options, ); err != nil { // The synchronous HTTP path must not turn an asynchronous queue handoff // into false acknowledgement. Its handler restores the claimed bundle on @@ -405,6 +405,7 @@ func (s *Service) resumeClarificationViaFallback(ctx context.Context, data clari false, nil, false, + launchOriginAutomatic, promptTaskOptions{expectedCurrentTurnID: data.ClarificationTurnID}, ); err != nil { if !s.retryClarificationAfterCancel(ctx, data, prompt, err) { diff --git a/apps/backend/internal/orchestrator/event_handlers_github.go b/apps/backend/internal/orchestrator/event_handlers_github.go index e2b0444180..4d24a68d0c 100644 --- a/apps/backend/internal/orchestrator/event_handlers_github.go +++ b/apps/backend/internal/orchestrator/event_handlers_github.go @@ -669,6 +669,13 @@ func (s *Service) autoStartReviewTask( true, nil, ) + if errors.Is(err, ErrCeilingLaunchDeferred) { + s.logger.Info("review auto-start deferred by session ceiling; will replay once capacity frees up", + zap.String("task_id", task.ID), + zap.Int("pr_number", evt.PR.Number)) + s.completeAutoStartOnCreate(ctx, task.ID, "review.auto_start") + return + } if err != nil { s.logger.Error("failed to auto-start review task", zap.String("task_id", task.ID), @@ -1721,6 +1728,12 @@ func (s *Service) autoStartIssueTask( true, nil, ) + if errors.Is(err, ErrCeilingLaunchDeferred) { + s.logger.Info("issue auto-start deferred by session ceiling; will replay once capacity frees up", + zap.String("task_id", task.ID), + zap.Int(issueNumberKey, evt.Issue.Number)) + return + } if err != nil { s.logger.Error("failed to auto-start issue task", zap.String("task_id", task.ID), diff --git a/apps/backend/internal/orchestrator/event_handlers_queue_round64_test.go b/apps/backend/internal/orchestrator/event_handlers_queue_round64_test.go index 59986612db..f40a865208 100644 --- a/apps/backend/internal/orchestrator/event_handlers_queue_round64_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_queue_round64_test.go @@ -127,6 +127,37 @@ func TestLifecycleManualRecoveryReplacesReservedRow(t *testing.T) { } } +func TestQueuedMessageExecutionError_RequeuesSeam3Refusal(t *testing.T) { + ctx := context.Background() + repo := messagequeue.NewMemoryRepository() + queue := messagequeue.NewService(repo, messagequeue.DefaultMaxPerSession, testLogger()) + svc := &Service{messageQueue: queue, logger: testLogger()} + + queued, err := queue.QueueMessage( + ctx, "session-seam3-retry", "task-seam3-retry", "retry me", "", messagequeue.QueuedByAgent, false, nil, + ) + if err != nil { + t.Fatalf("queue message: %v", err) + } + reserved, dispatched, autoRun := queue.ReserveQueuedWithAutoRun(ctx, queued.SessionID) + if !autoRun || !dispatched || reserved == nil { + t.Fatalf("reserve message: auto_run=%t dispatched=%t message=%v", autoRun, dispatched, reserved) + } + + svc.handleQueuedMessageExecutionError( + ctx, queued.SessionID, reserved, nil, false, false, + &seam3Refusal{reasonCode: ceilingReasonRefused}, + ) + + entries, err := repo.ListBySession(ctx, queued.SessionID) + if err != nil { + t.Fatalf("list requeued messages: %v", err) + } + if len(entries) != 1 || entries[0].ID != queued.ID || entries[0].IsReservedInFlight() { + t.Fatalf("seam-3 refusal queue state = %#v, want one unreserved original message", entries) + } +} + func TestReuseSessionRestoresPrimaryWhenQueueTransferFails(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/orchestrator/event_handlers_streaming.go b/apps/backend/internal/orchestrator/event_handlers_streaming.go index 6964b7e186..c999631a62 100644 --- a/apps/backend/internal/orchestrator/event_handlers_streaming.go +++ b/apps/backend/internal/orchestrator/event_handlers_streaming.go @@ -1214,6 +1214,7 @@ func (s *Service) persistTaskSessionState( nextState models.TaskSessionState, errorMessage string, ) (*models.TaskSession, *time.Time, bool) { + priorState := session.State if updater, ok := s.repo.(conditionalTaskSessionStateUpdater); ok { changed, updatedAt, err := updater.UpdateTaskSessionStateIfCurrent( ctx, sessionID, session.State, nextState, errorMessage, @@ -1228,6 +1229,7 @@ func (s *Service) persistTaskSessionState( persisted := taskSessionAfterStateWrite(session, nextState, errorMessage, updatedAt) persisted = s.refreshTaskSessionOr(ctx, sessionID, persisted) t := updatedAt.UTC() + s.releaseCeilingIfLeftPopulation(sessionID, priorState, nextState) return persisted, &t, true } @@ -1236,6 +1238,7 @@ func (s *Service) persistTaskSessionState( return session, nil, false } refreshed := s.refreshTaskSessionOr(ctx, sessionID, session) + s.releaseCeilingIfLeftPopulation(sessionID, priorState, nextState) if refreshed.UpdatedAt.IsZero() { return refreshed, nil, true } @@ -1469,12 +1472,32 @@ func (s *Service) publishAcceptedTaskSessionState( s.maybePromotePrimary(ctx, taskID, sessionID, nextState) } +// persistStrictTaskSessionState is transitionTaskSessionState's persistence +// funnel — a second, independent funnel from persistTaskSessionState (AC-51). +// It releases the ceiling reservation itself, after dispatching to whichever +// branch actually performed the write, so the release applies uniformly +// however the write was made. func (s *Service) persistStrictTaskSessionState( ctx context.Context, sessionID string, session *models.TaskSession, nextState models.TaskSessionState, errorMessage string, +) (bool, *models.TaskSession, *time.Time, error) { + priorState := session.State + changed, refreshed, updatedAt, err := s.persistStrictTaskSessionStateDispatch(ctx, sessionID, session, nextState, errorMessage) + if err == nil && changed { + s.releaseCeilingIfLeftPopulation(sessionID, priorState, nextState) + } + return changed, refreshed, updatedAt, err +} + +func (s *Service) persistStrictTaskSessionStateDispatch( + ctx context.Context, + sessionID string, + session *models.TaskSession, + nextState models.TaskSessionState, + errorMessage string, ) (bool, *models.TaskSession, *time.Time, error) { if canceller, ok := s.repo.(activeTaskSessionCanceller); ok && nextState == models.TaskSessionStateCancelled { return s.cancelActiveTaskSessionState(ctx, canceller, sessionID, session, errorMessage) diff --git a/apps/backend/internal/orchestrator/event_handlers_transient.go b/apps/backend/internal/orchestrator/event_handlers_transient.go index bd27fe22ae..aca6ddf0bb 100644 --- a/apps/backend/internal/orchestrator/event_handlers_transient.go +++ b/apps/backend/internal/orchestrator/event_handlers_transient.go @@ -31,19 +31,28 @@ const recoveryCancelRetryButtonTestID = "recovery-cancel-retry-button" // keys are built in more than one place in this package (recovery + retry // status messages), which otherwise trips goconst on new code. const ( - metaKeyVariant = "variant" - metaKeySessionID = "session_id" - metaKeyTaskID = "task_id" - metaKeyAgentID = "agent_id" - metaKeyNewState = "new_state" - metaKeyAgentProfileID = "agent_profile_id" - metaKeyUpdatedAt = "updated_at" + metaKeyVariant = "variant" + metaKeySessionID = "session_id" + metaKeyTaskID = "task_id" + metaKeyAgentID = "agent_id" + metaKeyNewState = "new_state" + metaKeyAgentProfileID = "agent_profile_id" + metaKeyUpdatedAt = "updated_at" + metaKeyExecutorProfile = "executor_profile_id" + metaKeyWorkflowStepID = "workflow_step_id" + metaKeyPrompt = "prompt" + metaKeyPlanMode = "plan_mode" + metaKeyAttachments = "attachments" ) // metaVariantWarning is the status-message variant that drives the frontend's // yellow (non-alarming) styling, as opposed to the red "error" variant. const metaVariantWarning = "warning" +// metaVariantCeiling is the status-message variant AC-49 requires for every +// session-ceiling card note, including AC-17c's drop note. +const metaVariantCeiling = "ceiling" + // transientRetryBackoff is the per-attempt delay before re-driving a turn that // failed transiently. Index is attempt-1 (5s → 10s → 20s → 40s → 60s). var transientRetryBackoff = []time.Duration{ @@ -316,7 +325,7 @@ func (s *Service) retryTransientPrompt(ctx context.Context, taskID, sessionID, e return } - if _, err := s.promptTask(ctx, taskID, sessionID, cp.text, cp.model, cp.planMode, cp.attachments, false, promptTaskOptions{ + if _, err := s.promptTask(ctx, taskID, sessionID, cp.text, cp.model, cp.planMode, cp.attachments, false, launchOriginAutomatic, promptTaskOptions{ onAccepted: cp.onAccepted, }); err != nil { if ctx.Err() != nil { diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow.go b/apps/backend/internal/orchestrator/event_handlers_workflow.go index 4aa13b9bc6..9ce112c9e5 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow.go @@ -598,7 +598,7 @@ func (s *Service) updateTransitionTaskWithCapacity( targetStep *wfmodels.WorkflowStep, ) error { if targetStep == nil { - return s.repo.UpdateTask(ctx, task) + return s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task) } admissionRepo, ok := s.repo.(workflowMoveAdmissionRepository) if !ok { @@ -791,15 +791,7 @@ func (s *Service) workflowMovePendingOptions(task *models.Task) *workflowmove.En // has been dispatched. A missing marker is a no-op. The options are already // captured by the caller (overlaid step or threaded value) before this runs. func (s *Service) clearWorkflowMovePending(ctx context.Context, taskID string) { - task, err := s.repo.GetTask(ctx, taskID) - if err != nil || task == nil || task.Metadata == nil { - return - } - if _, ok := task.Metadata[models.MetaKeyWorkflowMovePending]; !ok { - return - } - delete(task.Metadata, models.MetaKeyWorkflowMovePending) - if err := s.repo.UpdateTask(ctx, task); err != nil { + if _, err := s.repo.RemoveTaskMetadataKey(ctx, taskID, models.MetaKeyWorkflowMovePending); err != nil { s.logger.Warn("failed to clear workflow move pending marker", zap.String("task_id", taskID), zap.Error(err)) } @@ -1676,7 +1668,7 @@ func (s *Service) syncTaskStateForQueuePromotion(ctx context.Context, task *mode return nil } task.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTask(ctx, task); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { return fmt.Errorf("persist promoted task state: %w", err) } s.publishTaskUpdated(ctx, task) @@ -2058,6 +2050,20 @@ func (s *Service) autoStartTaskForLoadedStep(ctx context.Context, task *models.T EntryOptions: moveOptions, WorkflowEntryID: stepTransitionID, }) + if errors.Is(err, ErrCeilingLaunchDeferred) { + // The sweep already persisted a replay record and owns retrying + // this launch; restoring the claim tokens here (as the generic + // failure path below does) would let a second auto-start attempt + // race that replay into a double launch, and marking the task's + // auto-start-failed marker would mislabel a queued launch as a + // failed one. + s.logger.Debug(eventName+": auto-start deferred by session ceiling; will replay once capacity frees up", + zap.String("task_id", task.ID)) + if restoreAutoStartOnCreate { + s.completeAutoStartOnCreate(asyncCtx, task.ID, eventName) + } + return + } if err != nil { s.logger.Error(eventName+": failed to auto-start task", zap.String("task_id", task.ID), @@ -2289,7 +2295,7 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve } go func() { launchCtx := context.WithoutCancel(ctx) - metadataClaimed, claimOK := s.claimDeferredLaunch(launchCtx, task.ID, eventName) + wip, claimOK := s.claimDeferredLaunch(launchCtx, task.ID, eventName) if !claimOK { if createClaimed { s.restoreAutoStartOnCreate(launchCtx, task.ID, eventName) @@ -2307,7 +2313,7 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve launchErr = errors.New("deferred launch returned an unsuccessful response") } s.logger.Error(eventName+": failed to launch deferred task", zap.String("task_id", task.ID), zap.Error(launchErr)) - s.restoreDeferredLaunch(launchCtx, task.ID, raw, eventName, metadataClaimed) + s.restoreDeferredLaunch(launchCtx, task.ID, wip, eventName) if restoreQueuePromotion { s.restoreTaskLifecycleToken(launchCtx, task.ID, models.MetaKeyQueuePromotionPending, queuePromotionLifecycleToken(task), eventName) } @@ -2322,46 +2328,48 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve if intent.RecordRecentUse { s.recordSuccessfulDeferredTaskProfileAsync(launchCtx, intent.UserID, launchResp.AgentProfileID) } - delete(task.Metadata, models.MetaKeyDeferredLaunch) - if metadataClaimed { - task.UpdatedAt = time.Now().UTC() - s.publishTaskUpdated(launchCtx, task) - return - } - task.UpdatedAt = time.Now().UTC() - if updateErr := s.repo.UpdateTask(launchCtx, task); updateErr == nil { - s.publishTaskUpdated(launchCtx, task) + // Publish the repository-backed task, not the pre-claim snapshot this + // closure captured: a concurrent ceiling deferral can write a fresh + // deferred_launch record while LaunchSession runs, and deleting the + // key from the stale in-memory copy would both discard that + // concurrent write and publish it as gone. + current, err := s.repo.GetTask(launchCtx, task.ID) + if err != nil { + s.logger.Warn(eventName+": failed to reload task after deferred launch; publishing pre-launch snapshot", + zap.String("task_id", task.ID), zap.Error(err)) + current = task + delete(current.Metadata, models.MetaKeyDeferredLaunch) + current.UpdatedAt = time.Now().UTC() } + s.publishTaskUpdated(launchCtx, current) }() return true } -func (s *Service) claimDeferredLaunch(ctx context.Context, taskID, eventName string) (bool, bool) { - remover, ok := s.repo.(taskMetadataKeyRemover) - if !ok { - return false, true +// claimDeferredLaunch reserves the launch-intent keys of a task's shared +// deferred_launch record for the gate that is about to fire (WIP promotion or +// dependency resolution). It takes only the launch-intent keys and leaves any +// concurrently-written ceiling_* keys in place (AC-59a) — the same sub-key +// take/restore protocol claimDeferredLaunchForStart uses for the direct-start +// path, so the two consumers of the shared record can never clobber each +// other's half of it. +func (s *Service) claimDeferredLaunch(ctx context.Context, taskID, eventName string) (map[string]interface{}, bool) { + wip, claimed, err := s.repo.TakeTaskDeferredLaunchWIPKeys(ctx, taskID) + if err != nil { + s.logger.Warn(eventName+": failed to claim deferred launch", zap.String("task_id", taskID), zap.Error(err)) + return nil, false } - claimed, err := remover.RemoveTaskMetadataKey(ctx, taskID, models.MetaKeyDeferredLaunch) - if err != nil || !claimed { - if err != nil { - s.logger.Warn(eventName+": failed to claim deferred launch", zap.String("task_id", taskID), zap.Error(err)) - } - return false, false - } - return true, true + return wip, claimed } -func (s *Service) restoreDeferredLaunch(ctx context.Context, taskID string, raw interface{}, eventName string, claimed bool) { - if !claimed { +// restoreDeferredLaunch puts a failed launch's claimed keys back, merging +// into whatever the record holds now rather than replacing it — so a ceiling +// record written while the launch was in flight survives the restore. +func (s *Service) restoreDeferredLaunch(ctx context.Context, taskID string, wip map[string]interface{}, eventName string) { + if len(wip) == 0 { return } - setter, ok := s.repo.(interface { - SetTaskMetadataKey(context.Context, string, string, interface{}) error - }) - if !ok { - return - } - if err := setter.SetTaskMetadataKey(ctx, taskID, models.MetaKeyDeferredLaunch, raw); err != nil { + if err := s.repo.RestoreTaskDeferredLaunchWIPKeys(ctx, taskID, wip); err != nil { s.logger.Warn(eventName+": failed to restore deferred launch intent", zap.String("task_id", taskID), zap.Error(err)) } } @@ -3565,6 +3573,8 @@ func (s *Service) prepareWorkflowReplacementSession( zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(terminalErr)) + } else { + s.releaseCeilingReservation(sessionID) } } return nil, resolutionErr @@ -3635,6 +3645,8 @@ func (s *Service) rollbackNewWorkflowProfileSwitch( ctx, destination.ID, models.TaskSessionStateFailed, cleanupErr.Error(), ); terminalErr != nil { cleanupErr = errors.Join(cleanupErr, fmt.Errorf("terminalize failed workflow destination: %w", terminalErr)) + } else { + s.releaseCeilingReservation(destination.ID) } return errors.Join(cause, cleanupErr) } @@ -3645,6 +3657,8 @@ func (s *Service) rollbackNewWorkflowProfileSwitch( ctx, destination.ID, models.TaskSessionStateFailed, deleteErr.Error(), ); terminalErr != nil { deleteErr = errors.Join(deleteErr, fmt.Errorf("terminalize failed workflow destination: %w", terminalErr)) + } else { + s.releaseCeilingReservation(destination.ID) } return errors.Join(cause, deleteErr) } @@ -3693,6 +3707,7 @@ func (s *Service) retainFailedWorkflowDestination( ); err != nil { return fmt.Errorf("terminalize retained workflow destination: %w", err) } + s.releaseCeilingReservation(destination.ID) s.logger.Warn("retained failed workflow destination for durable recovery", zap.String("task_id", taskID), zap.String("session_id", destination.ID), @@ -4898,7 +4913,7 @@ func (s *Service) syncTaskStateForPendingMove(ctx context.Context, taskID, fromS oldState := task.State task.State = v1.TaskStateTODO task.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTask(ctx, task); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { s.taskRuntimeStateMu.Unlock() s.logger.Warn("pending move state sync: failed to reopen completed task", zap.String("task_id", taskID), @@ -5628,7 +5643,7 @@ func (s *Service) autoStartStepPrompt( const maxRetryAttempts = 5 for attempt := 1; attempt <= maxRetryAttempts; attempt++ { - _, err := s.promptTask(ctx, taskID, sessionID, dispatchPrompt, "", planMode, attachments, false, promptTaskOptions{ + _, err := s.promptTask(ctx, taskID, sessionID, dispatchPrompt, "", planMode, attachments, false, launchOriginAutomatic, promptTaskOptions{ requireNonterminalSession: true, // dispatchPrompt is already fully composed for this step entry // (handoff included, see the ErrExecutionNotFound branch below): @@ -5647,6 +5662,18 @@ func (s *Service) autoStartStepPrompt( requeueTaken() return err } + // AC-47c2: the two seam-3 dispositions carry opposite restoration + // obligations. A deferred refusal already holds the prompt and its + // AC-47c(c) strings in the written record, so requeuing here would + // redeliver the same content twice; an undeferred one (AC-47c1) is + // the caller's own to restore, exactly like the terminalized branch + // above. + if refusal, ok := isSeam3Refusal(err); ok { + if !refusal.deferred { + requeueTaken() + } + return err + } // ErrExecutionNotFound means ResumeSession landed on an execution that // the lifecycle manager no longer has (e.g. the post-resume agent @@ -5997,7 +6024,7 @@ func (s *Service) queueAutoStartPrompt( if handoffText != "" { meta[messagequeue.MetadataStepHandoff] = handoffText } - _, err := s.messageQueue.QueueMessageWithMetadata( + queued, err := s.messageQueue.QueueMessageWithMetadata( ctx, sessionID, taskID, @@ -6012,7 +6039,7 @@ func (s *Service) queueAutoStartPrompt( return fmt.Errorf("failed to queue workflow auto-start prompt: %w", err) } s.publishQueueStatusEvent(ctx, sessionID) - s.scheduleAutoResumeForWorkflowQueue(ctx, sessionID) + s.scheduleAutoResumeForWorkflowQueue(ctx, sessionID, queued.ID) return nil } @@ -6027,14 +6054,14 @@ func (s *Service) queueAutoStartPrompt( // crashed just before the on_enter transition). If the agent is alive when // the queue is written but dies later, the queue is drained by the next // handleAgentBootReady (manual or automatic resume). -func (s *Service) scheduleAutoResumeForWorkflowQueue(ctx context.Context, sessionID string) { +func (s *Service) scheduleAutoResumeForWorkflowQueue(ctx context.Context, sessionID, queuedMessageID string) { if s.executor == nil { return } if exec, ok := s.executor.GetExecutionBySession(sessionID); ok && exec != nil { return } - go s.tryEnsureExecution(context.WithoutCancel(ctx), sessionID) + go s.tryEnsureExecution(context.WithoutCancel(ctx), sessionID, seam3CallShapeQueueDrain, launchOriginAutomatic, queuedMessageID) } // flipStaleRunningToWaiting flips the session to WAITING_FOR_INPUT when its diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_ceiling_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_ceiling_test.go new file mode 100644 index 0000000000..90d116b462 --- /dev/null +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_ceiling_test.go @@ -0,0 +1,122 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// This file pins AC-59/AC-59a for the second writer of the shared +// deferred_launch record: claimDeferredLaunch/restoreDeferredLaunch, used by +// the gate that fires on WIP promotion or dependency resolution. It mirrors +// deferred_launch_consume_ceiling_test.go, which pins the same contract for +// the direct-start path (claimDeferredLaunchForStart/releaseIfHeld) — the +// two consumers of the record must never clobber each other's half of it. + +// TestClaimDeferredLaunchLeavesACeilingRecordInPlace pins the claim half: the +// gate takes only the launch-intent keys it owns and must not delete a +// ceiling deferral sharing the record. Deleting it is a silently dropped +// launch that the AC-17a sweep can no longer find. +func TestClaimDeferredLaunchLeavesACeilingRecordInPlace(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "gate-claim-mixed", Title: "Mixed", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + wip, claimed := svc.claimDeferredLaunch(ctx, "gate-claim-mixed", "test.event") + if !claimed { + t.Fatal("claim did not succeed; the WIP intent should have been claimable") + } + if _, ok := wip[models.DeferredLaunchStartWhenUnblockedKey]; !ok { + t.Fatalf("claim did not return its own key: %+v", wip) + } + + after := deferredLaunchOf(t, svc, "gate-claim-mixed") + if after == nil { + t.Fatal("the whole deferred_launch record was deleted by the claim") + } + if after[models.CeilingDeferredKey] != true || after[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("ceiling keys did not survive the claim: %+v", after) + } + if _, still := after[models.DeferredLaunchStartWhenUnblockedKey]; still { + t.Fatalf("the claim did not take its own key out of the durable record: %+v", after) + } +} + +// TestRestoreDeferredLaunchMergesRatherThanOverwriting pins the restore half: +// a ceiling record written while the launch was in flight must survive a +// failed launch's restore. Restoring a pre-claim snapshot wholesale would +// erase it — merging back only the claimed keys is what AC-59a requires. +func TestRestoreDeferredLaunchMergesRatherThanOverwriting(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "gate-claim-release", Title: "Release", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + models.DeferredLaunchUserIDKey: "user-7", + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + wip, claimed := svc.claimDeferredLaunch(ctx, "gate-claim-release", "test.event") + if !claimed { + t.Fatal("claim did not succeed") + } + if record := deferredLaunchOf(t, svc, "gate-claim-release"); record != nil { + t.Fatalf("a WIP-only record should have been removed entirely, got %+v", record) + } + + // A ceiling refusal lands while the launch is in flight. + if err := repo.SetTaskMetadataKey(ctx, "gate-claim-release", models.MetaKeyDeferredLaunch, map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }); err != nil { + t.Fatalf("SetTaskMetadataKey: %v", err) + } + + svc.restoreDeferredLaunch(ctx, "gate-claim-release", wip, "test.event") + + after := deferredLaunchOf(t, svc, "gate-claim-release") + if after[models.CeilingDeferredKey] != true || after[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("restore overwrote the ceiling record: %+v", after) + } + if after[models.DeferredLaunchStartWhenUnblockedKey] != true || after[models.DeferredLaunchUserIDKey] != "user-7" { + t.Fatalf("restore did not put the claimed keys back: %+v", after) + } +} + +// TestClaimDeferredLaunchDoesNotClaimACeilingOnlyRecord keeps the two +// meanings separate in the other direction: a ceiling deferral is replayed by +// its own sweep, so the gate has nothing to claim and must leave it +// untouched. +func TestClaimDeferredLaunchDoesNotClaimACeilingOnlyRecord(t *testing.T) { + svc, repo := newServiceWithRealRepo(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "gate-claim-ceiling-only", Title: "CeilingOnly", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.CeilingDeferredKey: true, + }}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + wip, claimed := svc.claimDeferredLaunch(ctx, "gate-claim-ceiling-only", "test.event") + if claimed { + t.Fatalf("a ceiling-only record must not be claimed by the gate path, got wip=%+v", wip) + } + if after := deferredLaunchOf(t, svc, "gate-claim-ceiling-only"); after[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling record disturbed: %+v", after) + } +} diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow_deferred_launch_race_test.go b/apps/backend/internal/orchestrator/event_handlers_workflow_deferred_launch_race_test.go new file mode 100644 index 0000000000..b4b6ee31ca --- /dev/null +++ b/apps/backend/internal/orchestrator/event_handlers_workflow_deferred_launch_race_test.go @@ -0,0 +1,113 @@ +package orchestrator + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// deferredLaunchPublishRaceRepo wraps the orchestrator's repo handle and, on +// the first GetTask call for the watched task, writes a fresh +// deferred_launch record through the repository's own CAS primitive before +// returning — modeling the session ceiling admission controller deferring a +// concurrent launch in the window between launchDeferredTask's detached +// goroutine finishing LaunchSession and reloading the task to publish. +type deferredLaunchPublishRaceRepo struct { + sessionExecutorStore + t *testing.T + watchTaskID string + mu sync.Mutex + fired bool +} + +func (r *deferredLaunchPublishRaceRepo) GetTask(ctx context.Context, id string) (*models.Task, error) { + task, err := r.sessionExecutorStore.GetTask(ctx, id) + r.mu.Lock() + shouldFire := id == r.watchTaskID && !r.fired + if shouldFire { + r.fired = true + } + r.mu.Unlock() + if shouldFire { + _, prior, priorErr := r.GetTaskDeferredLaunch(ctx, id) + require.NoError(r.t, priorErr) + stored, lostCompare, setErr := r.SetTaskDeferredLaunchIfUnchanged(ctx, id, prior, + map[string]interface{}{"prompt": "concurrent", "ceiling_deferred": true}) + require.NoError(r.t, setErr) + require.False(r.t, lostCompare) + require.True(r.t, stored) + } + return task, err +} + +// capturingTaskEvents records every PublishTaskUpdated call's full task +// argument. recordingTaskEventPublisher (office_task_status_test.go) only +// keeps the task ID, which cannot pin an assertion on deferred_launch's +// published contents. +type capturingTaskEvents struct { + mu sync.Mutex + tasks []*models.Task +} + +func (c *capturingTaskEvents) PublishTaskUpdated(_ context.Context, task *models.Task, _ ...string) { + c.mu.Lock() + defer c.mu.Unlock() + c.tasks = append(c.tasks, task) +} + +func (c *capturingTaskEvents) PublishTaskStateChanged(context.Context, *models.Task, v1.TaskState) {} + +func (c *capturingTaskEvents) PublishTaskActivityIfChanged(context.Context, string) {} + +func (c *capturingTaskEvents) last() *models.Task { + c.mu.Lock() + defer c.mu.Unlock() + if len(c.tasks) == 0 { + return nil + } + return c.tasks[len(c.tasks)-1] +} + +// TestLaunchDeferredTaskPublishesTheReloadedTaskAfterAConcurrentCeilingWrite +// pins the fix at launchDeferredTask's post-launch publish: a concurrent +// ceiling deferral writing a fresh deferred_launch record while +// LaunchSession runs must survive the publish, not be discarded by the +// pre-claim in-memory snapshot the closure captured. +func TestLaunchDeferredTaskPublishesTheReloadedTaskAfterAConcurrentCeilingWrite(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedChainStepTask(t, repo, deferredChainTaskID) + task, err := repo.GetTask(ctx, deferredChainTaskID) + require.NoError(t, err) + + svc := newDeferredLaunchTestService(t, repo, newLaunchCounter()) + raceRepo := &deferredLaunchPublishRaceRepo{sessionExecutorStore: svc.repo, t: t, watchTaskID: deferredChainTaskID} + svc.repo = raceRepo + events := &capturingTaskEvents{} + svc.SetTaskEventPublisher(events) + + require.True(t, svc.launchDeferredTask(ctx, task, "test.deferred_launch", false)) + awaitLaunchedSession(t, repo, deferredChainTaskID) + + deadline := time.Now().Add(2 * time.Second) + var published *models.Task + for time.Now().Before(deadline) { + if published = events.last(); published != nil { + break + } + time.Sleep(5 * time.Millisecond) + } + require.NotNil(t, published, "launchDeferredTask must publish the reloaded task after a successful launch") + require.True(t, raceRepo.fired, "the concurrent ceiling race must have actually been injected") + + deferred, ok := published.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "published task's deferred_launch missing or wrong shape: %#v", published.Metadata[models.MetaKeyDeferredLaunch]) + require.Equal(t, true, deferred["ceiling_deferred"], + "launchDeferredTask must publish a task reflecting the concurrent ceiling CAS write, not the stale pre-launch snapshot") +} diff --git a/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit.go b/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit.go new file mode 100644 index 0000000000..02ef41deaa --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit.go @@ -0,0 +1,48 @@ +package executor + +import ( + "context" + + "go.uber.org/zap" +) + +// CeilingBackingChecker answers, for a session_id, whether the orchestrator's +// session-ceiling admission controller currently backs it with an in-flight +// reservation or a counted AC-1 population row (AC-41). It is the single +// permitted route by which this package observes the controller: the +// reservation set lives in package orchestrator, which already imports this +// package, so the dependency runs the other way and this interface is +// declared here rather than the check being read directly. +type CeilingBackingChecker interface { + IsSessionCeilingBacked(ctx context.Context, sessionID string) (bool, error) +} + +// auditCeilingBypass is AC-4b's bypass detector for the three entry points +// that start an agent process: runAgentProcessAsync (the sole ERROR site, +// AC-4b1), LaunchPreparedSession and ResumeSessionWithOptions (DEBUG only). +// AC-41a: with no checker injected — every existing construction site, +// including all of this package's tests — this is a complete no-op: no +// detection, no log, no error, no behaviour change. AC-41b: a lookup failure +// (the AC-1 row half is a repository read and can fail) emits no ERROR for +// that launch, logs the failure at DEBUG, and never blocks, retries or delays +// the launch either way — the audit only ever observes. +func (e *Executor) auditCeilingBypass(ctx context.Context, entryPoint, sessionID string, emitError bool, extra ...zap.Field) { + if e.ceilingBackingChecker == nil || sessionID == "" { + return + } + backed, err := e.ceilingBackingChecker.IsSessionCeilingBacked(ctx, sessionID) + if err != nil { + e.logger.Debug("could not determine whether a launch is backed by the session ceiling", + zap.String("session_id", sessionID), zap.String("entry_point", entryPoint), zap.Error(err)) + return + } + if backed { + return + } + fields := append([]zap.Field{zap.String("session_id", sessionID), zap.String("entry_point", entryPoint)}, extra...) + if emitError { + e.logger.Error("agent process launch reached an entry point with no session-ceiling reservation or counted row", fields...) + return + } + e.logger.Debug("agent process launch reached an entry point with no session-ceiling reservation or counted row", fields...) +} diff --git a/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit_test.go b/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit_test.go new file mode 100644 index 0000000000..3bec09571f --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/ceiling_bypass_audit_test.go @@ -0,0 +1,163 @@ +package executor + +import ( + "context" + "errors" + "testing" + + "go.uber.org/zap/zapcore" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// fakeCeilingBackingChecker is a call-recording, scriptable double for +// CeilingBackingChecker. +type fakeCeilingBackingChecker struct { + calls []string + backed bool + err error +} + +func (f *fakeCeilingBackingChecker) IsSessionCeilingBacked(_ context.Context, sessionID string) (bool, error) { + f.calls = append(f.calls, sessionID) + return f.backed, f.err +} + +// TestAuditCeilingBypass_NilCheckerIsANoOp pins AC-41a: with no checker +// injected, every construction site behaves exactly as before this card — +// no detection, no log, no error. +func TestAuditCeilingBypass_NilCheckerIsANoOp(t *testing.T) { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + exec.auditCeilingBypass(context.Background(), "someEntryPoint", "session-x", true) + if logs.Len() != 0 { + t.Fatalf("expected no log entries with no checker injected, got %d: %v", logs.Len(), logs.All()) + } +} + +// TestAuditCeilingBypass_EmptySessionIDIsANoOp pins that an entry point with +// no session id yet never reaches the checker at all. +func TestAuditCeilingBypass_EmptySessionIDIsANoOp(t *testing.T) { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + checker := &fakeCeilingBackingChecker{backed: false} + exec.SetCeilingBackingChecker(checker) + + exec.auditCeilingBypass(context.Background(), "someEntryPoint", "", true) + + if len(checker.calls) != 0 { + t.Fatalf("checker must not be consulted for an empty session id, got calls=%v", checker.calls) + } + if logs.Len() != 0 { + t.Fatalf("expected no log entries, got %d: %v", logs.Len(), logs.All()) + } +} + +// TestAuditCeilingBypass_BackedSessionLogsNothing covers the ordinary case: +// a launch reaching an instrumented entry point for a session the ceiling +// already backs is silent, whether or not the entry point is the ERROR site. +func TestAuditCeilingBypass_BackedSessionLogsNothing(t *testing.T) { + for _, emitError := range []bool{true, false} { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + exec.SetCeilingBackingChecker(&fakeCeilingBackingChecker{backed: true}) + + exec.auditCeilingBypass(context.Background(), "someEntryPoint", "session-x", emitError) + + if logs.Len() != 0 { + t.Fatalf("emitError=%v: expected no log entries for a backed session, got %d: %v", emitError, logs.Len(), logs.All()) + } + } +} + +// TestAuditCeilingBypass_UnbackedEmitsErrorAtTheDesignatedSite pins AC-4b1: +// the sole ERROR site (runAgentProcessAsync, emitError=true) logs at ERROR, +// carrying the session id, the entry point name, and any extra fields (the +// agent_execution_id runAgentProcessAsync passes). +func TestAuditCeilingBypass_UnbackedEmitsErrorAtTheDesignatedSite(t *testing.T) { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + exec.SetCeilingBackingChecker(&fakeCeilingBackingChecker{backed: false}) + + exec.auditCeilingBypass(context.Background(), "runAgentProcessAsync", "session-x", true) + + errs := logs.FilterLevelExact(zapcore.ErrorLevel).All() + if len(errs) != 1 { + t.Fatalf("error entries = %d, want 1; all=%v", len(errs), logs.All()) + } + fields := errs[0].ContextMap() + if fields["session_id"] != "session-x" || fields["entry_point"] != "runAgentProcessAsync" { + t.Fatalf("unexpected error fields: %+v", fields) + } +} + +// TestAuditCeilingBypass_UnbackedLogsDebugAtTheOtherTwoSites pins AC-4b1: the +// other two entry points (LaunchPreparedSession, ResumeSessionWithOptions, +// emitError=false) log the same observation at DEBUG rather than ERROR, so +// the bypass is diagnosable without duplicating the reportable ERROR. +func TestAuditCeilingBypass_UnbackedLogsDebugAtTheOtherTwoSites(t *testing.T) { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + exec.SetCeilingBackingChecker(&fakeCeilingBackingChecker{backed: false}) + + exec.auditCeilingBypass(context.Background(), "LaunchPreparedSession", "session-x", false) + + if errs := logs.FilterLevelExact(zapcore.ErrorLevel).Len(); errs != 0 { + t.Fatalf("error entries = %d, want 0", errs) + } + debugs := logs.FilterLevelExact(zapcore.DebugLevel).All() + if len(debugs) != 1 { + t.Fatalf("debug entries = %d, want 1; all=%v", len(debugs), logs.All()) + } +} + +// TestAuditCeilingBypass_LookupFailureNeverEmitsErrorEvenAtTheErrorSite pins +// AC-41b: a failed lookup (the AC-1 row half is a repository read that can +// fail) logs the failure at DEBUG and emits no AC-4b ERROR, even when called +// from the designated ERROR site — a missed detection is bounded by AC-8's +// re-derivation; a false one is not bounded by anything. +func TestAuditCeilingBypass_LookupFailureNeverEmitsErrorEvenAtTheErrorSite(t *testing.T) { + exec, logs := newObservedExecutor(t, zapcore.DebugLevel) + exec.SetCeilingBackingChecker(&fakeCeilingBackingChecker{err: errors.New("lookup failed")}) + + exec.auditCeilingBypass(context.Background(), "runAgentProcessAsync", "session-x", true) + + if errs := logs.FilterLevelExact(zapcore.ErrorLevel).Len(); errs != 0 { + t.Fatalf("error entries = %d, want 0 on a lookup failure", errs) + } + debugs := logs.FilterLevelExact(zapcore.DebugLevel).All() + if len(debugs) != 1 { + t.Fatalf("debug entries = %d, want 1; all=%v", len(debugs), logs.All()) + } + if debugs[0].ContextMap()["session_id"] != "session-x" { + t.Fatalf("debug entry must carry the session id: %+v", debugs[0].ContextMap()) + } +} + +// TestLaunchPreparedSession_AuditsOnlyWhenStartingAnAgent pins AC-4c: a +// LaunchPreparedSession call with StartAgent false prepares a workspace and +// starts no process, so it must not be instrumented at all; StartAgent true +// must consult the checker. +func TestLaunchPreparedSession_AuditsOnlyWhenStartingAnAgent(t *testing.T) { + for _, tt := range []struct { + name string + startAgent bool + wantCalls int + }{ + {"StartAgent true audits", true, 1}, + {"StartAgent false is not instrumented", false, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-x"] = &models.TaskSession{ID: "session-x", TaskID: "other-task"} + exec := newTestExecutor(t, &mockAgentManager{}, repo) + checker := &fakeCeilingBackingChecker{backed: true} + exec.SetCeilingBackingChecker(checker) + + // The session belongs to a different task, so LaunchPreparedSession + // returns right after the audit call fires, before any process + // dispatch machinery runs. + _, _ = exec.LaunchPreparedSession(context.Background(), &v1.Task{ID: "task-x"}, "session-x", LaunchOptions{StartAgent: tt.startAgent}) + + if len(checker.calls) != tt.wantCalls { + t.Fatalf("checker calls = %d, want %d (calls=%v)", len(checker.calls), tt.wantCalls, checker.calls) + } + }) + } +} diff --git a/apps/backend/internal/orchestrator/executor/ceiling_release_callback_test.go b/apps/backend/internal/orchestrator/executor/ceiling_release_callback_test.go new file mode 100644 index 0000000000..1eb7ee7f42 --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/ceiling_release_callback_test.go @@ -0,0 +1,59 @@ +package executor + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// TestMarkCompletedBySession_ReleasesTheCeilingReservation pins AC-51a: this +// write reaches the repository directly, bypassing every onSessionStateChange +// funnel the orchestrator package instruments itself, so the executor must +// invoke its own dependency-inverted release callback after a successful write. +func TestMarkCompletedBySession_ReleasesTheCeilingReservation(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-1"] = &models.TaskSession{ID: "session-1", State: models.TaskSessionStateRunning} + exec := newTestExecutor(t, nil, repo) + + var released []string + exec.SetOnCeilingReservationRelease(func(sessionID string) { released = append(released, sessionID) }) + + exec.MarkCompletedBySession(context.Background(), "session-1", v1.TaskSessionStateCompleted) + + require.Equal(t, []string{"session-1"}, released) +} + +// TestMarkCompletedBySession_NoCallbackIsANoOp pins the nil-safety every other +// optional Executor callback field already follows. +func TestMarkCompletedBySession_NoCallbackIsANoOp(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-2"] = &models.TaskSession{ID: "session-2", State: models.TaskSessionStateRunning} + exec := newTestExecutor(t, nil, repo) + + require.NotPanics(t, func() { + exec.MarkCompletedBySession(context.Background(), "session-2", v1.TaskSessionStateCompleted) + }) +} + +// TestMarkCompletedBySession_WriteFailureDoesNotRelease pins that the release +// only fires after a successful write: an update failure must not free a +// reservation for a launch whose session state was never actually settled. +func TestMarkCompletedBySession_WriteFailureDoesNotRelease(t *testing.T) { + repo := newMockRepository() + repo.updateTaskSessionStateFunc = func(context.Context, string, models.TaskSessionState, string) error { + return errors.New("update failed") + } + exec := newTestExecutor(t, nil, repo) + + var released []string + exec.SetOnCeilingReservationRelease(func(sessionID string) { released = append(released, sessionID) }) + + exec.MarkCompletedBySession(context.Background(), "session-3", v1.TaskSessionStateCompleted) + + require.Empty(t, released) +} diff --git a/apps/backend/internal/orchestrator/executor/executor.go b/apps/backend/internal/orchestrator/executor/executor.go index 0e1e2a33cf..6dd24691b7 100644 --- a/apps/backend/internal/orchestrator/executor/executor.go +++ b/apps/backend/internal/orchestrator/executor/executor.go @@ -888,6 +888,14 @@ type AgentProcessStartFailedFunc func(ctx context.Context, taskID, sessionID, ag // creating repository-scoped user-facing status messages tied to launch errors. type LaunchFailedFunc func(ctx context.Context, taskID, sessionID, repositoryID string, err error) +// CeilingReservationReleaseFunc releases the orchestrator's session-ceiling +// reservation for a session this package just moved out of the counted +// population through a write that bypasses onSessionStateChange / +// onSessionStateTransition (MarkCompletedBySession, the resume-failure +// rollback). Releasing a session that held no reservation is a defined +// no-op, so this can be called unconditionally on a successful write. +type CeilingReservationReleaseFunc func(sessionID string) + // LaunchFailureReviewEligibilityFunc reports whether a failed launch can offer // the mark-review-done recovery action. The resolver owns workflow and PR // lookups; an error omits the action without blocking failure persistence. @@ -1014,6 +1022,14 @@ type Executor struct { // Callback for session launch failures (pre-start). Allows orchestrator // to emit user-friendly guidance for known failure patterns. onLaunchFailed LaunchFailedFunc + + // Callback releasing the session-ceiling reservation for writes this + // package makes directly against the repository, bypassing + // onSessionStateChange / onSessionStateTransition (AC-51a). + onCeilingReservationRelease CeilingReservationReleaseFunc + // Optional observation-only bypass detector for the three entry points + // that start an agent process (AC-41/AC-41a). + ceilingBackingChecker CeilingBackingChecker // Optional resolver for the mark-review-done recovery action. launchFailureReviewEligibility LaunchFailureReviewEligibilityFunc // Optional compatibility gate for legacy adapters. @@ -1369,6 +1385,23 @@ func (e *Executor) SetOnAgentProcessStartFailed(fn AgentProcessStartFailedFunc) e.onAgentProcessStartFailed = fn } +// SetOnCeilingReservationRelease sets the callback used by writes this +// package makes directly against the repository, bypassing +// onSessionStateChange / onSessionStateTransition, to release the +// session-ceiling reservation for a session that just left the counted +// population (AC-51a). +func (e *Executor) SetOnCeilingReservationRelease(fn CeilingReservationReleaseFunc) { + e.onCeilingReservationRelease = fn +} + +// SetCeilingBackingChecker wires the AC-41 dependency-inverted bypass +// detector. Leaving it unset (every existing construction site) is AC-41a's +// contract: the three instrumented entry points behave exactly as they do +// without this card at all. +func (e *Executor) SetCeilingBackingChecker(checker CeilingBackingChecker) { + e.ceilingBackingChecker = checker +} + // SetOnPrimarySessionSet sets a callback for when the first session for a task // is marked primary. This publishes a task.updated event so the frontend // receives primary_session_id. diff --git a/apps/backend/internal/orchestrator/executor/executor_execute.go b/apps/backend/internal/orchestrator/executor/executor_execute.go index 446b92423a..efd089c9cd 100644 --- a/apps/backend/internal/orchestrator/executor/executor_execute.go +++ b/apps/backend/internal/orchestrator/executor/executor_execute.go @@ -151,6 +151,7 @@ func executorNeedsResolvedCredentials(executorType string) bool { // WithCancellableResumeContext retain cancellation so an explicit stop can // interrupt a startup that is still waiting for ACP readiness. func (e *Executor) runAgentProcessAsync(ctx context.Context, taskID, sessionID, agentExecutionID string, onSuccess func(context.Context), escalateTaskOnFailure, fromResume bool) { + e.auditCeilingBypass(ctx, "runAgentProcessAsync", sessionID, true, zap.String("agent_execution_id", agentExecutionID)) go func() { startParent := context.WithoutCancel(ctx) updateCtx := startParent @@ -1402,6 +1403,11 @@ func (e *Executor) LaunchPreparedSession(ctx context.Context, task *v1.Task, ses executorID := opts.ExecutorID prompt := opts.Prompt startAgent := opts.StartAgent + if startAgent { + // AC-4c: StartAgent false prepares a workspace and starts no process, + // so it is out of AC-4b's scope and must not be instrumented. + e.auditCeilingBypass(ctx, "LaunchPreparedSession", sessionID, false) + } // Serialise concurrent launches for the same session. Two callers reach // this path on every task: PrepareTaskSession spawns a background launch // (workspace only) the moment a session is created, and StartCreatedSession diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index 5c33271380..55bdfbc7eb 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -851,6 +851,9 @@ func (m *mockRepository) UpdateTask(ctx context.Context, task *models.Task) erro func (m *mockRepository) UpdateTaskWithExplicitPosition(ctx context.Context, task *models.Task) error { return nil } +func (m *mockRepository) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + return nil +} func (m *mockRepository) DeleteTask(ctx context.Context, id string) error { return nil } func (m *mockRepository) ListTasks(ctx context.Context, workflowID string) ([]*models.Task, error) { return nil, nil diff --git a/apps/backend/internal/orchestrator/executor/executor_resume.go b/apps/backend/internal/orchestrator/executor/executor_resume.go index 176eb91ee1..619fd98438 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume.go @@ -748,6 +748,14 @@ type ResumeOptions struct { // or a pinned follow-up dispatch. It does not change the global terminal // session predicate or permit implicit resume paths. AllowCompletedSessionResume bool + // Origin carries the session ceiling's explicit automatic/manual launch + // classification ("automatic" or "manual") from the caller into + // ResumeTaskSessionWithOptions's admission gate. A plain string rather + // than the orchestrator package's own type, since this package must not + // import orchestrator. Left empty, the gate classifies the resume as + // automatic and logs the omission — it is never silently treated as a + // manual override. + Origin string } type cancellableResumeContextKey struct{} @@ -808,6 +816,9 @@ func (e *Executor) ResumeSessionWithOptions( startAgent bool, options ResumeOptions, ) (*TaskExecution, error) { + if session != nil { + e.auditCeilingBypass(ctx, "ResumeSessionWithOptions", session.ID, false) + } return e.resumeSession(ctx, session, startAgent, options) } @@ -1073,6 +1084,14 @@ func (e *Executor) rollbackResumeStateAfterFailure( resumeErr error, credentialSnapshot *resumeCredentialSnapshotBackup, ) { + // This rollback always leaves STARTING (successfully, or a no-op if the + // session had already left it), so the ceiling reservation is released + // unconditionally (AC-51a). Safe even when onSessionStateTransition + // already routes the write through the orchestrator's own funnel, since + // releasing a session that holds no reservation is a defined no-op. + if e.onCeilingReservationRelease != nil { + defer e.onCeilingReservationRelease(sessionID) + } e.restoreResumeCredentialSnapshotIfStarting(ctx, sessionID, credentialSnapshot) if e.onSessionStateTransition != nil { current, err := e.repo.GetTaskSession(ctx, sessionID) diff --git a/apps/backend/internal/orchestrator/executor/executor_state.go b/apps/backend/internal/orchestrator/executor/executor_state.go index be3889a9a3..66ce3ae387 100644 --- a/apps/backend/internal/orchestrator/executor/executor_state.go +++ b/apps/backend/internal/orchestrator/executor/executor_state.go @@ -86,6 +86,13 @@ func (e *Executor) MarkCompletedBySession(ctx context.Context, sessionID string, e.logger.Error("failed to update agent session status in database", zap.String("session_id", sessionID), zap.Error(err)) + return + } + // This write reaches the repository directly rather than through + // onSessionStateChange, so it releases the session-ceiling reservation + // itself (AC-51a). + if e.onCeilingReservationRelease != nil { + e.onCeilingReservationRelease(sessionID) } } diff --git a/apps/backend/internal/orchestrator/foreground_claim_test.go b/apps/backend/internal/orchestrator/foreground_claim_test.go index 20024842b7..bc4c66c6b3 100644 --- a/apps/backend/internal/orchestrator/foreground_claim_test.go +++ b/apps/backend/internal/orchestrator/foreground_claim_test.go @@ -207,7 +207,7 @@ func TestPromptTask_SupersededQueuedDispatchReleasesForegroundClaim(t *testing.T svc.markForegroundIdle(sessionID) _, err := svc.promptTask( - context.Background(), taskID, sessionID, "queued prompt", "", false, nil, false, + context.Background(), taskID, sessionID, "queued prompt", "", false, nil, false, launchOriginAutomatic, promptTaskOptions{claimEntryID: "stale-entry"}, ) if !errors.Is(err, ErrAgentPromptInProgress) { diff --git a/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go b/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go index 6af49c078e..e01081eba9 100644 --- a/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go +++ b/apps/backend/internal/orchestrator/prompt_dispatch_identity_test.go @@ -27,7 +27,7 @@ func TestPromptTaskExpectedIdentityRejectsReplacementAfterClaim(t *testing.T) { } _, err = svc.promptTask( - ctx, taskID, sessionID, "queued prompt", "", false, nil, true, + ctx, taskID, sessionID, "queued prompt", "", false, nil, true, launchOriginAutomatic, promptTaskOptions{ expectedSessionIdentity: &identity, afterClaim: func() error { diff --git a/apps/backend/internal/orchestrator/prompt_idle_session_test.go b/apps/backend/internal/orchestrator/prompt_idle_session_test.go index 1fa80b14dc..4e208eeb9b 100644 --- a/apps/backend/internal/orchestrator/prompt_idle_session_test.go +++ b/apps/backend/internal/orchestrator/prompt_idle_session_test.go @@ -228,7 +228,7 @@ func TestEnsureSessionRunning_IdleSessionTriggersResume(t *testing.T) { t.Fatalf("reload: %v", err) } - if err := svc.ensureSessionRunning(ctx, "session1", session); err != nil { + if err := svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual); err != nil { t.Fatalf("ensureSessionRunning failed for IDLE session: %v", err) } select { @@ -311,7 +311,7 @@ func TestEnsureSessionRunning_WaitsForPromptReadyAfterResume(t *testing.T) { done := make(chan error, 1) go func() { - done <- svc.ensureSessionRunning(ctx, "session1", session) + done <- svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) }() select { @@ -425,7 +425,7 @@ func TestEnsureSessionRunning_SurvivesCallerContextCancellationDuringResume(t *t done := make(chan error, 1) go func() { - done <- svc.ensureSessionRunning(ctx, "session1", session) + done <- svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) }() // Simulate the caller's request context expiring mid-resume (e.g. a @@ -484,7 +484,7 @@ func TestEnsureSessionRunning_FailedStateWriteSurvivesCancelledCallerCtx(t *test } seedExecutorRunning(t, repo, session.ID, session.TaskID, "exec-idle-1") - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) if err == nil { t.Fatal("expected ensureSessionRunning to return the simulated resume failure") } @@ -552,7 +552,7 @@ func TestEnsureSessionRunning_WaitsForPromptReadyWhenExecutionExists(t *testing. done := make(chan error, 1) go func() { - done <- svc.ensureSessionRunning(ctx, "session1", session) + done <- svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) }() select { @@ -605,7 +605,7 @@ func TestEnsureSessionRunning_CancelledContextDoesNotReapExecution(t *testing.T) svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) agentMgr.mu.Lock() stopCalls := append([]stopAgentCall(nil), agentMgr.stopAgentWithReasonArgs...) agentMgr.mu.Unlock() diff --git a/apps/backend/internal/orchestrator/queue_send_now.go b/apps/backend/internal/orchestrator/queue_send_now.go index 889116c0da..0e6d36be40 100644 --- a/apps/backend/internal/orchestrator/queue_send_now.go +++ b/apps/backend/internal/orchestrator/queue_send_now.go @@ -574,7 +574,7 @@ func (s *Service) promptSendNowClaim(ctx context.Context, claim *messagequeue.Se } _, err := s.promptTask(ctx, claim.Dispatch.TaskID, sessionID, promptContent, claim.Dispatch.Model, - claim.Dispatch.PlanMode, attachments, false, promptTaskOptions{ + claim.Dispatch.PlanMode, attachments, false, launchOriginManual, promptTaskOptions{ claimEntryID: claim.Dispatch.ID, afterClaim: s.sendNowAfterClaim(ctx, claim, attachments, durablePlanComments), beforeDispatch: s.sendNowDeliveryBoundary(ctx, claim, durablePlanComments, &deliveryAttempted), diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index c497280ccf..493b0819e4 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -438,6 +438,14 @@ type sessionExecutorStore interface { // than an idempotent same-owner observation. ClaimTaskTitleSession(ctx context.Context, taskID, sessionID string) (owned bool, newlyClaimed bool, err error) UpdateTask(ctx context.Context, task *models.Task) error + // UpdateTaskPreservingDeferredLaunch is UpdateTask for a caller holding a + // task snapshot old enough to have missed a concurrent write to + // deferred_launch (any read-modify-write over a task fetched earlier in the + // same handler). It performs the same write as UpdateTask, except + // deferred_launch in the payload is replaced by the row's own current + // value at write time, so the stale snapshot can never resurrect or + // clobber a concurrent ceiling CAS write. + UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error // SetTaskMetadataKey / RemoveTaskMetadataKey are concurrent-key-safe JSON // patch helpers on tasks.metadata (implemented by the sqlite/Postgres // repository). Used by startup reconciliation to mark interrupted tasks @@ -448,6 +456,18 @@ type sessionExecutorStore interface { // check and the write cannot be separated by a concurrent archive). SetTaskMetadataKeyIfNotArchived(ctx context.Context, taskID, key string, value interface{}) (bool, error) RemoveTaskMetadataKey(ctx context.Context, taskID, key string) (bool, error) + // TakeTaskDeferredLaunchWIPKeys / RestoreTaskDeferredLaunchWIPKeys claim and + // release the launch-intent half of the shared deferred_launch record + // without touching the keys another writer owns. + TakeTaskDeferredLaunchWIPKeys(ctx context.Context, taskID string) (map[string]interface{}, bool, error) + RestoreTaskDeferredLaunchWIPKeys(ctx context.Context, taskID string, wip map[string]interface{}) error + // GetTaskDeferredLaunch / SetTaskDeferredLaunchIfUnchanged provide the + // row-locked compare-and-set the session ceiling uses to write a + // ceiling_deferred record without losing a concurrent refusal's payload. + // The prior-state token is threaded as interface{}, opaque to this package, + // because it is produced by the repository from the stored row's own bytes. + GetTaskDeferredLaunch(ctx context.Context, taskID string) (map[string]interface{}, interface{}, error) + SetTaskDeferredLaunchIfUnchanged(ctx context.Context, taskID string, prior interface{}, value map[string]interface{}) (stored bool, lostCompare bool, err error) ListChildCompletionRows(ctx context.Context, parentID string) ([]models.ChildCompletionRow, error) // Git snapshots and commits GetLatestGitSnapshot(ctx context.Context, sessionID string) (*models.GitSnapshot, error) @@ -1065,6 +1085,22 @@ type Service struct { // / stopIdleSessionReaper no-op. See idle_session_reaper.go. idleReaper *idleSessionReaper + // sessionCeiling is the instance-wide admission controller for agent + // session launches. Its ceiling is resolved once here, at construction, + // and is constant for the lifetime of the process. + sessionCeiling *sessionCeilingController + + // ceilingSweeper is the single background goroutine that expires stale + // reservations (AC-7) and retries deferred launches (AC-15, AC-17a). + // Nil-safe like idleReaper: callers that don't need it leave it nil and + // startCeilingSweeper / stopCeilingSweeper no-op. See ceiling_sweep.go. + ceilingSweeper *ceilingSweeper + + // ceilingCredentialReminter re-mints short-lived Office runtime + // credentials immediately before a ceiling-deferred "start" replay. Nil + // is the common case; see CeilingLaunchCredentialReminter. + ceilingCredentialReminter CeilingLaunchCredentialReminter + // lifecycleSweepCancel / lifecycleSweepWorkers own the one-shot // background goroutine that runs reconcileTaskLifecycleTokens and // reconcileDependencyLaunchesOnStartup after the watcher and scheduler @@ -1635,6 +1671,8 @@ func NewService( dynamicSuccessorCtx: dynamicSuccessorCtx, dynamicSuccessorCancel: dynamicSuccessorCancel, idleReaper: newIdleSessionReaper(), + ceilingSweeper: newCeilingSweeper(), + sessionCeiling: newSessionCeilingForRepo(repo, svcLogger.Zap()), backgroundProbeConfig: LoadBackgroundProbeConfig(svcLogger), parkedStates: make(map[string]*parkedSessionState), taskParkedStates: make(map[string]*taskParkedState), @@ -1740,6 +1778,8 @@ func NewService( exec.SetOnAgentStartFailed(s.handleAgentStartFailed) exec.SetOnAgentProcessStarted(s.handleAgentProcessStarted) exec.SetOnAgentProcessStartFailed(s.handleAgentProcessStartFailed) + exec.SetOnCeilingReservationRelease(s.releaseCeilingReservation) + exec.SetCeilingBackingChecker(s) if caps, ok := agentManager.(executor.ExecutorTypeCapabilities); ok { exec.SetCapabilities(caps) } @@ -3166,6 +3206,7 @@ func (s *Service) Start(ctx context.Context) error { // background goroutine owns the reclaim tick; Service.Stop joins it // before tearing down repo / agentManager. s.startIdleSessionReaper(ctx) + s.startCeilingSweeper(ctx) s.logger.Info("orchestrator service started successfully") return nil @@ -3216,6 +3257,7 @@ func (s *Service) Stop() error { // Stop components in reverse order var errs []error s.stopIdleSessionReaper() + s.stopCeilingSweeper() s.stopReservedPromptCallbacks() if err := s.scheduler.Stop(); err != nil { diff --git a/apps/backend/internal/orchestrator/session_ceiling.go b/apps/backend/internal/orchestrator/session_ceiling.go new file mode 100644 index 0000000000..01a1e21b0d --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling.go @@ -0,0 +1,466 @@ +package orchestrator + +import ( + "context" + "sync" + "time" + + "github.com/google/uuid" + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/constants" +) + +// reservationExpiryAllowance is the slice of the backstop that follows the session's +// STARTING write: the start deadline inside the launch goroutine. The preparation +// phase before it is already bounded by constants.AgentLaunchTimeout, which is read +// rather than copied so this tracks an operator-raised preparation budget. +const reservationExpiryAllowance = 5 * time.Minute + +// launchOrigin is the explicit automatic/manual classification threaded from the +// caller into the admission controller. It is never inferred from the transport or +// the handler, both of which carry human clicks and server-initiated automation. +type launchOrigin string + +const ( + launchOriginAutomatic launchOrigin = "automatic" + launchOriginManual launchOrigin = "manual" +) + +// ceilingFieldReasonCode and ceilingFieldCeiling name the log field and card +// metadata key shared across every ceiling log line, message row and audit +// record, so the reason code and the ceiling value are always found under +// the same name regardless of which call site wrote them. +const ( + ceilingFieldReasonCode = "reason_code" + ceilingFieldCeiling = "ceiling" +) + +// Reason codes carried verbatim by the admission log line and the card surface. +const ( + ceilingReasonRefused = "ceiling" + ceilingReasonManualOverride = "ceiling_manual_override" + ceilingReasonUnknownPopulation = "ceiling_unknown_population" + ceilingReasonSuperseded = "ceiling_superseded" + ceilingReasonDeferWriteFailed = "ceiling_defer_write_failed" + // ceilingReasonDroppedTaskIneligible covers AC-17b(a)-(d): the task is + // archived, cancelled, names a session that no longer exists, or moved + // to a step that no longer auto-starts. + ceilingReasonDroppedTaskIneligible = "ceiling_dropped_task_ineligible" + // ceilingReasonDroppedLaunchGateDeclined is AC-17b(e): the launch's own + // precondition (today, the terminal-PR guard) declined it at replay time. + ceilingReasonDroppedLaunchGateDeclined = "ceiling_dropped_launch_gate_declined" + // ceilingReasonDroppedUnreplayableRecord is AC-17b(f): the record's own + // ceiling_launch_kind is absent or outside the closed set. + ceilingReasonDroppedUnreplayableRecord = "ceiling_dropped_unreplayable_record" + // ceilingReasonSurfaceWriteFailed is AC-49g: the deferral itself persisted + // successfully, but the card note describing it could not be written. + ceilingReasonSurfaceWriteFailed = "ceiling_surface_write_failed" +) + +// admittedSessionLister supplies the persisted half of the population. It returns +// ids rather than a count because the population unions rows with reservations by +// session id, so a session holding both is counted once. +type admittedSessionLister interface { + ListAdmittedSessionIDs(ctx context.Context) ([]string, error) +} + +// ceilingReservation is an in-flight admission held until the session is observed +// running or the launch fails. sessionID is empty while the reservation is still +// keyed by a launch-scoped identifier, which is the window between admission at +// seam 1 and the session's creation. +type ceilingReservation struct { + sessionID string + takenAt time.Time +} + +// admissionRequest is one launch asking for capacity. +type admissionRequest struct { + taskID string + sessionID string + origin launchOrigin + seam string +} + +// admissionDecision is the controller's answer. +type admissionDecision struct { + admitted bool + manualOverride bool + reservationKey string + population int + populationKnown bool + ceiling int + reasonCode string + handedOff bool +} + +// sessionCeilingController is the single admission controller. Every mutation of +// the reservation set happens under its one mutex, together with the population +// read it is compared against. +type sessionCeilingController struct { + mu sync.Mutex + ceiling int + reservations map[string]*ceilingReservation + lister admittedSessionLister + logger *zap.Logger + now func() time.Time + newKey func() string +} + +func newSessionCeilingController(ceiling int, lister admittedSessionLister, logger *zap.Logger) *sessionCeilingController { + if logger == nil { + logger = zap.NewNop() + } + return &sessionCeilingController{ + ceiling: ceiling, + reservations: make(map[string]*ceilingReservation), + lister: lister, + logger: logger, + now: time.Now, + newKey: func() string { return "launch-" + uuid.NewString() }, + } +} + +// population returns the admitted session population. +func (c *sessionCeilingController) population(ctx context.Context) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + counted, err := c.countedRowsLocked(ctx) + if err != nil { + return 0, err + } + return c.populationLocked(counted), nil +} + +// countedRowsLocked reads the persisted half of the population. It is derived at +// decision time rather than kept as a running tally, so a restart, a panic or a +// missed release cannot leak capacity permanently. +func (c *sessionCeilingController) countedRowsLocked(ctx context.Context) (map[string]struct{}, error) { + if c.lister == nil { + return map[string]struct{}{}, nil + } + ids, err := c.lister.ListAdmittedSessionIDs(ctx) + if err != nil { + return nil, err + } + counted := make(map[string]struct{}, len(ids)) + for _, id := range ids { + counted[id] = struct{}{} + } + return counted, nil +} + +// populationLocked unions the persisted rows with the session-bound reservations by +// session id, then adds the launch-scoped reservations, each of which can collide +// with no row because no row for it exists yet. It is deliberately not the sum of +// two independent numbers. +func (c *sessionCeilingController) populationLocked(counted map[string]struct{}) int { + population := len(counted) + for _, reservation := range c.reservations { + if reservation.sessionID == "" { + population++ + continue + } + if _, alreadyCounted := counted[reservation.sessionID]; !alreadyCounted { + population++ + } + } + return population +} + +// reserveLocked records an in-flight reservation, keyed by session id where one +// exists and by a launch-scoped identifier otherwise. +func (c *sessionCeilingController) reserveLocked(sessionID string) string { + key := sessionID + if key == "" { + key = c.newKey() + } + c.reservations[key] = &ceilingReservation{sessionID: sessionID, takenAt: c.now()} + return key +} + +// admit decides one launch and, where it admits, records the reservation before +// returning. The population read and the reservation write share one critical +// section, so two launches arriving together cannot both see the same free slot. +func (c *sessionCeilingController) admit(ctx context.Context, req admissionRequest) admissionDecision { + if c == nil { + // A Service constructed without going through NewService (most + // commonly a test fixture built for narrow coverage of unrelated + // logic) has no ceiling resolved at all. Treat that the same as an + // explicitly unlimited ceiling rather than panicking on every launch. + return admissionDecision{admitted: true} + } + origin := req.origin + if origin != launchOriginManual && origin != launchOriginAutomatic { + origin = launchOriginAutomatic + c.logger.Warn("launch reached the session ceiling with no origin set; classified as automatic", + zap.String("seam", req.seam), zap.String("task_id", req.taskID), zap.String("session_id", req.sessionID)) + } + + c.mu.Lock() + defer c.mu.Unlock() + + counted, err := c.countedRowsLocked(ctx) + if err != nil { + return c.decideUnknownPopulationLocked(req, origin, err) + } + return c.decideLocked(req, origin, counted) +} + +// decideLocked is the ordinary admission decision, taken against a population the +// caller has already read inside the critical section. +func (c *sessionCeilingController) decideLocked(req admissionRequest, origin launchOrigin, counted map[string]struct{}) admissionDecision { + population := c.populationLocked(counted) + + if decision, ok := c.alreadyAdmittedLocked(req, counted, population); ok { + return decision + } + + decision := admissionDecision{ + population: population, + populationKnown: true, + ceiling: c.ceiling, + } + switch { + case c.ceiling == unlimitedSessionCeiling || population < c.ceiling: + decision.admitted = true + decision.reservationKey = c.reserveLocked(req.sessionID) + case origin == launchOriginManual: + decision.admitted = true + decision.manualOverride = true + decision.reasonCode = ceilingReasonManualOverride + decision.reservationKey = c.reserveLocked(req.sessionID) + default: + decision.reasonCode = ceilingReasonRefused + } + c.logDecision(req, origin, decision) + return decision +} + +// alreadyAdmittedLocked answers a request for a session that already holds a +// reservation or is already counted, so one launch consumes at most one unit of +// capacity however many seams it passes through. +func (c *sessionCeilingController) alreadyAdmittedLocked(req admissionRequest, counted map[string]struct{}, population int) (admissionDecision, bool) { + if req.sessionID == "" { + return admissionDecision{}, false + } + decision := admissionDecision{ + admitted: true, + population: population, + populationKnown: true, + ceiling: c.ceiling, + } + if _, held := c.reservations[req.sessionID]; held { + decision.reservationKey = req.sessionID + return decision, true + } + if _, isCounted := counted[req.sessionID]; isCounted { + return decision, true + } + return admissionDecision{}, false +} + +// decideUnknownPopulationLocked fails closed for automatic launches and open for +// manual ones. Reservations already held are untouched: the failure concerns the +// counted rows only. +func (c *sessionCeilingController) decideUnknownPopulationLocked(req admissionRequest, origin launchOrigin, err error) admissionDecision { + c.logger.Error("session ceiling could not read the admitted session population", + zap.String("seam", req.seam), zap.String("task_id", req.taskID), + zap.String("session_id", req.sessionID), zap.String("origin", string(origin)), zap.Error(err)) + + decision := admissionDecision{ + ceiling: c.ceiling, + reasonCode: ceilingReasonUnknownPopulation, + } + if origin == launchOriginManual { + decision.admitted = true + decision.manualOverride = true + decision.reservationKey = c.reserveLocked(req.sessionID) + } + c.logDecision(req, origin, decision) + return decision +} + +// logDecision emits the admission log line, which carries every reason code and is +// the only universal observable of a decision. +func (c *sessionCeilingController) logDecision(req admissionRequest, origin launchOrigin, decision admissionDecision) { + fields := []zap.Field{ + zap.String("seam", req.seam), + zap.String("task_id", req.taskID), + zap.String("session_id", req.sessionID), + zap.String("origin", string(origin)), + zap.Int(ceilingFieldCeiling, decision.ceiling), + zap.Bool("admitted", decision.admitted), + } + if decision.populationKnown { + fields = append(fields, zap.Int("population", decision.population)) + } + if decision.reasonCode != "" { + fields = append(fields, zap.String(ceilingFieldReasonCode, decision.reasonCode)) + } + c.logger.Info("session ceiling admission decision", fields...) +} + +// release drops a reservation by its key. Releasing an unknown or already-released +// key succeeds and decrements nothing. +func (c *sessionCeilingController) release(key string) { + if key == "" { + return + } + c.mu.Lock() + defer c.mu.Unlock() + delete(c.reservations, key) +} + +// rebind moves a launch-scoped reservation onto the session id that launch just +// created, as one operation rather than a release followed by an acquire, so the +// population never momentarily drops and no concurrent admission can take the +// freed unit. Where a reservation already occupies that session id — two +// launches racing to the same session, such as an Office identity-owned +// session two concurrent starts converge onto via +// EnsureSessionForAgentWithCreation — the launch-scoped reservation is +// released instead of overwriting the winner's slot: this is still one +// launch, mirroring rekey's own collision handling. Returning false leaves the +// caller's key pointing at the now-deleted launch-scoped entry, so its own +// later release is a harmless no-op rather than deleting the winner's still +// in-flight reservation. +func (c *sessionCeilingController) rebind(launchKey, sessionID string) bool { + if launchKey == "" || sessionID == "" { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + reservation, held := c.reservations[launchKey] + if !held { + return false + } + delete(c.reservations, launchKey) + if _, collision := c.reservations[sessionID]; collision { + return false + } + reservation.sessionID = sessionID + c.reservations[sessionID] = reservation + return true +} + +// rekey moves a reservation from the session that was gated onto the replacement +// session that will actually be launched. Where the replacement is already counted +// or already reserved, the original is released and no second unit is consumed: +// this is still one launch. +func (c *sessionCeilingController) rekey(ctx context.Context, fromSessionID, toSessionID string) bool { + if fromSessionID == "" || toSessionID == "" || fromSessionID == toSessionID { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + reservation, held := c.reservations[fromSessionID] + if !held { + return false + } + delete(c.reservations, fromSessionID) + + if _, alreadyReserved := c.reservations[toSessionID]; alreadyReserved { + // Do not transfer ownership onto a reservation the caller does not + // hold: the source key is already gone (deleted above), so the + // caller's later releaseIfNotConsumed becomes a no-op instead of + // deleting the actual holder's reservation and undercounting the + // population, matching rebind's collision behavior. + return false + } + if counted, err := c.countedRowsLocked(context.WithoutCancel(ctx)); err == nil { + if _, alreadyCounted := counted[toSessionID]; alreadyCounted { + return true + } + } + reservation.sessionID = toSessionID + c.reservations[toSessionID] = reservation + return true +} + +// handOffOrAdmit serves the dynamic-route relaunch seam. Where the session is still +// in the population, its counted membership is converted into a reservation under +// the same mutex: a relaunch replaces one agent process with another rather than +// adding one, so the hand-off is never refused and never changes the count. +// Where the session is not counted there is no slot to hand off and this is an +// ordinary admission request. +func (c *sessionCeilingController) handOffOrAdmit(ctx context.Context, req admissionRequest) admissionDecision { + if c == nil { + return admissionDecision{admitted: true} + } + origin := req.origin + if origin != launchOriginManual && origin != launchOriginAutomatic { + origin = launchOriginAutomatic + } + + c.mu.Lock() + defer c.mu.Unlock() + + counted, err := c.countedRowsLocked(ctx) + if err != nil { + return c.decideUnknownPopulationLocked(req, origin, err) + } + if _, isCounted := counted[req.sessionID]; req.sessionID == "" || !isCounted { + return c.decideLocked(req, origin, counted) + } + + decision := admissionDecision{ + admitted: true, + handedOff: true, + reservationKey: req.sessionID, + population: c.populationLocked(counted), + populationKnown: true, + ceiling: c.ceiling, + } + c.reservations[req.sessionID] = &ceilingReservation{sessionID: req.sessionID, takenAt: c.now()} + c.logDecision(req, origin, decision) + return decision +} + +// isSessionCeilingBacked implements AC-41: whether a session_id is currently +// backed by an in-flight reservation or a counted AC-1 population row. A nil +// controller (a Service built without one resolved, e.g. narrow test +// fixtures) reports every session as backed, the same "no ceiling in effect" +// posture admit/handOffOrAdmit already take. This method is consulted only +// by Executor's observation-only bypass detector (AC-41a) and never gates, +// delays or fails a launch itself. +func (c *sessionCeilingController) isSessionCeilingBacked(ctx context.Context, sessionID string) (bool, error) { + if c == nil || sessionID == "" { + return true, nil + } + c.mu.Lock() + defer c.mu.Unlock() + if _, reserved := c.reservations[sessionID]; reserved { + return true, nil + } + counted, err := c.countedRowsLocked(ctx) + if err != nil { + return false, err + } + _, ok := counted[sessionID] + return ok, nil +} + +// expireStaleReservations releases reservations whose launch neither reached a +// counted state nor reported failure inside the launch budget. It is the backstop, +// not the primary release edge. +func (c *sessionCeilingController) expireStaleReservations() int { + budget := constants.AgentLaunchTimeout + reservationExpiryAllowance + now := c.now() + + c.mu.Lock() + defer c.mu.Unlock() + released := 0 + for key, reservation := range c.reservations { + if now.Sub(reservation.takenAt) <= budget { + continue + } + delete(c.reservations, key) + released++ + c.logger.Warn("session ceiling released a reservation whose launch never reached a counted state", + zap.String("reservation_key", key), + zap.String("session_id", reservation.sessionID), + zap.Duration("held_for", now.Sub(reservation.takenAt)), + zap.Duration("budget", budget)) + } + return released +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_config.go b/apps/backend/internal/orchestrator/session_ceiling_config.go new file mode 100644 index 0000000000..b0b03d765e --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_config.go @@ -0,0 +1,92 @@ +package orchestrator + +import ( + "os" + "runtime" + "strconv" + "strings" + + "go.uber.org/zap" +) + +// maxConcurrentSessionsEnvVar is the only source of an operator-supplied session +// ceiling. There is no YAML setting and no stored setting for it by contract, which +// is why it is registered in the startup configuration inventory as an exclusion +// rather than an entry. +const maxConcurrentSessionsEnvVar = "KANDEV_MAX_CONCURRENT_SESSIONS" + +// unlimitedSessionCeiling is the configured value that disables refusal, matching +// the wip_limit convention. +const unlimitedSessionCeiling = 0 + +// minimumDefaultSessionCeiling floors the derived default so a single-core host +// still admits enough sessions to make progress. +const minimumDefaultSessionCeiling = 2 + +// defaultSessionCeiling derives the ceiling used when no operator value is set: +// half the host's cores, floored at minimumDefaultSessionCeiling. +func defaultSessionCeiling(numCPU int) int { + if half := numCPU / 2; half > minimumDefaultSessionCeiling { + return half + } + return minimumDefaultSessionCeiling +} + +// resolveSessionCeiling turns the raw environment value into the effective ceiling. +// It returns the rejected raw value when one was supplied but could not be used, so +// the caller can warn; an unset or blank value is not a rejection and takes the +// default silently. +func resolveSessionCeiling(raw string, numCPU int) (ceiling int, rejected string) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return defaultSessionCeiling(numCPU), "" + } + parsed, err := strconv.Atoi(trimmed) + if err != nil || parsed < 0 { + return defaultSessionCeiling(numCPU), raw + } + return parsed, "" +} + +// newSessionCeilingControllerFromEnv resolves maxConcurrentSessionsEnvVar exactly +// once, at controller construction, and builds the controller from the result. +// A rejected raw value is logged at WARN here rather than propagated, since this +// is the only production call site and the caller has nothing further to do with +// it besides log it. +func newSessionCeilingControllerFromEnv(lister admittedSessionLister, logger *zap.Logger) *sessionCeilingController { + ceiling, rejected := resolveSessionCeiling(os.Getenv(maxConcurrentSessionsEnvVar), runtime.NumCPU()) + if rejected != "" { + effectiveLogger := logger + if effectiveLogger == nil { + effectiveLogger = zap.NewNop() + } + effectiveLogger.Warn("session ceiling environment value could not be parsed; using the derived default", + zap.String("env_var", maxConcurrentSessionsEnvVar), + zap.String("value", rejected), + zap.Int("default", ceiling)) + } + return newSessionCeilingController(ceiling, lister, logger) +} + +// newSessionCeilingForRepo builds the controller the service owns, binding the +// repository as the persisted half of its population source. +// +// The repository is reached through a narrow consumer-side interface and a type +// assertion, following the idiom the other repository consumers in this package +// use. That idiom degrades silently, and here the degraded state is dangerous +// rather than merely reduced: an unbound controller still answers, but it counts +// only its own in-process reservations, so every session already running on the +// instance is invisible to it and the ceiling admits without bound. A build-time +// assertion covers the production repository; this warning covers everything else. +func newSessionCeilingForRepo(repo interface{}, logger *zap.Logger) *sessionCeilingController { + if logger == nil { + logger = zap.NewNop() + } + lister, ok := repo.(admittedSessionLister) + if !ok { + logger.Warn("repository cannot enumerate admitted sessions; the session ceiling will count only in-process reservations", + zap.String("env_var", maxConcurrentSessionsEnvVar)) + lister = nil + } + return newSessionCeilingControllerFromEnv(lister, logger) +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_config_test.go b/apps/backend/internal/orchestrator/session_ceiling_config_test.go new file mode 100644 index 0000000000..c72b990e2a --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_config_test.go @@ -0,0 +1,128 @@ +package orchestrator + +import ( + "runtime" + "strconv" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestDefaultSessionCeilingIsHalfTheCoresWithAFloorOfTwo(t *testing.T) { + for _, tc := range []struct { + numCPU int + want int + }{ + {numCPU: 1, want: 2}, + {numCPU: 2, want: 2}, + {numCPU: 3, want: 2}, + {numCPU: 4, want: 2}, + {numCPU: 8, want: 4}, + {numCPU: 14, want: 7}, + {numCPU: 64, want: 32}, + } { + if got := defaultSessionCeiling(tc.numCPU); got != tc.want { + t.Errorf("defaultSessionCeiling(%d) = %d, want %d", tc.numCPU, got, tc.want) + } + } +} + +func TestResolveSessionCeilingUsesTheEnvironmentValue(t *testing.T) { + for _, tc := range []struct { + name string + raw string + want int + }{ + {name: "zero is unlimited", raw: "0", want: unlimitedSessionCeiling}, + {name: "one", raw: "1", want: 1}, + {name: "twenty", raw: "20", want: 20}, + {name: "surrounding whitespace is tolerated", raw: " 6 ", want: 6}, + } { + t.Run(tc.name, func(t *testing.T) { + got, rejected := resolveSessionCeiling(tc.raw, 8) + if got != tc.want { + t.Errorf("resolveSessionCeiling(%q) ceiling = %d, want %d", tc.raw, got, tc.want) + } + if rejected != "" { + t.Errorf("resolveSessionCeiling(%q) rejected = %q, want no rejection", tc.raw, rejected) + } + }) + } +} + +// An unset or blank variable is the ordinary "no override" case: it takes the +// default and is not reported as a rejection, so it must not produce a WARN. +func TestResolveSessionCeilingFallsThroughSilentlyWhenUnset(t *testing.T) { + for _, raw := range []string{"", " "} { + got, rejected := resolveSessionCeiling(raw, 8) + if want := defaultSessionCeiling(8); got != want { + t.Errorf("resolveSessionCeiling(%q) ceiling = %d, want default %d", raw, got, want) + } + if rejected != "" { + t.Errorf("resolveSessionCeiling(%q) rejected = %q, want no rejection", raw, rejected) + } + } +} + +// A supplied-but-unusable value takes the default AND reports the raw value, which +// is what the caller logs at WARN. Reporting it is the whole difference from unset. +func TestResolveSessionCeilingRejectsUnusableValues(t *testing.T) { + for _, raw := range []string{"-1", "-20", "abc", "4.5", "1e3", "٤", "9999999999999999999999"} { + got, rejected := resolveSessionCeiling(raw, 8) + if want := defaultSessionCeiling(8); got != want { + t.Errorf("resolveSessionCeiling(%q) ceiling = %d, want default %d", raw, got, want) + } + if rejected != raw { + t.Errorf("resolveSessionCeiling(%q) rejected = %q, want %q", raw, rejected, raw) + } + } +} + +// The production default must be reachable through the same path the tests pin. +func TestResolveSessionCeilingDefaultMatchesThisHost(t *testing.T) { + got, _ := resolveSessionCeiling("", runtime.NumCPU()) + want := defaultSessionCeiling(runtime.NumCPU()) + if got != want { + t.Fatalf("resolveSessionCeiling(\"\") = %d, want %d", got, want) + } + if want < 2 { + t.Fatalf("default ceiling %s is below the floor of 2", strconv.Itoa(want)) + } +} + +// AC-18a: the ceiling is resolved from maxConcurrentSessionsEnvVar once, at +// controller construction, rather than read ad hoc by callers. +func TestNewSessionCeilingControllerFromEnvUsesTheOperatorValue(t *testing.T) { + t.Setenv(maxConcurrentSessionsEnvVar, "3") + c := newSessionCeilingControllerFromEnv(nil, zap.NewNop()) + if c.ceiling != 3 { + t.Fatalf("ceiling = %d, want 3", c.ceiling) + } +} + +func TestNewSessionCeilingControllerFromEnvFallsBackToDefaultWhenUnset(t *testing.T) { + t.Setenv(maxConcurrentSessionsEnvVar, "") + c := newSessionCeilingControllerFromEnv(nil, zap.NewNop()) + want := defaultSessionCeiling(runtime.NumCPU()) + if c.ceiling != want { + t.Fatalf("ceiling = %d, want default %d", c.ceiling, want) + } +} + +// AC-20: a set-but-unparseable value takes the default and is reported at WARN, +// distinguishing it from the silent unset case above. +func TestNewSessionCeilingControllerFromEnvWarnsOnRejectedValue(t *testing.T) { + t.Setenv(maxConcurrentSessionsEnvVar, "not-a-number") + core, logs := observer.New(zapcore.WarnLevel) + c := newSessionCeilingControllerFromEnv(nil, zap.New(core)) + + want := defaultSessionCeiling(runtime.NumCPU()) + if c.ceiling != want { + t.Fatalf("ceiling = %d, want default %d", c.ceiling, want) + } + if n := logs.FilterMessageSnippet("could not be parsed").Len(); n != 1 { + t.Fatalf("expected exactly one WARN about the rejected value, got %d: %v", n, logs.All()) + } +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_lifecycle_test.go b/apps/backend/internal/orchestrator/session_ceiling_lifecycle_test.go new file mode 100644 index 0000000000..427ffec6ec --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_lifecycle_test.go @@ -0,0 +1,226 @@ +package orchestrator + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/constants" +) + +// A seam-1 launch is admitted before its session exists, so its reservation starts +// launch-scoped and is moved onto the session id once the session is created. The +// population must not dip during the move. +func TestRebindMovesALaunchScopedReservationOntoItsSession(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 4, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", origin: launchOriginAutomatic, seam: "seam1"}) + if !d.admitted || d.reservationKey == "" { + t.Fatalf("admit = %+v, want admitted with a launch-scoped key", d) + } + before := mustPopulation(t, c) + + if !c.rebind(d.reservationKey, "s1") { + t.Fatal("rebind reported no reservation to move") + } + if got := mustPopulation(t, c); got != before { + t.Fatalf("population after rebind = %d, want %d unchanged", got, before) + } + + // The reservation now answers to the session id, so the session's own row + // landing does not double-count it. + lister.set("s1") + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population once the row lands = %d, want 1", got) + } + c.release("s1") + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after releasing the rebound key = %d, want 1 (the row remains)", got) + } +} + +func TestRebindReportsAnUnknownLaunchKey(t *testing.T) { + c := newTestController(t, 4, &fakeAdmittedLister{}) + if c.rebind("launch-does-not-exist", "s1") { + t.Fatal("rebind reported success for a key it does not hold") + } +} + +// startCreatedSession can redirect to a different session after it has gated, +// locked and reserved the original. The reservation must follow, or the acceptance +// edge fires with an id that matches nothing and the original leaks until the +// backstop. +func TestRekeyMovesTheReservationToTheReplacementSession(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 4, lister) + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "original", origin: launchOriginAutomatic, seam: "seam2"}) + + if !c.rekey(context.Background(), "original", "replacement") { + t.Fatal("rekey reported no reservation to move") + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after rekey = %d, want 1 (still one launch)", got) + } + + c.release("original") + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("releasing the original key changed the population to %d, want 1", got) + } + c.release("replacement") + if got := mustPopulation(t, c); got != 0 { + t.Fatalf("population after releasing the replacement = %d, want 0", got) + } +} + +// Where the replacement is already counted, the original is released and no second +// unit is consumed. +func TestRekeyConsumesNoSecondUnitWhenTheReplacementIsAlreadyCounted(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("replacement") + c := newTestController(t, 4, lister) + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "original", origin: launchOriginAutomatic, seam: "seam2"}) + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("precondition: population = %d, want 2", got) + } + + if !c.rekey(context.Background(), "original", "replacement") { + t.Fatal("rekey reported no reservation to move") + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after rekey onto a counted session = %d, want 1", got) + } +} + +// Where the replacement session already holds its own unrelated reservation +// (not a counted row), rekey must not report ownership transferring onto it: +// the caller would then release someone else's reservation instead of its +// own already-deleted, already-no-op key. +func TestRekeyDoesNotTransferOwnershipOnAReservationCollision(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 4, lister) + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "original", origin: launchOriginAutomatic, seam: "seam2"}) + c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "replacement", origin: launchOriginAutomatic, seam: "seam2"}) + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("precondition: population = %d, want 2", got) + } + + if c.rekey(context.Background(), "original", "replacement") { + t.Fatal("rekey reported ownership transferring onto a reservation collision, want false") + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after the collision = %d, want 1 (original released, replacement untouched)", got) + } + + // A caller told "false" leaves its key at "original" (already deleted by + // rekey), so its own cleanup is a no-op rather than deleting the + // replacement's still-live reservation. + c.release("original") + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after releasing the stale original key = %d, want 1 (no-op)", got) + } + c.release("replacement") + if got := mustPopulation(t, c); got != 0 { + t.Fatalf("population after releasing the replacement's own reservation = %d, want 0", got) + } +} + +// The automatic dynamic-route path enters while its session is still RUNNING. The +// hand-off converts that membership into a reservation without consulting the +// ceiling, so the population is flat across the CREATED write instead of dipping. +func TestHandOffConvertsCountedMembershipWithoutChangingThePopulation(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 2, lister) + + d := c.handOffOrAdmit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam5"}) + if !d.admitted || !d.handedOff { + t.Fatalf("handOffOrAdmit = %+v, want admitted as a hand-off", d) + } + if d.reservationKey != "s1" { + t.Fatalf("reservationKey = %q, want %q", d.reservationKey, "s1") + } + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population after the hand-off = %d, want 2 unchanged", got) + } + + // The relaunch moves the session out of the counted rows; the reservation is + // what keeps its slot held. + lister.set("s2") + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population across the CREATED write = %d, want 2 (the reservation holds the slot)", got) + } +} + +// A hand-off is never refused, even at a full ceiling, because it adds nothing. +func TestHandOffIsNeverRefusedAtTheCeiling(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1") + c := newTestController(t, 1, lister) + + d := c.handOffOrAdmit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam5"}) + if !d.admitted || !d.handedOff { + t.Fatalf("handOffOrAdmit at a full ceiling = %+v, want admitted as a hand-off", d) + } +} + +// A relaunch that aborts before dispatching a replacement releases the handed-off +// slot: a hand-off transfers a slot, it does not pin one. +func TestHandOffIsReleasedLikeAnyOtherReservation(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1") + c := newTestController(t, 2, lister) + c.handOffOrAdmit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam5"}) + + lister.set() + c.release("s1") + if got := mustPopulation(t, c); got != 0 { + t.Fatalf("population after releasing an aborted hand-off = %d, want 0", got) + } +} + +// The manual and recovery callers of seam 5 arrive after the route has been parked, +// so their session is not counted and there is no slot to hand off. That is an +// ordinary admission request and it is refusable. +func TestHandOffFallsBackToOrdinaryAdmissionForAnUncountedSession(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("other-1", "other-2") + c := newTestController(t, 2, lister) + + d := c.handOffOrAdmit(context.Background(), admissionRequest{taskID: "t1", sessionID: "parked", origin: launchOriginAutomatic, seam: "seam5"}) + if d.handedOff { + t.Fatalf("handOffOrAdmit = %+v, want an ordinary request for an uncounted session", d) + } + if d.admitted { + t.Fatalf("handOffOrAdmit = %+v, want refused at the ceiling", d) + } + if d.reasonCode != ceilingReasonRefused { + t.Fatalf("reasonCode = %q, want %q", d.reasonCode, ceilingReasonRefused) + } +} + +// A launch that neither reaches a counted state nor reports failure is released by +// the backstop, sized to the whole admission-to-STARTING window. +func TestExpireStaleReservationsReleasesOnlyReservationsPastTheLaunchBudget(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 10, lister) + + base := time.Now() + c.now = func() time.Time { return base } + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "stale", origin: launchOriginAutomatic, seam: "seam2"}) + + budget := constants.AgentLaunchTimeout + 5*time.Minute + c.now = func() time.Time { return base.Add(budget - time.Second) } + c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "fresh", origin: launchOriginAutomatic, seam: "seam2"}) + if released := c.expireStaleReservations(); released != 0 { + t.Fatalf("expireStaleReservations released %d reservations one second inside the budget, want 0", released) + } + + c.now = func() time.Time { return base.Add(budget + time.Second) } + if released := c.expireStaleReservations(); released != 1 { + t.Fatalf("expireStaleReservations released %d, want 1 (only the stale one)", released) + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after expiry = %d, want 1 (the fresh reservation survived)", got) + } +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_repo_binding_test.go b/apps/backend/internal/orchestrator/session_ceiling_repo_binding_test.go new file mode 100644 index 0000000000..68efc15f9e --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_repo_binding_test.go @@ -0,0 +1,21 @@ +package orchestrator + +import ( + "testing" + + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// TestRepositorySatisfiesAdmittedSessionLister pins the binding between the +// admission controller's population source and the repository method that backs +// it. The controller reaches its repository through a narrow consumer-side +// interface and a type assertion, following the in-tree idiom +// (lifecycleTaskMetadataLister). That idiom degrades silently: a renamed or +// re-signatured repository method does not break the build, it just makes the +// assertion fail at runtime and leaves the controller permanently unable to read +// the population. For every other consumer that means a skipped sweep; here it +// would mean the ceiling never counts persisted sessions at all. This +// compile-time assertion turns that into a build failure instead. +func TestRepositorySatisfiesAdmittedSessionLister(t *testing.T) { + var _ admittedSessionLister = (*sqlite.Repository)(nil) +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_test.go b/apps/backend/internal/orchestrator/session_ceiling_test.go new file mode 100644 index 0000000000..8f805fc151 --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_test.go @@ -0,0 +1,344 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "go.uber.org/zap" +) + +type fakeAdmittedLister struct { + mu sync.Mutex + ids []string + err error +} + +func (f *fakeAdmittedLister) ListAdmittedSessionIDs(context.Context) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return nil, f.err + } + return append([]string(nil), f.ids...), nil +} + +func (f *fakeAdmittedLister) set(ids ...string) { + f.mu.Lock() + defer f.mu.Unlock() + f.ids = ids +} + +func (f *fakeAdmittedLister) fail(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.err = err +} + +func newTestController(t *testing.T, ceiling int, lister *fakeAdmittedLister) *sessionCeilingController { + t.Helper() + c := newSessionCeilingController(ceiling, lister, zap.NewNop()) + if c == nil { + t.Fatal("newSessionCeilingController returned nil") + } + return c +} + +func mustPopulation(t *testing.T, c *sessionCeilingController) int { + t.Helper() + got, err := c.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + return got +} + +// The persisted rows alone are the population when nothing is in flight. +func TestPopulationCountsPersistedRows(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2", "s3") + c := newTestController(t, 10, lister) + if got := mustPopulation(t, c); got != 3 { + t.Fatalf("population = %d, want 3", got) + } +} + +// A session holding both a row and a reservation is one session, not two. Counting +// it twice would refuse a launch the instance has room for. +func TestPopulationUnionsRowsWithReservationsBySessionID(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 10, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginAutomatic, seam: "seam2"}) + if !d.admitted || d.reservationKey != "s9" { + t.Fatalf("admit = %+v, want admitted holding a reservation for s9", d) + } + if got := mustPopulation(t, c); got != 3 { + t.Fatalf("population = %d, want 3 (two rows plus one reservation)", got) + } + + // s9's launch reaches STARTING: it now holds a row AND its reservation, and it + // is still one session. Counting both would refuse a launch there is room for. + lister.set("s1", "s2", "s9") + if got := mustPopulation(t, c); got != 3 { + t.Fatalf("population = %d, want 3 (s9's row and reservation are one session)", got) + } +} + +// A seam-1 reservation has no session id yet and can collide with no row, so it +// adds to the population. Omitting it would make every seam-1 admission invisible +// to the ceiling until its session row lands. +func TestPopulationCountsLaunchScopedReservations(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1") + c := newTestController(t, 10, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", origin: launchOriginAutomatic, seam: "seam1"}) + if !d.admitted { + t.Fatalf("admit refused below the ceiling: %+v", d) + } + if d.reservationKey == "" { + t.Fatal("admit returned no reservation key for a seam-1 launch") + } + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population = %d, want 2 (one row plus one launch-scoped reservation)", got) + } +} + +func TestAdmitBelowCeilingReservesCapacity(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 2, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}) + if !d.admitted || d.manualOverride { + t.Fatalf("admit = %+v, want admitted without override", d) + } + if d.reservationKey != "s1" { + t.Fatalf("reservationKey = %q, want %q", d.reservationKey, "s1") + } + if d.ceiling != 2 || d.population != 0 || !d.populationKnown { + t.Fatalf("decision reported population %d/%d (known=%v), want 0/2 known", d.population, d.ceiling, d.populationKnown) + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after admit = %d, want 1", got) + } +} + +func TestAdmitAtCeilingRefusesAutomaticLaunch(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 2, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginAutomatic, seam: "seam2"}) + if d.admitted { + t.Fatalf("admit = %+v, want refused at the ceiling", d) + } + if d.reasonCode != ceilingReasonRefused { + t.Fatalf("reasonCode = %q, want %q", d.reasonCode, ceilingReasonRefused) + } + if d.reservationKey != "" { + t.Fatalf("a refusal took reservation %q, want none", d.reservationKey) + } + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population after a refusal = %d, want 2 unchanged", got) + } +} + +// A manual launch is admitted over the ceiling and still counts, so the population +// rises above the ceiling and subsequent automatic launches stay refused. +func TestAdmitAtCeilingAdmitsManualLaunchAndCountsIt(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 2, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginManual, seam: "seam2"}) + if !d.admitted || !d.manualOverride { + t.Fatalf("admit = %+v, want admitted as a manual override", d) + } + if d.reasonCode != ceilingReasonManualOverride { + t.Fatalf("reasonCode = %q, want %q", d.reasonCode, ceilingReasonManualOverride) + } + if got := mustPopulation(t, c); got != 3 { + t.Fatalf("population after a manual override = %d, want 3 (above the ceiling)", got) + } + next := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s10", origin: launchOriginAutomatic, seam: "seam2"}) + if next.admitted { + t.Fatalf("automatic launch admitted while the population is above the ceiling: %+v", next) + } +} + +// A second request for a session that already holds a reservation consumes no +// second unit: one launch is capped at one unit of capacity however many seams it +// passes through. +func TestAdmitIsIdempotentPerSessionID(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 2, lister) + + first := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam5"}) + second := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}) + if !first.admitted || !second.admitted { + t.Fatalf("first = %+v, second = %+v, want both admitted", first, second) + } + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after two admissions of one session = %d, want 1", got) + } +} + +// A session already counted as a row is admitted without taking a reservation, so a +// duplicate retry of a running launch cannot consume capacity. +func TestAdmitShortCircuitsForAnAlreadyCountedSession(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 2, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam3"}) + if !d.admitted { + t.Fatalf("admit = %+v, want admitted for an already-counted session even at the ceiling", d) + } + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population = %d, want 2 unchanged", got) + } +} + +func TestReleaseIsIdempotentAndIgnoresUnknownKeys(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 4, lister) + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}) + + c.release("does-not-exist") + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after releasing an unknown key = %d, want 1", got) + } + c.release("s1") + c.release("s1") + if got := mustPopulation(t, c); got != 0 { + t.Fatalf("population after a double release = %d, want 0", got) + } +} + +// Zero is unlimited, matching the wip_limit convention. +func TestUnlimitedCeilingAdmitsEverything(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2", "s3", "s4", "s5") + c := newTestController(t, unlimitedSessionCeiling, lister) + + for i := 0; i < 20; i++ { + d := c.admit(context.Background(), admissionRequest{taskID: "t", sessionID: fmt.Sprintf("new-%d", i), origin: launchOriginAutomatic, seam: "seam2"}) + if !d.admitted || d.manualOverride { + t.Fatalf("admit %d = %+v, want plain admitted under an unlimited ceiling", i, d) + } + } +} + +func TestCeilingOfOneAdmitsExactlyOneSession(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 1, lister) + + if d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}); !d.admitted { + t.Fatalf("first admit = %+v, want admitted", d) + } + if d := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginAutomatic, seam: "seam2"}); d.admitted { + t.Fatalf("second admit = %+v, want refused", d) + } +} + +// A failed population query fails CLOSED for automatic launches and open for manual +// ones, and neither surface may report a number it does not have. +func TestUnknownPopulationFailsClosedForAutomaticAndOpenForManual(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.fail(errors.New("database is locked")) + c := newTestController(t, 4, lister) + + auto := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}) + if auto.admitted { + t.Fatalf("automatic admit = %+v, want refused when the population is unknown", auto) + } + if auto.reasonCode != ceilingReasonUnknownPopulation { + t.Fatalf("automatic reasonCode = %q, want %q", auto.reasonCode, ceilingReasonUnknownPopulation) + } + if auto.populationKnown { + t.Fatal("automatic decision reported the population as known") + } + + manual := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginManual, seam: "seam2"}) + if !manual.admitted || !manual.manualOverride { + t.Fatalf("manual admit = %+v, want admitted as an override", manual) + } + if manual.populationKnown { + t.Fatal("manual decision reported the population as known") + } + if manual.reasonCode != ceilingReasonUnknownPopulation { + t.Fatalf("manual reasonCode = %q, want %q", manual.reasonCode, ceilingReasonUnknownPopulation) + } +} + +// The failure concerns the counted rows only: reservations already held stay held. +func TestUnknownPopulationDoesNotReleaseHeldReservations(t *testing.T) { + lister := &fakeAdmittedLister{} + c := newTestController(t, 4, lister) + c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}) + + lister.fail(errors.New("database is locked")) + c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginAutomatic, seam: "seam2"}) + + lister.fail(nil) + if got := mustPopulation(t, c); got != 1 { + t.Fatalf("population after a failed query = %d, want 1 (the held reservation survived)", got) + } +} + +// An unlabelled launch is automatic, so it is refused and deferred rather than +// silently granted an override. +func TestMissingOriginIsTreatedAsAutomatic(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + c := newTestController(t, 2, lister) + + d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", seam: "seam4"}) + if d.admitted { + t.Fatalf("admit with no origin = %+v, want refused as automatic", d) + } +} + +// Two launches racing for one free slot produce exactly one admission. +func TestConcurrentAdmissionsForOneFreeSlotAdmitExactlyOne(t *testing.T) { + lister := &fakeAdmittedLister{} + lister.set("s1") + c := newTestController(t, 2, lister) + + const racers = 32 + var wg sync.WaitGroup + results := make([]admissionDecision, racers) + start := make(chan struct{}) + for i := 0; i < racers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i] = c.admit(context.Background(), admissionRequest{ + taskID: fmt.Sprintf("t%d", i), + sessionID: fmt.Sprintf("race-%d", i), + origin: launchOriginAutomatic, + seam: "seam2", + }) + }(i) + } + close(start) + wg.Wait() + + admitted := 0 + for _, d := range results { + if d.admitted { + admitted++ + } + } + if admitted != 1 { + t.Fatalf("%d of %d racers admitted, want exactly 1", admitted, racers) + } + if got := mustPopulation(t, c); got != 2 { + t.Fatalf("population = %d, want 2 (the ceiling)", got) + } +} diff --git a/apps/backend/internal/orchestrator/session_ceiling_wiring_test.go b/apps/backend/internal/orchestrator/session_ceiling_wiring_test.go new file mode 100644 index 0000000000..2e9abce683 --- /dev/null +++ b/apps/backend/internal/orchestrator/session_ceiling_wiring_test.go @@ -0,0 +1,57 @@ +package orchestrator + +import ( + "context" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +type repoWithoutAdmittedLister struct{} + +// TestNewSessionCeilingForRepoBindsARepositoryThatCanCount pins the production +// wiring: the controller the service holds must read the persisted half of the +// population, not just its own reservations. +func TestNewSessionCeilingForRepoBindsARepositoryThatCanCount(t *testing.T) { + t.Setenv(maxConcurrentSessionsEnvVar, "4") + lister := &fakeAdmittedLister{} + lister.set("s1", "s2") + + controller := newSessionCeilingForRepo(lister, zap.NewNop()) + if controller == nil { + t.Fatal("newSessionCeilingForRepo returned nil") + } + if controller.ceiling != 4 { + t.Fatalf("ceiling = %d, want 4", controller.ceiling) + } + got, err := controller.population(context.Background()) + if err != nil { + t.Fatalf("population: %v", err) + } + if got != 2 { + t.Fatalf("population = %d, want 2; the repository is not bound", got) + } +} + +// TestNewSessionCeilingForRepoWarnsWhenTheRepositoryCannotCount covers the +// degrade path. A repository that cannot enumerate admitted sessions leaves the +// controller counting only its own reservations, which under-counts every +// persisted session. That is the one failure here that is invisible at runtime, +// so it is announced rather than accepted silently. +func TestNewSessionCeilingForRepoWarnsWhenTheRepositoryCannotCount(t *testing.T) { + t.Setenv(maxConcurrentSessionsEnvVar, "4") + core, logs := observer.New(zapcore.WarnLevel) + + controller := newSessionCeilingForRepo(repoWithoutAdmittedLister{}, zap.New(core)) + if controller == nil { + t.Fatal("newSessionCeilingForRepo returned nil") + } + if controller.lister != nil { + t.Fatal("a repository without ListAdmittedSessionIDs must not be bound as a lister") + } + if n := logs.FilterMessageSnippet("cannot enumerate admitted sessions").Len(); n != 1 { + t.Fatalf("expected exactly one WARN about the unbound repository, got %d: %v", n, logs.All()) + } +} diff --git a/apps/backend/internal/orchestrator/session_ensure.go b/apps/backend/internal/orchestrator/session_ensure.go index ae6b9b79c3..461c632250 100644 --- a/apps/backend/internal/orchestrator/session_ensure.go +++ b/apps/backend/internal/orchestrator/session_ensure.go @@ -82,7 +82,7 @@ func (s *Service) EnsureSession(ctx context.Context, taskID string, opts ...Ensu if existing := s.findExistingSession(ctx, taskID); existing != nil { if o.EnsureExecution { - s.tryEnsureExecution(ctx, existing.SessionID) + s.tryEnsureExecution(ctx, existing.SessionID, seam3CallShapeViewing, launchOriginManual, "") } return existing, nil } @@ -256,16 +256,42 @@ func (s *Service) existingResponse(ctx context.Context, taskID string, sess *mod // Failures are logged but not propagated — the session is still usable for chat // even if the execution can't be resumed (file/terminal panels will show // appropriate "not available" states). -func (s *Service) tryEnsureExecution(ctx context.Context, sessionID string) { +// +// callShape distinguishes this call's two production shapes (AC-47e): a seam-3 +// refusal is swallowed for the viewing shape (nothing to replay, nobody +// waiting) but deferred for the queue-drain shape, which supplies +// queuedMessageID so the deferred record can be matched back to its queued +// message at retry time. An unrecognized shape defaults to deferring +// (AC-47e1) — the unsafe direction here is silently dropping a launch, not +// refusing one. +func (s *Service) tryEnsureExecution( + ctx context.Context, sessionID string, callShape seam3CallShape, origin launchOrigin, queuedMessageID string, +) { session, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil || session == nil { return } - if err := s.ensureSessionRunning(ctx, sessionID, session); err != nil { - s.logger.Debug("ensure execution for existing session (non-fatal)", - zap.String("session_id", sessionID), - zap.Error(err)) + err = s.ensureSessionRunning(ctx, sessionID, session, origin) + if err == nil { + return } + if refusal, ok := isSeam3Refusal(err); ok { + switch callShape { + case seam3CallShapeViewing: + return + case seam3CallShapeQueueDrain: + s.deferSeam3QueueDrainRefusal(ctx, session.TaskID, sessionID, queuedMessageID, refusal) + return + default: + s.logger.Zap().Warn("tryEnsureExecution saw an unrecognized seam-3 call shape; defaulting to deferring", + zap.String("session_id", sessionID), zap.String("call_shape", string(callShape))) + s.deferSeam3QueueDrainRefusal(ctx, session.TaskID, sessionID, queuedMessageID, refusal) + return + } + } + s.logger.Debug("ensure execution for existing session (non-fatal)", + zap.String("session_id", sessionID), + zap.Error(err)) } // resolveTaskAgentProfile applies the 5-step resolution chain on the backend: diff --git a/apps/backend/internal/orchestrator/session_launch.go b/apps/backend/internal/orchestrator/session_launch.go index d80ca8bcd0..b3783a9549 100644 --- a/apps/backend/internal/orchestrator/session_launch.go +++ b/apps/backend/internal/orchestrator/session_launch.go @@ -313,6 +313,15 @@ func (s *Service) launchStart(ctx context.Context, req *LaunchSessionRequest) (* req.WorkflowStepID, req.PlanMode, req.AutoStart, req.Attachments, startTaskOptions{ProfileExplicit: req.ProfileExplicit, SpawnOrigin: req.SpawnOrigin}, ) + if errors.Is(err, ErrCeilingLaunchDeferred) { + // A ceiling-deferred launch (only reachable when req.AutoStart is + // true — a manual launch is always admitted, never deferred) is not + // this caller's failure to report: the sweep already persisted a + // replay record and owns retrying it. Fall through to the same + // no-op response the nil-execution case below returns, so the + // task-owned card note (not an RPC error) is the recovery surface. + execution, err = nil, nil + } if err != nil { return nil, err } @@ -376,6 +385,7 @@ func (s *Service) launchResume(ctx context.Context, req *LaunchSessionRequest) ( execution, err := s.ResumeTaskSessionWithOptions(ctx, req.TaskID, req.SessionID, executor.ResumeOptions{ AllowBranchReplacement: req.AllowBranchReplacement, AllowCompletedSessionResume: req.AllowCompletedSessionResume, + Origin: string(originFromAutoStart(req.AutoStart)), }) if err != nil { return nil, err diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index 043254a8c5..47bc5cf749 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -662,6 +662,16 @@ func (s *Service) startCreatedSession( return nil, errOfficeTaskStartRequiresScheduler } + seam2Res, deferred, err := s.admitOrDeferSeam2(ctx, taskID, sessionID, originFromAutoStart(autoStart), + seam2StartCreatedPayload(sessionID, agentProfileID, prompt, skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext, options)) + if err != nil { + return nil, err + } + if deferred { + return nil, nil + } + defer seam2Res.releaseIfNotConsumed() + // Reserve any pending "start it later" intent for the rest of this start. // Taken here rather than just before the launch: everything below persists // session state this start owns, and a gate that claims the intent during @@ -769,9 +779,11 @@ func (s *Service) startCreatedSession( return nil, fmt.Errorf("session was switched but no active session found: %w", activeErr) } session = activeSession + seam2Res.rekeyToSession(ctx, activeSession.ID) sessionID = activeSession.ID effectiveProfileID = activeSession.AgentProfileID } + s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam2Res.manualOverride, seam2Res.population, seam2Res.populationKnown, seam2Res.ceiling) if effectiveProfileID, err = s.resolveDynamicLaunchExecution(ctx, session, effectiveProfileID, true); err != nil { return nil, err @@ -885,6 +897,7 @@ func (s *Service) startCreatedSession( // The agent is running, so the reservation becomes a consumption. launchClaim.consume(ctx) + seam2Res.consume() // Ensure a PR watch exists so the poller can detect PRs created by the agent. // PrepareTaskSession may have already created one, but if that goroutine failed @@ -1104,10 +1117,17 @@ func (s *Service) StartTaskWithEnv(ctx context.Context, taskID string, agentProf // StartTaskWithEnvAndSkills starts a task with launch-scoped environment and // skills selected by the Office run rather than by the durable profile. +// +// Its only production caller is the Office scheduler's task starter +// (backendapp's newOfficeTaskStarter), which always passes autoStart=false — +// the user kicked off the task; Office only chose the agent and skills. AC-13d +// requires that traffic be classified automatic regardless, so the origin is +// forced here rather than trusted from autoStart. func (s *Service) StartTaskWithEnvAndSkills(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, priority string, prompt string, workflowStepID string, planMode, autoStart bool, attachments []v1.MessageAttachment, env map[string]string, additionalSkillSlugs []string) (*executor.TaskExecution, error) { return s.startTask(ctx, taskID, agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, startTaskOptions{ Env: env, AdditionalSkillSlugs: append([]string(nil), additionalSkillSlugs...), + Origin: launchOriginAutomatic, }) } @@ -1138,6 +1158,12 @@ type startTaskOptions struct { // WorkflowEntryID pins source bindings and explicit route retries to the // immutable step-entry ledger row that initiated an automatic launch. WorkflowEntryID int64 + // Origin overrides the automatic/manual classification the session ceiling + // would otherwise derive from autoStart (AC-13a). It exists for callers + // AC-13d names, whose autoStart=false does not mean a human asked for this + // process to start now: an Office scheduler launch only chose a provider. + // Zero value means "derive from autoStart". + Origin launchOrigin } // StartTaskWithRoute launches a stable Office identity through a complete @@ -1162,6 +1188,10 @@ func (s *Service) StartTaskWithRoute( Env: launch.Env, AdditionalSkillSlugs: append([]string(nil), launch.AdditionalSkillSlugs...), Route: &route, + // AC-13d: a routed Office launch always passes autoStart=false + // (the user kicked off the task; Office only chose the provider), + // which is not the ceiling's manual/automatic question. + Origin: launchOriginAutomatic, }) } @@ -1269,6 +1299,20 @@ func (s *Service) startTask(ctx context.Context, taskID string, agentProfileID s zap.Bool("auto_start", autoStart), zap.Int("attachments", len(attachments))) + origin := opts.Origin + if origin == "" { + origin = originFromAutoStart(autoStart) + } + seam1Res, deferred, err := s.admitOrDeferSeam1(ctx, taskID, origin, + seam1StartPayload(agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, opts)) + if err != nil { + return nil, err + } + if deferred { + return nil, ErrCeilingLaunchDeferred + } + defer seam1Res.releaseIfNotConsumed() + // Reserve any pending "start it later" intent for the whole of this start, // taken before the session is prepared rather than just before the launch: // the gate only needs the intent to still be there, so any window in which @@ -1444,6 +1488,9 @@ func (s *Service) startTask(ctx context.Context, taskID string, agentProfileID s return nil, fmt.Errorf("explicit workflow start session became terminal before promotion") } } + seam1Res.rebindToSession(sessionID) + s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam1Res.manualOverride, seam1Res.population, seam1Res.populationKnown, seam1Res.ceiling) + // Seed a matching conditional session configuration before lifecycle // startup. The ACP manager applies this durable runtime layer after the // selected profile and before the first prompt, preserving the original @@ -1614,6 +1661,7 @@ func (s *Service) startTask(ctx context.Context, taskID string, agentProfileID s // The agent is running, so the reservation becomes a consumption. launchClaim.consume(ctx) + seam1Res.consume() // Note: Task stays in SCHEDULING state until the agent is fully initialized. // The executor will transition to IN_PROGRESS after StartAgentProcess() succeeds. @@ -2115,7 +2163,7 @@ func (s *Service) moveTaskToWorkflowStep(ctx context.Context, taskID, workflowSt } dbTask.WorkflowStepID = workflowStepID dbTask.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTask(ctx, dbTask); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, dbTask); err != nil { s.logger.Warn("failed to move task to workflow step", zap.String("task_id", taskID), zap.String("workflow_step_id", workflowStepID), @@ -2677,7 +2725,7 @@ func (s *Service) ResumeTaskSessionAndPrompt( ctx, taskID, sessionID, - executor.ResumeOptions{}, + executor.ResumeOptions{Origin: string(launchOriginManual)}, func(resumeCtx context.Context, attempt *resumeAttempt, _ *executor.TaskExecution) error { var promptErr error result, promptErr = s.promptTask( @@ -2689,6 +2737,7 @@ func (s *Service) ResumeTaskSessionAndPrompt( planMode, attachments, false, + launchOriginManual, promptTaskOptions{resumeAttempt: attempt}, ) return promptErr @@ -2797,6 +2846,16 @@ func (s *Service) resumeTaskSessionWithContinuation( if isOfficeTask { return nil, decorateResumeFailure(errOfficeTaskResumeRequiresScheduler) } + seam4Res, deferred, err := s.admitOrDeferSeam4(ctx, taskID, sessionID, launchOrigin(options.Origin), seam4ResumePayload(sessionID, options)) + if err != nil { + return nil, err + } + if deferred { + return nil, nil + } + defer seam4Res.releaseIfNotConsumed() + s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam4Res.manualOverride, seam4Res.population, seam4Res.populationKnown, seam4Res.ceiling) + if _, err := s.resolveDynamicLaunchExecution(resumeCtx, session, session.AgentProfileID, true); err != nil { if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil { s.cleanupCancelledResumeAttempt(attempt) @@ -2894,6 +2953,7 @@ func (s *Service) resumeTaskSessionWithContinuation( return nil, attemptErr } execution.SessionState = v1.TaskSessionState(readySession.State) + seam4Res.consume() persistBranchRecovery() // Backfill the initial user message when a prior failed launch never got @@ -3075,6 +3135,16 @@ func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessi return fmt.Errorf("session is pending approval - use Approve button to proceed or send a message to request changes") } + preConsultRes, refused, err := s.admitOrDeferWorkflowStepEnsure(ctx, taskID, sessionID, workflowStepID) + if err != nil { + return err + } + if refused { + return errSeam3WorkflowStepEnsureDeferred + } + defer preConsultRes.releaseIfNotConsumed() + s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, preConsultRes.manualOverride, preConsultRes.population, preConsultRes.populationKnown, preConsultRes.ceiling) + s.advanceTaskWorkflowStep(ctx, dbTask, workflowStepID, session) effectivePrompt, err := s.buildWorkflowEntryPrompt( @@ -3084,9 +3154,10 @@ func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessi return fmt.Errorf("failed to build workflow prompt: %w", err) } - if err := s.ensureSessionRunning(ctx, sessionID, session); err != nil { + if err := s.ensureSessionRunning(ctx, sessionID, session, launchOriginAutomatic); err != nil { return err } + preConsultRes.consume() // Apply conditional session settings after a manual resume and before the // step prompt. The helper reloads the session so a resume-created runtime @@ -3118,7 +3189,7 @@ func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessi // included): if promptTask's own internal ErrExecutionNotFound recovery // fires, it must reuse this composed prompt rather than recomposing from // the destination step's own template and discarding the handoff. - _, err = s.promptTask(ctx, taskID, sessionID, effectivePrompt, "", stepPlanMode, nil, false, promptTaskOptions{ + _, err = s.promptTask(ctx, taskID, sessionID, effectivePrompt, "", stepPlanMode, nil, false, launchOriginAutomatic, promptTaskOptions{ promptAlreadyComposed: true, fallbackLaunchPrompt: composedLaunchPrompt, fallbackRetryPrompt: effectivePrompt, @@ -3144,7 +3215,7 @@ func (s *Service) advanceTaskWorkflowStep(ctx context.Context, task *models.Task } task.WorkflowStepID = workflowStepID task.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTask(ctx, task); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { s.logger.Warn("failed to update task workflow step", zap.String("task_id", task.ID), zap.String("workflow_step_id", workflowStepID), @@ -3162,8 +3233,8 @@ func (s *Service) advanceTaskWorkflowStep(ctx context.Context, task *models.Task // ensureSessionRunning resumes the session if the agent is not actually running. // After lazy recovery, a session may be in WAITING_FOR_INPUT with no agent process; // this function detects that case and triggers a resume. -func (s *Service) ensureSessionRunning(ctx context.Context, sessionID string, session *models.TaskSession) error { - attempt, err := s.ensureSessionRunningWithAttempt(ctx, sessionID, session) +func (s *Service) ensureSessionRunning(ctx context.Context, sessionID string, session *models.TaskSession, origin launchOrigin) error { + attempt, err := s.ensureSessionRunningWithAttempt(ctx, sessionID, session, origin) if attempt != nil { attempt.finish(s.resumeAttemptStore()) } @@ -3178,6 +3249,7 @@ func (s *Service) ensureSessionRunningWithAttempt( ctx context.Context, sessionID string, session *models.TaskSession, + origin launchOrigin, ) (*resumeAttempt, error) { releaseLifecycleLock := s.acquireSessionLifecycleLock(sessionID) defer releaseLifecycleLock() @@ -3201,7 +3273,7 @@ func (s *Service) ensureSessionRunningWithAttempt( if !owner { return s.waitForSharedResumeAttempt(ctx, sessionID, startupAttempt) } - return s.runResumeAttempt(ctx, sessionID, session, isOfficeTask, startupAttempt) + return s.runResumeAttempt(ctx, sessionID, session, isOfficeTask, startupAttempt, origin) } func (s *Service) sessionAlreadyPromptReady(ctx context.Context, sessionID string) bool { @@ -3244,6 +3316,7 @@ func (s *Service) runResumeAttempt( session *models.TaskSession, isOfficeTask bool, startupAttempt *resumeAttempt, + origin launchOrigin, ) (*resumeAttempt, error) { resumeCtx := cancellableResumeContext(startupAttempt) preparedSession, ready, err := s.prepareExistingSessionForResume( @@ -3260,7 +3333,7 @@ func (s *Service) runResumeAttempt( s.logger.Debug("agent not running for session, attempting resume", zap.String("session_id", sessionID), zap.String("session_state", string(session.State))) - return s.coldResumeSession(resumeCtx, sessionID, session, isOfficeTask, startupAttempt) + return s.coldResumeSession(resumeCtx, sessionID, session, isOfficeTask, startupAttempt, origin) } func (s *Service) finishResumeAttemptWithError( @@ -3326,7 +3399,14 @@ func (s *Service) coldResumeSession( session *models.TaskSession, isOfficeTask bool, startupAttempt *resumeAttempt, + origin launchOrigin, ) (*resumeAttempt, error) { + seam3Res, refusal := s.admitSeam3(ctx, session.TaskID, sessionID, origin) + if refusal != nil { + return startupAttempt, refusal + } + defer seam3Res.releaseIfNotConsumed() + s.recordManualOverrideIfAdmitted(ctx, session.TaskID, sessionID, seam3Res.manualOverride, seam3Res.population, seam3Res.populationKnown, seam3Res.ceiling) // Bounded to two attempts: a fresh cold resume, and — if the launched // agent never reports prompt-ready — one reap-and-retry, mirroring the @@ -3336,9 +3416,13 @@ func (s *Service) coldResumeSession( // surfaced a bare "agent not ready after resume: ... context deadline // exceeded" with no self-heal, unlike every other resume path in this // file. + // + // The gate above runs once, before the first attempt: a retry here is + // recovering an already-admitted launch, not requesting a new one. for attemptNumber := 1; ; attemptNumber++ { retryable, err := s.attemptColdResume(ctx, sessionID, session, isOfficeTask, startupAttempt) if err == nil { + seam3Res.consume() if validationErr := s.validateResumeAttempt(startupAttempt); validationErr != nil { s.cleanupCancelledResumeAttempt(startupAttempt) return startupAttempt, validationErr @@ -5180,7 +5264,7 @@ func (s *Service) persistSessionCommit(ctx context.Context, sessionID string, tr // If planMode is true, a plan mode prefix is prepended to the prompt. // Attachments (images) are passed through to the agent if provided. func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment, dispatchOnly bool) (*PromptResult, error) { - return s.promptTask(ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, promptTaskOptions{}) + return s.promptTask(ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, launchOriginManual, promptTaskOptions{}) } type promptTaskOptions struct { @@ -5322,31 +5406,15 @@ func (promptTaskOptions) failureContext(ctx context.Context) (context.Context, c // executeQueuedMessage's own deferred cleanup instead // (clearQueuedDispatchInFlightIfCurrent), which is safe since none of them // block on an agent turn. -func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment, dispatchOnly bool, options promptTaskOptions) (*PromptResult, error) { +func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment, dispatchOnly bool, origin launchOrigin, options promptTaskOptions) (*PromptResult, error) { s.logPromptTaskCall(taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly) - if err := s.validatePromptTaskStart(sessionID); err != nil { - return nil, err - } - // A compound resume passes its cancellable attempt context through the - // entire retry. Check ownership before any repository read can observe the - // cancellation as a generic context error, so the cancellation owner keeps - // the typed outcome and the old prompt cannot enter admission. - if err := s.validateResumeAttempt(options.resumeAttempt); err != nil { + // Check resume-attempt ownership before any repository read can observe a + // compound resume's cancellation as a generic context error. + if err := s.validatePromptTaskPreconditions(sessionID, options.resumeAttempt); err != nil { return nil, err } - // Only allow prompts when the session is ready for input. - session, err := s.loadPromptableSession(ctx, taskID, sessionID) - if err != nil { - return nil, err - } - if options.requireNonterminalSession { - session, err = s.loadNonterminalWorkflowAutoStartSession(ctx, sessionID) - if err != nil { - return nil, err - } - } - foregroundClaim, err := s.claimForegroundForPrompt(taskID, sessionID, session) + session, foregroundClaim, err := s.prepareSessionAndForegroundClaimForPrompt(ctx, taskID, sessionID, options) if err != nil { return nil, err } @@ -5354,59 +5422,162 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom // Apply config-mode and plan-mode prompt transforms. effectivePrompt := s.effectivePromptForSession(sessionID, prompt, planMode, session) - // Ensure the agent process is actually running. After a lazy backend restart, - // the session may be in WAITING_FOR_INPUT but no agent process exists yet. + // After a lazy backend restart the session may be WAITING_FOR_INPUT with no agent process yet. _, hadExecutionBeforeEnsure := s.executor.GetExecutionBySession(sessionID) - resumeAttempt := options.resumeAttempt - ownsResumeAttempt := false - resumedForPrompt := resumeAttempt != nil || !hadExecutionBeforeEnsure - if resumeAttempt == nil { - resumeAttempt, err = s.ensureSessionRunningWithAttempt(ctx, sessionID, session) - ownsResumeAttempt = resumeAttempt != nil - } - if resumeAttempt != nil && ownsResumeAttempt { - defer resumeAttempt.finish(s.resumeAttemptStore()) - } - resumePromptCtx := ctx - if resumeAttempt != nil { - resumePromptCtx = cancellableResumeContext(resumeAttempt) - if err := s.validateResumeAttempt(resumeAttempt); err != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) - return nil, err - } + resumedForPrompt := options.resumeAttempt != nil || !hadExecutionBeforeEnsure + resumeAttempt, finishResumeAttempt, resumePromptCtx, err := s.resolveAndAdmitResumeAttempt( + ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, origin, options, session, foregroundClaim, + ) + if finishResumeAttempt != nil { + defer finishResumeAttempt() } if err != nil { - s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) - if errors.Is(err, errSessionAwaitingRuntimeLaunch) { - return nil, fmt.Errorf("%w: failed to ensure session is running: %w", ErrSessionRuntimeUnavailable, err) - } - return nil, fmt.Errorf("failed to ensure session is running: %w", err) - } - if err := s.validateResumeAttempt(resumeAttempt); err != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) return nil, err } - // Reload session after ensureSessionRunning. If a resume happened, ResumeSession - // updated the session's AgentExecutionID via persistResumeState — but only on the - // pointer it received. If anything along the way swapped the pointer or the - // caller's struct is otherwise stale (e.g. a concurrent write), executor.Prompt - // would call PromptAgent with the OLD execution ID and get ErrExecutionNotFound. - // Re-reading from the DB after ensureSessionRunning guarantees we use the - // freshly-persisted AgentExecutionID. - session = s.reloadPromptSession(resumePromptCtx, sessionID, session) - // Re-apply transforms in case metadata changed during ensureSessionRunning. - effectivePrompt = s.effectivePromptForSession(sessionID, prompt, planMode, session) - activityExecutionID, _ := s.agentManager.GetExecutionIDForSession(resumePromptCtx, sessionID) - foregroundDispatch := s.beginForegroundDispatch(sessionID, foregroundClaim, activityExecutionID) - if foregroundDispatch == nil { - s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) - return nil, fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress) + // Reload the session so executor.Prompt uses the freshly-persisted + // AgentExecutionID, then begin the foreground dispatch on it. + session, effectivePrompt, foregroundDispatch, err := s.beginForegroundDispatchForPrompt( + resumePromptCtx, taskID, sessionID, prompt, planMode, session, foregroundClaim, + ) + if err != nil { + return nil, err } - beforeDispatch := options.beforeDispatch - runBeforeDispatch := func() error { + runBeforeDispatch := runBeforeDispatchOnce(options.beforeDispatch) + + // Cache the replay identity and acquire the model-switch guard before switching. + modelSwitchGuard, err := s.prepareModelSwitchGuard( + resumePromptCtx, taskID, sessionID, prompt, model, planMode, attachments, options, session, + foregroundDispatch, resumeAttempt, + ) + if err != nil { + return nil, err + } + if modelSwitchGuard != nil { + defer func() { + if modelSwitchGuard != nil { + modelSwitchGuard.release() + } + }() + } + + switchResult, switchHandled, modelSwitchGuard, switchErr := s.resolveModelSwitchAttempt( + resumePromptCtx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, runBeforeDispatch, + resumeAttempt, modelSwitchGuard, + ) + if switchHandled { + return switchResult, switchErr + } + + session, rollback, releaseDispatchGuard, err := s.claimAndGuardDispatch( + resumePromptCtx, taskID, sessionID, foregroundClaim, options, foregroundDispatch, resumeAttempt, + ) + if err != nil { + return nil, err + } + if releaseDispatchGuard != nil { + // The agentctl dispatch callback is the acceptance boundary; releasing here keeps + // terminalization exclusive with admission without holding the guard for the whole turn. + defer releaseDispatchGuard() + } + + return s.runPromptTurn( + resumePromptCtx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, dispatchOnly, + effectivePrompt, session, rollback, options, foregroundDispatch, runBeforeDispatch, + releaseDispatchGuard, resumeAttempt, + ) +} + +// runPromptTurn is promptTask's dispatch tail: it validates and runs the +// dispatch boundary, prepares the dispatch callback, invokes the executor, +// and folds the outcome into promptTask's single PromptResult/error return. +func (s *Service) runPromptTurn( + ctx context.Context, + taskID, sessionID, prompt string, + planMode, resumedForPrompt bool, + attachments []v1.MessageAttachment, + dispatchOnly bool, + effectivePrompt string, + session *models.TaskSession, + rollback promptClaimRollback, + options promptTaskOptions, + foregroundDispatch *foregroundDispatch, + runBeforeDispatch func() error, + releaseDispatchGuard func(), + resumeAttempt *resumeAttempt, +) (*PromptResult, error) { + promptCtx, session, earlyResult, err := s.validateAndRunDispatchBoundary( + ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, + session, rollback, options, foregroundDispatch, runBeforeDispatch, + ) + if err != nil || earlyResult != nil { + return earlyResult, err + } + onDispatched, dispatchOutcome := s.preparePromptDispatchCallback( + promptCtx, taskID, sessionID, session, rollback, options, foregroundDispatch, releaseDispatchGuard, + ) + result, execErr := s.executor.PromptWithDispatchCallback( + promptCtx, taskID, sessionID, effectivePrompt, attachments, dispatchOnly, + onDispatched, session, + ) + return s.finishPromptExecutorDispatch( + ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, + rollback, options, foregroundDispatch, releaseDispatchGuard, result, execErr, dispatchOutcome, resumeAttempt, + ) +} + +// resolveResumeAttempt returns the requested resume attempt unchanged when +// the caller already holds one (options.resumeAttempt, a compound resume's +// own cancellable attempt), or acquires a fresh one via +// ensureSessionRunningWithAttempt otherwise. owns reports whether promptTask +// itself is responsible for finishing the returned attempt. +func (s *Service) resolveResumeAttempt( + ctx context.Context, + sessionID string, + session *models.TaskSession, + origin launchOrigin, + requested *resumeAttempt, +) (attempt *resumeAttempt, owns bool, err error) { + if requested != nil { + return requested, false, nil + } + attempt, err = s.ensureSessionRunningWithAttempt(ctx, sessionID, session, origin) + return attempt, attempt != nil, err +} + +// resolveAndAdmitResumeAttempt resolves promptTask's resume attempt and admits +// it in one step, returning a finish func for the caller to defer (nil when +// promptTask does not own the attempt) instead of deferring it here, since a +// defer inside this helper would fire when this helper returns rather than +// when promptTask itself does. +func (s *Service) resolveAndAdmitResumeAttempt( + ctx context.Context, + taskID, sessionID, prompt, model string, + planMode bool, + attachments []v1.MessageAttachment, + dispatchOnly bool, + origin launchOrigin, + options promptTaskOptions, + session *models.TaskSession, + foregroundClaim *foregroundClaim, +) (resumeAttempt *resumeAttempt, finish func(), resumePromptCtx context.Context, err error) { + resumeAttempt, owns, err := s.resolveResumeAttempt(ctx, sessionID, session, origin, options.resumeAttempt) + if resumeAttempt != nil && owns { + finish = func() { resumeAttempt.finish(s.resumeAttemptStore()) } + } + resumePromptCtx, err = s.admitResumeAttemptForPrompt( + ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, options, + foregroundClaim, resumeAttempt, err, + ) + return resumeAttempt, finish, resumePromptCtx, err +} + +// runBeforeDispatchOnce wraps a caller-supplied pre-dispatch hook so it runs +// at most once: nil beforeDispatch is a no-op, and a successful call clears +// the hook so a later retry within the same promptTask attempt does not run +// it again. +func runBeforeDispatchOnce(beforeDispatch func() error) func() error { + return func() error { if beforeDispatch == nil { return nil } @@ -5416,165 +5587,283 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom beforeDispatch = nil return nil } +} - // Cache and reserve the replay identity before model switching. A restart - // based switch dispatches its prompt from StartAgentProcess, before the - // ordinary PromptTask admission path can initialize these records. - s.rememberTurnPromptWithAccepted(sessionID, prompt, model, planMode, attachments, options.onAccepted) - requiresModelSwitch := modelSwitchRequired(session, model) - var modelSwitchGuard *lockedCancelInFlightGuard - if requiresModelSwitch && resumeAttempt != nil { - modelSwitchGuard, err = s.lockResumeAttemptAdmission(resumePromptCtx, sessionID, resumeAttempt) - if err != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - s.rollbackForegroundDispatchOnFailure(resumePromptCtx, taskID, sessionID, foregroundDispatch) - return nil, err - } - defer func() { - if modelSwitchGuard != nil { - modelSwitchGuard.release() - } - }() - } - if requiresModelSwitch { - s.beginInitialPromptAttempt(sessionID, s.isDynamicPromptSession(session)) - if err := runBeforeDispatch(); err != nil { - s.rollbackForegroundDispatchOnFailure(resumePromptCtx, taskID, sessionID, foregroundDispatch) - s.clearPromptAttemptEvidence(sessionID, "", 0) - return nil, err - } - } - - if result, handled, switchErr := s.trySwitchModelForPrompt( - resumePromptCtx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, - ); handled { - if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - return nil, resumeErr - } - if switchErr != nil { - s.clearPromptAttemptEvidence(sessionID, "", 0) - } +// resolveModelSwitchAttempt runs attemptModelSwitchForPrompt and folds its +// three-way outcome (not handled / handled-but-resume-attempt-lost / +// handled) together with modelSwitchGuard's release into promptTask's single +// remaining decision: whether to return switchResult/switchErr immediately. +// The returned modelSwitchGuard reflects the same release-then-nil update +// promptTask's own deferred cleanup closure observes, since Go closures +// capture the reassigned variable, not a snapshot of it. +func (s *Service) resolveModelSwitchAttempt( + ctx context.Context, + taskID, sessionID, model, effectivePrompt string, + session *models.TaskSession, + foregroundDispatch *foregroundDispatch, + runBeforeDispatch func() error, + resumeAttempt *resumeAttempt, + modelSwitchGuard *lockedCancelInFlightGuard, +) (switchResult *PromptResult, handled bool, remainingGuard *lockedCancelInFlightGuard, switchErr error) { + result, handledSwitch, attemptErr := s.attemptModelSwitchForPrompt( + ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, runBeforeDispatch, + ) + if !handledSwitch { if modelSwitchGuard != nil { modelSwitchGuard.release() modelSwitchGuard = nil } - return result, switchErr + return nil, false, modelSwitchGuard, nil + } + if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { + s.cleanupCancelledResumeAttempt(resumeAttempt) + return nil, true, modelSwitchGuard, resumeErr } if modelSwitchGuard != nil { modelSwitchGuard.release() modelSwitchGuard = nil } + return result, true, modelSwitchGuard, attemptErr +} - session, rollback, err := s.claimPromptDispatchWithResumeAttempt( - resumePromptCtx, taskID, sessionID, options.claimEntryID, options.lifecyclePrompt, - options.reserveTurnUntilDispatch, options.promptDispatchRecovery, - options.afterClaim, foregroundClaim, options.expectedCurrentTurnID, - options.requireNonterminalSession, resumeAttempt, options.expectedSessionIdentity, - ) +// acquireModelSwitchGuard locks the resume attempt's admission guard for a +// prompt that both resumes a session and switches its model, when both +// conditions apply. It returns (nil, nil) when no guard is needed. +func (s *Service) acquireModelSwitchGuard( + ctx context.Context, + taskID, sessionID string, + foregroundDispatch *foregroundDispatch, + requiresModelSwitch bool, + resumeAttempt *resumeAttempt, +) (*lockedCancelInFlightGuard, error) { + if !requiresModelSwitch || resumeAttempt == nil { + return nil, nil + } + guard, err := s.lockResumeAttemptAdmission(ctx, sessionID, resumeAttempt) if err != nil { - if errors.Is(err, ErrResumeAttemptCancelled) { - s.cleanupCancelledResumeAttempt(resumeAttempt) - s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) - return nil, err - } - s.rollbackForegroundDispatchOnFailure(resumePromptCtx, taskID, sessionID, foregroundDispatch) + s.cleanupCancelledResumeAttempt(resumeAttempt) + s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch) return nil, err } - // beginForegroundDispatch atomically established foreground-generating - // ownership before any cleanup/provider event could interleave. Publish that - // flip now that session admission succeeded; claimed RUNNING prompts also need - // the broadcast because their earlier atomic claim made the same transition. - if foregroundDispatch.yieldedBeforeBegin || foregroundClaim != nil { - s.publishForegroundActivityChanged(resumePromptCtx, taskID, sessionID) - } - var releaseDispatchGuard func() - if rollback.dispatchGuardRelease != nil { - releaseDispatchGuard = rollback.dispatchGuardRelease - defer releaseDispatchGuard() - } - if options.requireNonterminalSession && releaseDispatchGuard == nil { - var guard *sync.Mutex - guard, releaseDispatchGuard = s.acquireCancelInFlightGuard(sessionID) - guard.Lock() - fresh, reloadErr := s.repo.GetTaskSession(resumePromptCtx, sessionID) - if reloadErr != nil { - guard.Unlock() - releaseDispatchGuard() - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, reloadErr - } - if fresh == nil || isTerminalSessionState(fresh.State) { - guard.Unlock() - releaseDispatchGuard() - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, newWorkflowAutoStartSessionTerminalizedError(fresh) - } - var releaseOnce sync.Once - originalRelease := releaseDispatchGuard - releaseDispatchGuard = func() { - releaseOnce.Do(func() { - guard.Unlock() - originalRelease() - }) - } - // The agentctl dispatch callback is the acceptance boundary. Releasing - // here keeps terminalization mutually exclusive with admission without - // holding the session guard for the rest of the (potentially long) turn. - defer releaseDispatchGuard() - } - if resumeAttempt != nil { - if releaseDispatchGuard == nil { - guard, err := s.lockResumeAttemptAdmission(resumePromptCtx, sessionID, resumeAttempt) - if err != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, err - } - var releaseOnce sync.Once - releaseDispatchGuard = func() { - releaseOnce.Do(func() { - guard.release() - }) - } - defer releaseDispatchGuard() - } else if err := s.validateResumeAttempt(resumeAttempt); err != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, err - } - } + return guard, nil +} - // Only bounded-ack callers preserve request context; ordinary prompts can take minutes. - promptCtx := options.executorContext(resumePromptCtx) +// prepareModelSwitchGuard caches and reserves the replay identity for the turn, +// then acquires the model-switch admission guard, if one is needed, in one +// step. Both must happen before any model-switch attempt. +func (s *Service) prepareModelSwitchGuard( + ctx context.Context, + taskID, sessionID, prompt, model string, + planMode bool, + attachments []v1.MessageAttachment, + options promptTaskOptions, + session *models.TaskSession, + foregroundDispatch *foregroundDispatch, + resumeAttempt *resumeAttempt, +) (*lockedCancelInFlightGuard, error) { + s.rememberTurnPromptWithAccepted(sessionID, prompt, model, planMode, attachments, options.onAccepted) + requiresModelSwitch := modelSwitchRequired(session, model) + return s.acquireModelSwitchGuard(ctx, taskID, sessionID, foregroundDispatch, requiresModelSwitch, resumeAttempt) +} + +// admitResumeAttemptForPrompt resolves promptTask's resume-scoped context and +// validates the resume attempt both before and after handling ensureErr (the +// error from ensureSessionRunningWithAttempt), so a cancellation landing in +// either window is caught rather than only the one checked first. ensureErr, +// when non-nil, is classified via classifyEnsureSessionRunningFailureForPrompt. +func (s *Service) admitResumeAttemptForPrompt( + ctx context.Context, + taskID, sessionID string, + prompt, model string, + planMode bool, + attachments []v1.MessageAttachment, + dispatchOnly bool, + options promptTaskOptions, + foregroundClaim *foregroundClaim, + resumeAttempt *resumeAttempt, + ensureErr error, +) (context.Context, error) { + resumePromptCtx := ctx + if resumeAttempt != nil { + resumePromptCtx = cancellableResumeContext(resumeAttempt) + if err := s.failPromptOnResumeAttempt(resumePromptCtx, taskID, sessionID, foregroundClaim, resumeAttempt); err != nil { + return resumePromptCtx, err + } + } + if ensureErr != nil { + s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim) + return resumePromptCtx, s.classifyEnsureSessionRunningFailureForPrompt( + ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, options, ensureErr, + ) + } + if err := s.failPromptOnResumeAttempt(resumePromptCtx, taskID, sessionID, foregroundClaim, resumeAttempt); err != nil { + return resumePromptCtx, err + } + return resumePromptCtx, nil +} + +// acquireResumeAttemptDispatchGuard is promptTask's post-claim resume-attempt +// guard step. When existingReleaseDispatchGuard is nil (claimDispatchAndAcquireGuard +// did not already acquire one), it locks the resume attempt's admission guard +// and returns the new release func for the caller to defer; a non-nil +// existingReleaseDispatchGuard is already deferred by the caller, so this only +// re-validates the attempt and returns (nil, nil) on either success or a nil +// resumeAttempt. +func (s *Service) acquireResumeAttemptDispatchGuard( + ctx context.Context, + taskID, sessionID string, + foregroundDispatch *foregroundDispatch, + rollback promptClaimRollback, + options promptTaskOptions, + resumeAttempt *resumeAttempt, + existingReleaseDispatchGuard func(), +) (func(), error) { + if resumeAttempt == nil { + return nil, nil + } + if existingReleaseDispatchGuard != nil { + if err := s.validateResumeAttempt(resumeAttempt); err != nil { + return nil, s.failPromptDispatch(ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, err) + } + return nil, nil + } + guard, err := s.lockResumeAttemptAdmission(ctx, sessionID, resumeAttempt) + if err != nil { + return nil, s.failPromptDispatch(ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, err) + } + var releaseOnce sync.Once + return func() { + releaseOnce.Do(func() { + guard.release() + }) + }, nil +} + +// failPromptOnResumeAttempt returns the resume attempt's validation error, if +// any, after releasing the foreground claim and cleaning up the cancelled +// attempt. promptTask's early admission checks all share this cleanup. +func (s *Service) failPromptOnResumeAttempt( + ctx context.Context, + taskID, sessionID string, + foregroundClaim *foregroundClaim, + resumeAttempt *resumeAttempt, +) error { + if err := s.validateResumeAttempt(resumeAttempt); err != nil { + s.cleanupCancelledResumeAttempt(resumeAttempt) + s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim) + return err + } + return nil +} + +// failPromptDispatch cleans up a resume attempt that failed after dispatch +// admission began, rolling back both the foreground dispatch claim and the +// prompt claim under a bounded failure context, and returns err unchanged so +// callers can return it directly. +func (s *Service) failPromptDispatch( + ctx context.Context, + taskID, sessionID string, + foregroundDispatch *foregroundDispatch, + rollback promptClaimRollback, + options promptTaskOptions, + resumeAttempt *resumeAttempt, + err error, +) error { + s.cleanupCancelledResumeAttempt(resumeAttempt) + failureCtx, cancel := options.failureContext(ctx) + defer cancel() + s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) + s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) + return err +} + +// finishPromptExecutorDispatch is promptTask's tail: it turns +// executor.PromptWithDispatchCallback's outcome, plus whatever onDispatched +// recorded via dispatchOutcome, into the one PromptResult/error pair promptTask +// returns. +func (s *Service) finishPromptExecutorDispatch( + ctx context.Context, taskID, sessionID, prompt string, planMode, resumedForPrompt bool, + attachments []v1.MessageAttachment, rollback promptClaimRollback, options promptTaskOptions, + foregroundDispatch *foregroundDispatch, releaseDispatchGuard func(), + result *executor.PromptResult, execErr error, dispatchOutcome *promptDispatchOutcome, + resumeAttempt *resumeAttempt, +) (*PromptResult, error) { + dispatchAccepted, publicationErr := dispatchOutcome.snapshot() + if execErr != nil { + // Missing-execution recovery reacquires the cancel guard while it resets + // the session. Release dispatch admission before entering that path. + if releaseDispatchGuard != nil { + releaseDispatchGuard() + } + if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { + s.cleanupCancelledResumeAttempt(resumeAttempt) + failureCtx, cancel := options.failureContext(ctx) + defer cancel() + s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) + s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) + return nil, resumeErr + } + return s.finishPromptDispatchFailure( + ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, + rollback, options, execErr, foregroundDispatch, dispatchAccepted, publicationErr, + ) + } + if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { + s.cleanupCancelledResumeAttempt(resumeAttempt) + failureCtx, cancel := options.failureContext(ctx) + defer cancel() + s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) + s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) + return nil, resumeErr + } + if publicationErr != nil { + return nil, &acceptedPromptDispatchError{err: publicationErr} + } + return &PromptResult{StopReason: result.StopReason, AgentMessage: result.AgentMessage, TurnID: rollback.turnID}, nil +} + +// validateAndRunDispatchBoundary resolves promptTask's queued-dispatch +// identity check and the caller's beforeDispatch hook — the two checks that +// must both pass immediately before the executor call — rolling back the +// foreground dispatch and prompt claim on either failure. Only bounded-ack +// callers preserve request context past this point; ordinary prompts can take +// minutes, so the executor context is resolved here and threaded back out. +func (s *Service) validateAndRunDispatchBoundary( + ctx context.Context, taskID, sessionID, prompt string, planMode, resumedForPrompt bool, + attachments []v1.MessageAttachment, session *models.TaskSession, rollback promptClaimRollback, + options promptTaskOptions, foregroundDispatch *foregroundDispatch, runBeforeDispatch func() error, +) (promptCtx context.Context, resolvedSession *models.TaskSession, earlyResult *PromptResult, err error) { + promptCtx = options.executorContext(ctx) session, identityValidationErr := s.validateQueuedPromptDispatch( promptCtx, rollback.sessionIdentity, taskID, sessionID, session, ) if identityValidationErr != nil { - return s.finishPromptDispatchFailure( - resumePromptCtx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, + result, err := s.finishPromptDispatchFailure( + ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, rollback, options, identityValidationErr, foregroundDispatch, false, nil, ) + return promptCtx, nil, result, err } if boundaryErr := runBeforeDispatch(); boundaryErr != nil { - failureCtx, cancel := options.failureContext(resumePromptCtx) + failureCtx, cancel := options.failureContext(ctx) defer cancel() s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, boundaryErr + return promptCtx, nil, nil, boundaryErr } + return promptCtx, session, nil, nil +} + +// preparePromptDispatchCallback binds the turn, arms the interactive-prompt +// attempt evidence, and builds the onDispatched callback executor. +// PromptWithDispatchCallback invokes once agentctl accepts the prompt — +// wrapping in the caller's afterDispatch hook and the nonterminal dispatch +// guard release, innermost first, when either applies. +func (s *Service) preparePromptDispatchCallback( + promptCtx context.Context, taskID, sessionID string, session *models.TaskSession, + rollback promptClaimRollback, options promptTaskOptions, foregroundDispatch *foregroundDispatch, + releaseDispatchGuard func(), +) (onDispatched func(), dispatchOutcome *promptDispatchOutcome) { s.bindPromptTurnID(promptCtx, session, rollback.turnID) s.beginInteractivePromptAttempt( promptCtx, @@ -5582,10 +5871,10 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom session.AgentExecutionID, s.isDynamicPromptSession(session), ) - dispatchOutcome := &promptDispatchOutcome{} + dispatchOutcome = &promptDispatchOutcome{} dispatchOutcome.turnID = rollback.turnID dispatchOutcome.onAccepted = options.onAccepted - onDispatched := s.promptDispatchCallbackForIdentity( + onDispatched = s.promptDispatchCallbackForIdentity( promptCtx, taskID, sessionID, rollback.sessionIdentity, session.AgentExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome, ) @@ -5609,42 +5898,7 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom releaseDispatchGuard() } } - result, err := s.executor.PromptWithDispatchCallback( - promptCtx, taskID, sessionID, effectivePrompt, attachments, dispatchOnly, - onDispatched, session, - ) - dispatchAccepted, publicationErr := dispatchOutcome.snapshot() - if err != nil { - // Missing-execution recovery reacquires the cancel guard while it resets - // the session. Release dispatch admission before entering that path. - if releaseDispatchGuard != nil { - releaseDispatchGuard() - } - if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, resumeErr - } - return s.finishPromptDispatchFailure( - resumePromptCtx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, - rollback, options, err, foregroundDispatch, dispatchAccepted, publicationErr, - ) - } - if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil { - s.cleanupCancelledResumeAttempt(resumeAttempt) - failureCtx, cancel := options.failureContext(resumePromptCtx) - defer cancel() - s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) - s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) - return nil, resumeErr - } - if publicationErr != nil { - return nil, &acceptedPromptDispatchError{err: publicationErr} - } - return &PromptResult{StopReason: result.StopReason, AgentMessage: result.AgentMessage, TurnID: rollback.turnID}, nil + return onDispatched, dispatchOutcome } func (s *Service) finishPromptDispatchFailure( @@ -5691,6 +5945,30 @@ func (s *Service) reloadPromptSession( return reloaded } +// beginForegroundDispatchForPrompt reloads the session after ensureSessionRunning +// (so executor.Prompt uses the freshly-persisted AgentExecutionID rather than a +// pointer staled by a concurrent write), re-applies config/plan-mode prompt +// transforms against the reloaded session, and begins the foreground dispatch. +// It releases the foreground claim itself on the "already in flight" failure, +// mirroring what promptTask's own failure paths do elsewhere. +func (s *Service) beginForegroundDispatchForPrompt( + ctx context.Context, + taskID, sessionID, prompt string, + planMode bool, + session *models.TaskSession, + foregroundClaim *foregroundClaim, +) (*models.TaskSession, string, *foregroundDispatch, error) { + session = s.reloadPromptSession(ctx, sessionID, session) + effectivePrompt := s.effectivePromptForSession(sessionID, prompt, planMode, session) + activityExecutionID, _ := s.agentManager.GetExecutionIDForSession(ctx, sessionID) + foregroundDispatch := s.beginForegroundDispatch(sessionID, foregroundClaim, activityExecutionID) + if foregroundDispatch == nil { + s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim) + return nil, "", nil, fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress) + } + return session, effectivePrompt, foregroundDispatch, nil +} + func (s *Service) bindPromptTurnID(ctx context.Context, session *models.TaskSession, turnID string) { setter, ok := s.agentManager.(executor.PromptTurnIDSetter) if turnID == "" || session == nil || !ok { @@ -5714,6 +5992,18 @@ func (s *Service) validatePromptTaskStart(sessionID string) error { return nil } +// validatePromptTaskPreconditions combines promptTask's two top-of-function +// checks: the ordinary session/reset-in-progress validation, and — for a +// compound resume passing its own cancellable attempt context — an early +// ownership check so a lost race surfaces its typed error before any +// repository read can observe the cancellation as a generic context error. +func (s *Service) validatePromptTaskPreconditions(sessionID string, resumeAttempt *resumeAttempt) error { + if err := s.validatePromptTaskStart(sessionID); err != nil { + return err + } + return s.validateResumeAttempt(resumeAttempt) +} + func (s *Service) logPromptTaskCall( taskID, sessionID, prompt, model string, planMode bool, @@ -5744,6 +6034,152 @@ func (s *Service) loadPromptableSession(ctx context.Context, taskID, sessionID s return session, nil } +// prepareSessionAndForegroundClaimForPrompt resolves promptTask's session (the +// nonterminal-reload variant when options.requireNonterminalSession) and takes +// its foreground claim, as one step so promptTask itself only has one error to +// check. +func (s *Service) prepareSessionAndForegroundClaimForPrompt( + ctx context.Context, taskID, sessionID string, options promptTaskOptions, +) (*models.TaskSession, *foregroundClaim, error) { + session, err := s.loadPromptableSession(ctx, taskID, sessionID) + if err != nil { + return nil, nil, err + } + if options.requireNonterminalSession { + session, err = s.loadNonterminalWorkflowAutoStartSession(ctx, sessionID) + if err != nil { + return nil, nil, err + } + } + foregroundClaim, err := s.claimForegroundForPrompt(taskID, sessionID, session) + if err != nil { + return nil, nil, err + } + return session, foregroundClaim, nil +} + +// claimAndGuardDispatch combines claimDispatchAndAcquireGuard with the +// resume-attempt dispatch guard step that follows it, so promptTask sees one +// error check and one guard-release defer instead of two of each. On failure +// it releases whatever guard claimDispatchAndAcquireGuard had already +// acquired before returning, mirroring the release the caller's own deferred +// call would otherwise have performed. +func (s *Service) claimAndGuardDispatch( + ctx context.Context, + taskID, sessionID string, + foregroundClaim *foregroundClaim, + options promptTaskOptions, + foregroundDispatch *foregroundDispatch, + resumeAttempt *resumeAttempt, +) (session *models.TaskSession, rollback promptClaimRollback, releaseDispatchGuard func(), err error) { + session, rollback, releaseDispatchGuard, err = s.claimDispatchAndAcquireGuard( + ctx, taskID, sessionID, foregroundClaim, options, foregroundDispatch, resumeAttempt, + ) + if err != nil { + return nil, promptClaimRollback{}, nil, err + } + newDispatchGuard, err := s.acquireResumeAttemptDispatchGuard( + ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, releaseDispatchGuard, + ) + if err != nil { + if releaseDispatchGuard != nil { + releaseDispatchGuard() + } + return nil, promptClaimRollback{}, nil, err + } + if newDispatchGuard != nil { + releaseDispatchGuard = newDispatchGuard + } + return session, rollback, releaseDispatchGuard, nil +} + +// claimDispatchAndAcquireGuard performs promptTask's admission claim +// (claimPromptDispatch), publishes the foreground-activity flip a claimed +// RUNNING prompt or the earlier beginForegroundDispatch may have caused, and +// resolves the nonterminal dispatch guard — the three steps that happen +// together between "foreground dispatch begun" and "ready to call the +// executor". +func (s *Service) claimDispatchAndAcquireGuard( + ctx context.Context, taskID, sessionID string, foregroundClaim *foregroundClaim, + options promptTaskOptions, foregroundDispatch *foregroundDispatch, resumeAttempt *resumeAttempt, +) (*models.TaskSession, promptClaimRollback, func(), error) { + session, rollback, err := s.claimPromptDispatchWithResumeAttempt( + ctx, taskID, sessionID, options.claimEntryID, options.lifecyclePrompt, + options.reserveTurnUntilDispatch, options.promptDispatchRecovery, + options.afterClaim, foregroundClaim, options.expectedCurrentTurnID, + options.requireNonterminalSession, resumeAttempt, options.expectedSessionIdentity, + ) + if err != nil { + if errors.Is(err, ErrResumeAttemptCancelled) { + s.cleanupCancelledResumeAttempt(resumeAttempt) + s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim) + return nil, promptClaimRollback{}, nil, err + } + s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch) + return nil, promptClaimRollback{}, nil, err + } + if foregroundDispatch.yieldedBeforeBegin || foregroundClaim != nil { + s.publishForegroundActivityChanged(ctx, taskID, sessionID) + } + releaseDispatchGuard, err := s.acquireNonterminalDispatchGuardForPrompt( + ctx, taskID, sessionID, options, foregroundDispatch, rollback, rollback.dispatchGuardRelease, + ) + if err != nil { + return nil, promptClaimRollback{}, nil, err + } + return session, rollback, releaseDispatchGuard, nil +} + +// acquireNonterminalDispatchGuardForPrompt takes the cancel-in-flight guard for +// a requireNonterminalSession prompt that did not already claim one via +// rollback.dispatchGuardRelease (existingRelease non-nil short-circuits, since +// one turn only ever needs one guard), and reloads the session under it to +// catch a concurrent terminalization race between claimPromptDispatch and +// here. Returns existingRelease unchanged when no new guard was needed. +func (s *Service) acquireNonterminalDispatchGuardForPrompt( + ctx context.Context, taskID, sessionID string, options promptTaskOptions, + foregroundDispatch *foregroundDispatch, rollback promptClaimRollback, existingRelease func(), +) (func(), error) { + if !options.requireNonterminalSession || existingRelease != nil { + return existingRelease, nil + } + guard, release := s.acquireCancelInFlightGuard(sessionID) + guard.Lock() + fresh, reloadErr := s.repo.GetTaskSession(ctx, sessionID) + if reloadErr != nil { + guard.Unlock() + release() + s.rollbackPromptDispatchOnGuardFailure(ctx, taskID, sessionID, options, foregroundDispatch, rollback) + return nil, reloadErr + } + if fresh == nil || isTerminalSessionState(fresh.State) { + guard.Unlock() + release() + s.rollbackPromptDispatchOnGuardFailure(ctx, taskID, sessionID, options, foregroundDispatch, rollback) + return nil, newWorkflowAutoStartSessionTerminalizedError(fresh) + } + var releaseOnce sync.Once + return func() { + releaseOnce.Do(func() { + guard.Unlock() + release() + }) + }, nil +} + +// rollbackPromptDispatchOnGuardFailure unwinds the foreground dispatch and +// prompt claim taken earlier in promptTask when the nonterminal dispatch guard +// itself fails to admit the turn. +func (s *Service) rollbackPromptDispatchOnGuardFailure( + ctx context.Context, taskID, sessionID string, options promptTaskOptions, + foregroundDispatch *foregroundDispatch, rollback promptClaimRollback, +) { + failureCtx, cancel := options.failureContext(ctx) + defer cancel() + s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch) + s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback) +} + // loadNonterminalWorkflowAutoStartSession reloads a session under the shared // cancellation guard before auto-start can trigger a lazy ACP resume. Manual // prompts intentionally do not use this path and can still resume terminal @@ -5881,6 +6317,32 @@ func (s *Service) promptDispatchCallbackForIdentity( } } +// attemptModelSwitchForPrompt runs promptTask's pre-dispatch model-switch +// handling: arming the initial-prompt-attempt evidence and the caller's +// beforeDispatch hook when a switch is required, then delegating to +// trySwitchModelForPrompt. handled reports whether the caller already has its +// full response — a dispatched switch, a switch failure, or a beforeDispatch +// failure — in which case promptTask returns (result, err) directly without +// proceeding to ordinary claim/dispatch. +func (s *Service) attemptModelSwitchForPrompt( + ctx context.Context, taskID, sessionID, model, effectivePrompt string, session *models.TaskSession, + foregroundDispatch *foregroundDispatch, runBeforeDispatch func() error, +) (result *PromptResult, handled bool, err error) { + if modelSwitchRequired(session, model) { + s.beginInitialPromptAttempt(sessionID, s.isDynamicPromptSession(session)) + if err := runBeforeDispatch(); err != nil { + s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch) + s.clearPromptAttemptEvidence(sessionID, "", 0) + return nil, true, err + } + } + result, handled, switchErr := s.trySwitchModelForPrompt(ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch) + if handled && switchErr != nil { + s.clearPromptAttemptEvidence(sessionID, "", 0) + } + return result, handled, switchErr +} + // trySwitchModelForPrompt keeps foreground admission consistent when a model // switch either dispatches the prompt itself or fails before reaching the agent. func (s *Service) trySwitchModelForPrompt(ctx context.Context, taskID, sessionID, model, prompt string, session *models.TaskSession, dispatch *foregroundDispatch) (*PromptResult, bool, error) { diff --git a/apps/backend/internal/orchestrator/task_operations_resume_cancellation_test.go b/apps/backend/internal/orchestrator/task_operations_resume_cancellation_test.go index 2ee201ce67..c13839e19c 100644 --- a/apps/backend/internal/orchestrator/task_operations_resume_cancellation_test.go +++ b/apps/backend/internal/orchestrator/task_operations_resume_cancellation_test.go @@ -223,6 +223,7 @@ func TestResumeTaskSessionAndPrompt_CancelAtContinuationBarrierDoesNotDispatchPr false, nil, false, + launchOriginManual, promptTaskOptions{resumeAttempt: attempt}, ) return promptErr @@ -312,6 +313,7 @@ func TestPromptTask_ResumeAttemptKeepsCancellationOutOfAcceptanceCallback(t *tes false, nil, false, + launchOriginManual, promptTaskOptions{ resumeAttempt: attempt, afterDispatch: func() error { diff --git a/apps/backend/internal/orchestrator/task_operations_test.go b/apps/backend/internal/orchestrator/task_operations_test.go index ea9c784606..e6cd1af6fa 100644 --- a/apps/backend/internal/orchestrator/task_operations_test.go +++ b/apps/backend/internal/orchestrator/task_operations_test.go @@ -7335,7 +7335,7 @@ func TestEnsureSessionRunning_OfficeWithoutRuntimeEnvFailsClosed(t *testing.T) { t.Fatalf("failed to reload session: %v", err) } - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) require.Error(t, err) assert.Contains(t, err.Error(), "office tasks must be restarted through Office") assert.False(t, startAgentProcessCalled) @@ -7363,7 +7363,7 @@ func TestEnsureSessionRunning_OfficeWaitingForInputFailsClosed(t *testing.T) { session, err := repo.GetTaskSession(ctx, "session1") require.NoError(t, err) - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) require.Error(t, err) assert.Contains(t, err.Error(), "office tasks must be resumed through Office") assert.False(t, launchCalled) @@ -7429,7 +7429,7 @@ func TestEnsureSessionRunning_WaitingForInputUsesResumePath(t *testing.T) { } // Should fail because there is no executor running record (resume path) - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) if err == nil { t.Fatal("expected error for WAITING_FOR_INPUT session without executor record") } @@ -7460,7 +7460,7 @@ func TestEnsureSessionRunning_CreatedWithoutExecutionUsesResumePath(t *testing.T // AgentExecutionID is empty → should NOT take prepared workspace path // Should fail with "not resumable" because no executor running record - err = svc.ensureSessionRunning(ctx, "session1", session) + err = svc.ensureSessionRunning(ctx, "session1", session, launchOriginManual) if err == nil { t.Fatal("expected error for CREATED session without executor record") } diff --git a/apps/backend/internal/orchestrator/watcher_dispatch.go b/apps/backend/internal/orchestrator/watcher_dispatch.go index f99a703f89..308b515e13 100644 --- a/apps/backend/internal/orchestrator/watcher_dispatch.go +++ b/apps/backend/internal/orchestrator/watcher_dispatch.go @@ -416,6 +416,12 @@ func (c *WatcherDispatchCoordinator) Dispatch(ctx context.Context, src WatcherSo params := src.AutoStartParams(evt) if err := c.startTask.Start(ctx, task.ID, stepID, task.Description, params); err != nil { + if errors.Is(err, ErrCeilingLaunchDeferred) { + c.logger.Info("watcher dispatch: auto-start deferred by session ceiling; will replay once capacity frees up", + zap.String("source", src.Name()), + zap.String("task_id", task.ID)) + return + } c.logger.Error("watcher dispatch: auto-start failed", zap.String("source", src.Name()), zap.String("task_id", task.ID), diff --git a/apps/backend/internal/orchestrator/workflow_store.go b/apps/backend/internal/orchestrator/workflow_store.go index 8839fea9e2..5be555924e 100644 --- a/apps/backend/internal/orchestrator/workflow_store.go +++ b/apps/backend/internal/orchestrator/workflow_store.go @@ -335,7 +335,7 @@ func (s *workflowStore) markDeferredMoveAppliedUnfenced(ctx context.Context, tas if err := markDeferredMoveApplied(task, moveID); err != nil { return err } - if err := s.repo.UpdateTask(ctx, task); err != nil { + if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { return fmt.Errorf("persist deferred move identity: %w", err) } return nil @@ -611,7 +611,7 @@ func markDeferredMoveApplied(task *models.Task, moveID string) error { func (s *workflowStore) updateTransitionTask(ctx context.Context, task *models.Task, fromStepID string, targetStep *wfmodels.WorkflowStep) error { if targetStep == nil { - return s.repo.UpdateTask(ctx, task) + return s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task) } admissionRepo, ok := s.repo.(workflowMoveAdmissionRepository) if !ok { diff --git a/apps/backend/internal/plugins/host_data_mappers.go b/apps/backend/internal/plugins/host_data_mappers.go index ac90e647eb..eef04b392c 100644 --- a/apps/backend/internal/plugins/host_data_mappers.go +++ b/apps/backend/internal/plugins/host_data_mappers.go @@ -123,7 +123,7 @@ func taskModelToDTO(t *taskmodels.Task) pluginsdk.Task { Identifier: t.Identifier, IsEphemeral: t.IsEphemeral, Repositories: repos, - Metadata: t.Metadata, + Metadata: taskmodels.PublicTaskMetadata(t.Metadata), ArchivedAt: timePtrToRFC3339(t.ArchivedAt), WorkflowStepID: t.WorkflowStepID, diff --git a/apps/backend/internal/plugins/host_data_task_fields_test.go b/apps/backend/internal/plugins/host_data_task_fields_test.go index 453ebe9573..5658a62958 100644 --- a/apps/backend/internal/plugins/host_data_task_fields_test.go +++ b/apps/backend/internal/plugins/host_data_task_fields_test.go @@ -58,6 +58,33 @@ func TestTaskModelToDTOLeavesArchivedAtNilForALiveTask(t *testing.T) { require.Nil(t, dto.QueuedAt) } +// The session-ceiling replay payload carries the launch-scoped environment +// map, the composed prompt, its attachments and entity references. A plugin +// or agent-authored canvas is a lower-trust surface than the host itself, so +// it must never see that payload even though it is free to see that a +// deferral is pending, of what kind, and why. +func TestTaskModelToDTORedactsCeilingLaunchPayload(t *testing.T) { + task := &taskmodels.Task{ + ID: "t1", + Metadata: map[string]interface{}{ + taskmodels.MetaKeyDeferredLaunch: map[string]interface{}{ + taskmodels.CeilingLaunchPayloadKey: map[string]interface{}{ + "prompt": "do the secret thing", + }, + taskmodels.CeilingReasonCodeKey: "ceiling_reached", + }, + }, + } + + dto := taskModelToDTO(task) + + deferred, ok := dto.Metadata[taskmodels.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "the discriminators must still be visible") + require.Equal(t, "ceiling_reached", deferred[taskmodels.CeilingReasonCodeKey]) + _, leaked := deferred[taskmodels.CeilingLaunchPayloadKey] + require.False(t, leaked, "ceiling_launch_payload must be redacted from the plugin host-data path") +} + // Review and check state ride along because kandev's PR watcher already syncs // them; a plugin asking "what can merge" would otherwise re-query the forge // once per pull request. diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index 08cc123f9a..380315abfc 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -92,6 +92,17 @@ func (m *mockRepository) UpdateTask(ctx context.Context, task *models.Task) erro func (m *mockRepository) UpdateTaskWithExplicitPosition(ctx context.Context, task *models.Task) error { return nil } +func (m *mockRepository) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + return nil +} +func (m *mockRepository) GetTaskDeferredLaunch(ctx context.Context, taskID string) (map[string]interface{}, interface{}, error) { + return nil, nil, nil +} +func (m *mockRepository) SetTaskDeferredLaunchIfUnchanged( + ctx context.Context, taskID string, prior interface{}, value map[string]interface{}, +) (bool, bool, error) { + return false, false, nil +} func (m *mockRepository) DeleteTask(ctx context.Context, id string) error { return nil } diff --git a/apps/backend/internal/task/handlers/task_http_handlers_external_id_test.go b/apps/backend/internal/task/handlers/task_http_handlers_external_id_test.go index 7a8f1e6fe4..afa5f1c455 100644 --- a/apps/backend/internal/task/handlers/task_http_handlers_external_id_test.go +++ b/apps/backend/internal/task/handlers/task_http_handlers_external_id_test.go @@ -138,6 +138,10 @@ func (r *idempotentCreateTaskRepo) UpdateTaskWithExplicitPosition(ctx context.Co return r.UpdateTask(ctx, task) } +func (r *idempotentCreateTaskRepo) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + return r.UpdateTask(ctx, task) +} + func (r *idempotentCreateTaskRepo) GetTaskByExternalID(_ context.Context, workspaceID, externalID string) (*models.Task, error) { r.mu.Lock() defer r.mu.Unlock() diff --git a/apps/backend/internal/task/handlers/task_port_forwarding_test.go b/apps/backend/internal/task/handlers/task_port_forwarding_test.go index 0e639d2d4a..8863438e8d 100644 --- a/apps/backend/internal/task/handlers/task_port_forwarding_test.go +++ b/apps/backend/internal/task/handlers/task_port_forwarding_test.go @@ -32,7 +32,7 @@ func (r *portForwardingHandlerRepo) GetTask(_ context.Context, _ string) (*model return r.task, nil } -func (r *portForwardingHandlerRepo) UpdateTask(_ context.Context, task *models.Task) error { +func (r *portForwardingHandlerRepo) UpdateTaskPreservingDeferredLaunch(_ context.Context, task *models.Task) error { r.updatedTask = task r.updateCalls++ return nil diff --git a/apps/backend/internal/task/models/canonical_json.go b/apps/backend/internal/task/models/canonical_json.go new file mode 100644 index 0000000000..dfe40e76e6 --- /dev/null +++ b/apps/backend/internal/task/models/canonical_json.go @@ -0,0 +1,44 @@ +package models + +import ( + "bytes" + "encoding/json" +) + +// CanonicalJSON returns a stable serialization of raw: object keys in sorted +// order, insignificant whitespace removed, and number literals preserved exactly +// as written. +// +// Preserving the literals is the point. The obvious implementation — unmarshal +// into interface{} and re-marshal — turns every number into a float64 and then +// back into JSON, so a Unix-second timestamp round-trips as 1.7576064e+09. Any +// comparison built on that would stop matching the stored bytes, which for a +// compare-and-set means a writer that can never win. json.Decoder.UseNumber +// keeps the literal, and json.Number re-marshals as itself. +// +// Empty, whitespace-only and JSON-null input all canonicalize to nil, so an +// absent value and an explicit null are one value. +func CanonicalJSON(raw []byte) ([]byte, error) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || string(trimmed) == "null" { + return nil, nil + } + decoder := json.NewDecoder(bytes.NewReader(trimmed)) + decoder.UseNumber() + var value interface{} + if err := decoder.Decode(&value); err != nil { + return nil, err + } + return json.Marshal(value) +} + +// CanonicalJSONOf canonicalizes a Go value by marshalling it first. It is the +// producer-side counterpart of CanonicalJSON: both sides of a comparison must be +// canonicalized through this package so key order and whitespace cannot decide it. +func CanonicalJSONOf(value interface{}) ([]byte, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + return CanonicalJSON(raw) +} diff --git a/apps/backend/internal/task/models/canonical_json_test.go b/apps/backend/internal/task/models/canonical_json_test.go new file mode 100644 index 0000000000..0305286f09 --- /dev/null +++ b/apps/backend/internal/task/models/canonical_json_test.go @@ -0,0 +1,75 @@ +package models + +import ( + "encoding/json" + "testing" +) + +func TestCanonicalJSONSortsKeysAndStripsWhitespace(t *testing.T) { + got, err := CanonicalJSON([]byte(`{ "b" : 1, "a" : { "d": true, "c": "x" } }`)) + if err != nil { + t.Fatalf("CanonicalJSON: %v", err) + } + want := `{"a":{"c":"x","d":true},"b":1}` + if string(got) != want { + t.Fatalf("CanonicalJSON = %s, want %s", got, want) + } +} + +// TestCanonicalJSONPreservesNumberLiterals is the regression this helper exists +// for: a float64 round-trip renders a Unix-second timestamp in exponent form and +// silently breaks every comparison built on the result. +func TestCanonicalJSONPreservesNumberLiterals(t *testing.T) { + raw := []byte(`{"ceiling_queued_at":1757606400,"attempts":3,"ratio":0.5}`) + got, err := CanonicalJSON(raw) + if err != nil { + t.Fatalf("CanonicalJSON: %v", err) + } + want := `{"attempts":3,"ceiling_queued_at":1757606400,"ratio":0.5}` + if string(got) != want { + t.Fatalf("CanonicalJSON = %s, want %s", got, want) + } +} + +func TestCanonicalJSONTreatsAbsentEmptyAndNullAsOneValue(t *testing.T) { + for name, raw := range map[string][]byte{ + "nil": nil, + "empty": []byte(""), + "whitespace": []byte(" \n"), + "null": []byte("null"), + } { + t.Run(name, func(t *testing.T) { + got, err := CanonicalJSON(raw) + if err != nil { + t.Fatalf("CanonicalJSON: %v", err) + } + if got != nil { + t.Fatalf("CanonicalJSON(%q) = %s, want nil", raw, got) + } + }) + } +} + +func TestCanonicalJSONIsStableAcrossMarshalOrder(t *testing.T) { + first, err := CanonicalJSONOf(map[string]interface{}{"z": 1, "a": 2}) + if err != nil { + t.Fatalf("CanonicalJSONOf: %v", err) + } + encoded, err := json.Marshal(map[string]interface{}{"a": 2, "z": 1}) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + second, err := CanonicalJSON(encoded) + if err != nil { + t.Fatalf("CanonicalJSON: %v", err) + } + if string(first) != string(second) { + t.Fatalf("canonical forms differ: %s vs %s", first, second) + } +} + +func TestCanonicalJSONRejectsInvalidInput(t *testing.T) { + if _, err := CanonicalJSON([]byte(`{"a":`)); err == nil { + t.Fatal("CanonicalJSON accepted malformed JSON") + } +} diff --git a/apps/backend/internal/task/models/ceiling_equivalence.go b/apps/backend/internal/task/models/ceiling_equivalence.go new file mode 100644 index 0000000000..c6aad917c1 --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_equivalence.go @@ -0,0 +1,98 @@ +package models + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// CeilingDeferralsEquivalent reports whether two deferrals describe the same +// launch, which is what decides between discarding a duplicate refusal and +// surfacing a collision. +// +// The kind is compared first: two refusals of different kinds for one task are +// never the same launch, whatever their payloads hold. Only the nested replay +// payload is then compared — the record's bookkeeping (queue timestamp, surface +// stamps, reason code) describes the deferral rather than the launch. +// +// Comparison is over canonical JSON rather than Go values, because the payload +// carries slices, maps and pointers for which Go's == is either a compile error or +// a pointer comparison. Both sides are normalized at comparison time rather than at +// write time, so a record written by an earlier version is judged on the same terms +// as a fresh one. +func CeilingDeferralsEquivalent(a, b CeilingDeferral) (bool, error) { + if a.Kind != b.Kind { + return false, nil + } + left, err := canonicalComparablePayload(a.Payload) + if err != nil { + return false, fmt.Errorf("failed to canonicalize the stored launch payload: %w", err) + } + right, err := canonicalComparablePayload(b.Payload) + if err != nil { + return false, fmt.Errorf("failed to canonicalize the incoming launch payload: %w", err) + } + return bytes.Equal(left, right), nil +} + +// canonicalComparablePayload renders a payload to its comparison form: marshalled +// through JSON so Go types collapse to the wire shape, decoded with number literals +// preserved, then stripped of values that carry no information. +func canonicalComparablePayload(payload map[string]interface{}) ([]byte, error) { + encoded, err := json.Marshal(payload) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + var decoded interface{} + if err := decoder.Decode(&decoded); err != nil { + return nil, err + } + normalized, present := normalizeComparableValue(decoded) + if !present { + return nil, nil + } + return json.Marshal(normalized) +} + +// normalizeComparableValue collapses absent, nil and empty to one value, reporting +// present=false for anything that carries no information. A nil slice, an empty +// slice, a nil map, an empty map, a nil pointer and an absent key all reduce to the +// same thing, because they all mean the caller supplied nothing. +// +// Slice elements are normalized but never dropped: order and arity are significant, +// since attachment order reaches the composed prompt, so removing an element would +// make two different launches compare equal. +func normalizeComparableValue(value interface{}) (interface{}, bool) { + switch typed := value.(type) { + case nil: + return nil, false + case map[string]interface{}: + normalized := make(map[string]interface{}, len(typed)) + for key, nested := range typed { + if cleaned, present := normalizeComparableValue(nested); present { + normalized[key] = cleaned + } + } + if len(normalized) == 0 { + return nil, false + } + return normalized, true + case []interface{}: + if len(typed) == 0 { + return nil, false + } + normalized := make([]interface{}, len(typed)) + for i, nested := range typed { + cleaned, present := normalizeComparableValue(nested) + if !present { + cleaned = nil + } + normalized[i] = cleaned + } + return normalized, true + default: + return typed, true + } +} diff --git a/apps/backend/internal/task/models/ceiling_equivalence_test.go b/apps/backend/internal/task/models/ceiling_equivalence_test.go new file mode 100644 index 0000000000..68b46a5715 --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_equivalence_test.go @@ -0,0 +1,124 @@ +package models + +import "testing" + +func deferralWith(kind CeilingLaunchKind, payload map[string]interface{}) CeilingDeferral { + return CeilingDeferral{Kind: kind, Payload: payload} +} + +func mustEquivalent(t *testing.T, a, b CeilingDeferral, want bool, why string) { + t.Helper() + got, err := CeilingDeferralsEquivalent(a, b) + if err != nil { + t.Fatalf("CeilingDeferralsEquivalent: %v", err) + } + if got != want { + t.Fatalf("equivalent = %v, want %v: %s", got, want, why) + } +} + +// TestDifferentKindsAreNeverDuplicates: two refusals of different kinds for one +// task are different launches, whatever their payloads look like. +func TestDifferentKindsAreNeverDuplicates(t *testing.T) { + payload := map[string]interface{}{"session_id": "s1"} + mustEquivalent(t, + deferralWith(CeilingLaunchResume, payload), + deferralWith(CeilingLaunchPromptEnsure, payload), + false, "kind is compared first") +} + +// TestAbsentNilAndEmptyAreOneValue is rule (a). Both refusals came from a caller +// supplying nothing, so treating null and [] as different would raise a collision +// on the card for two launches that are the same launch. +func TestAbsentNilAndEmptyAreOneValue(t *testing.T) { + var nilSlice []interface{} + var nilMap map[string]interface{} + + base := map[string]interface{}{"session_id": "s1"} + variants := map[string]map[string]interface{}{ + "nil slice": {"session_id": "s1", "attachments": nilSlice}, + "empty slice": {"session_id": "s1", "attachments": []interface{}{}}, + "nil map": {"session_id": "s1", "env": nilMap}, + "empty map": {"session_id": "s1", "env": map[string]interface{}{}}, + "explicit nil": {"session_id": "s1", "route": nil}, + "empty nested": {"session_id": "s1", "route": map[string]interface{}{"headers": map[string]interface{}{}}}, + } + for name, payload := range variants { + t.Run(name, func(t *testing.T) { + mustEquivalent(t, + deferralWith(CeilingLaunchStart, base), + deferralWith(CeilingLaunchStart, payload), + true, name+" must equal an absent key") + }) + } +} + +// TestMapKeyOrderIsNotADifference is rule (b). +func TestMapKeyOrderIsNotADifference(t *testing.T) { + mustEquivalent(t, + deferralWith(CeilingLaunchStart, map[string]interface{}{ + "env": map[string]interface{}{"A": "1", "B": "2"}, "prompt": "p", + }), + deferralWith(CeilingLaunchStart, map[string]interface{}{ + "prompt": "p", "env": map[string]interface{}{"B": "2", "A": "1"}, + }), + true, "map key order is not a difference") +} + +// TestSliceOrderIsADifference is rule (c). Attachment order reaches the composed +// prompt, so two launches whose attachments differ only in order are not the same +// launch and the collision must surface rather than the second being discarded. +func TestSliceOrderIsADifference(t *testing.T) { + mustEquivalent(t, + deferralWith(CeilingLaunchStart, map[string]interface{}{"attachments": []interface{}{"a", "b"}}), + deferralWith(CeilingLaunchStart, map[string]interface{}{"attachments": []interface{}{"b", "a"}}), + false, "slice order is a difference") +} + +// TestOnlyPayloadFieldsAreCompared is rule (d): the record's bookkeeping does not +// describe the launch, so two refusals queued at different times with different +// surface-attempt counts are still the same launch. +func TestOnlyPayloadFieldsAreCompared(t *testing.T) { + payload := map[string]interface{}{"session_id": "s1", "prompt": "p"} + first := CeilingDeferral{ + Kind: CeilingLaunchStartCreated, Payload: payload, + ReasonCode: "ceiling", Origin: "automatic", + } + second := CeilingDeferral{ + Kind: CeilingLaunchStartCreated, Payload: payload, + ReasonCode: "ceiling_superseded", Origin: "manual", + } + mustEquivalent(t, first, second, true, "only the nested payload is compared") +} + +func TestDifferentPayloadValuesAreNotDuplicates(t *testing.T) { + mustEquivalent(t, + deferralWith(CeilingLaunchStartCreated, map[string]interface{}{"prompt": "first"}), + deferralWith(CeilingLaunchStartCreated, map[string]interface{}{"prompt": "second"}), + false, "a differing prompt is a differing launch") +} + +// TestNumberLiteralsSurviveComparison guards the same float64 round-trip that the +// compare-and-set token avoids: a large integer must not compare unequal to itself. +func TestNumberLiteralsSurviveComparison(t *testing.T) { + mustEquivalent(t, + deferralWith(CeilingLaunchStart, map[string]interface{}{"priority": int64(1757606400)}), + deferralWith(CeilingLaunchStart, map[string]interface{}{"priority": int64(1757606400)}), + true, "a large integer equals itself") + mustEquivalent(t, + deferralWith(CeilingLaunchStart, map[string]interface{}{"priority": int64(1757606400)}), + deferralWith(CeilingLaunchStart, map[string]interface{}{"priority": int64(1757606401)}), + false, "adjacent large integers are distinguishable") +} + +// TestNormalizationAppliesToBothSides: a record written before the payload gained a +// field compares on the same terms as a fresh one, because normalization happens at +// comparison time rather than at write time. +func TestNormalizationAppliesToBothSides(t *testing.T) { + stored := map[string]interface{}{"prompt": "p", "attachments": []interface{}{}} + fresh := map[string]interface{}{"prompt": "p"} + mustEquivalent(t, + deferralWith(CeilingLaunchStart, stored), + deferralWith(CeilingLaunchStart, fresh), + true, "an older record normalizes the same way") +} diff --git a/apps/backend/internal/task/models/ceiling_launch.go b/apps/backend/internal/task/models/ceiling_launch.go new file mode 100644 index 0000000000..75f06b60cd --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_launch.go @@ -0,0 +1,87 @@ +package models + +import "strings" + +// The deferred_launch record carries three independent, simultaneously-possible +// meanings discriminated by key rather than by position: a dependency-chain intent +// (DeferredLaunchStartWhenUnblockedKey), the WIP-overflow meaning (the record's +// presence with neither discriminator), and a session-ceiling deferral. A record may +// carry the chain intent and the ceiling deferral at once, so every predicate over +// this record tolerates both. +// +// Every key this card adds is prefixed ceiling_, and that prefix is load-bearing +// rather than cosmetic: it is how the ceiling's writes and the pre-existing WIP +// writes partition the shared record without either side keeping a list of the +// other's keys in sync. +const ( + // CeilingDeferredKey is the boolean discriminator marking a launch deferred + // because the instance was at its session ceiling. + CeilingDeferredKey = "ceiling_deferred" + + // CeilingQueuedAtKey records when the launch first entered the queue, as + // RFC 3339 UTC. It is a diagnostic: nothing orders by it. + CeilingQueuedAtKey = "ceiling_queued_at" + + // CeilingLaunchKindKey discriminates which launch shape the record replays. + CeilingLaunchKindKey = "ceiling_launch_kind" + + // CeilingLaunchPayloadKey nests every replay field. It is redacted from the + // public projection, because the payload carries the launch-scoped environment, + // the composed prompt and its attachments. + CeilingLaunchPayloadKey = "ceiling_launch_payload" + + // CeilingLaunchOriginKey stores the automatic/manual classification so a + // deferred automatic launch cannot be re-admitted later as a manual override. + CeilingLaunchOriginKey = "ceiling_launch_origin" + + // CeilingReasonCodeKey carries the reason code a human reads on the card. + CeilingReasonCodeKey = "ceiling_reason_code" + + // CeilingSurfaceWrittenAtKey is stamped only once the card note has been + // written successfully, so a failed note is retried rather than suppressed. + CeilingSurfaceWrittenAtKey = "ceiling_surface_written_at" + + // CeilingSurfaceAttemptCountKey bounds those retries. It lives on the record + // rather than in process memory so a restart cannot reset the bound. + CeilingSurfaceAttemptCountKey = "ceiling_surface_attempt_count" + + // CeilingPopulationAtRefusalKey records the population the admission + // controller saw at refusal time, so a later retry's card note (AC-49g) + // reproduces the same content rather than a freshly re-queried number. It + // is absent, never zero, when the population was unknown at refusal + // (AC-33a): an absent key is the "unknown" signal, not a rendered word. + CeilingPopulationAtRefusalKey = "ceiling_population_at_refusal" + + // CeilingValueAtRefusalKey records the ceiling itself at refusal time. It + // is always known, unlike the population. + CeilingValueAtRefusalKey = "ceiling_value_at_refusal" + + ceilingRecordKeyPrefix = "ceiling_" +) + +// IsCeilingRecordKey reports whether a key inside the deferred_launch record +// belongs to the session ceiling rather than to the pre-existing WIP meaning. +func IsCeilingRecordKey(key string) bool { + return strings.HasPrefix(key, ceilingRecordKeyPrefix) +} + +// HasCeilingDeferredIntent reports whether the task holds a launch deferred by the +// session ceiling. +func HasCeilingDeferredIntent(task *Task) bool { + record, ok := deferredLaunchRecord(task) + if !ok { + return false + } + flag, ok := record[CeilingDeferredKey].(bool) + return ok && flag +} + +// deferredLaunchRecord reads the shared record as an object. Every existing reader +// already treats a non-object value as no intent, so this reports the same. +func deferredLaunchRecord(task *Task) (map[string]interface{}, bool) { + if task == nil || task.Metadata == nil { + return nil, false + } + record, ok := task.Metadata[MetaKeyDeferredLaunch].(map[string]interface{}) + return record, ok +} diff --git a/apps/backend/internal/task/models/ceiling_launch_test.go b/apps/backend/internal/task/models/ceiling_launch_test.go new file mode 100644 index 0000000000..530c8babe6 --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_launch_test.go @@ -0,0 +1,117 @@ +package models + +import "testing" + +func ceilingTask(record map[string]interface{}) *Task { + return &Task{Metadata: map[string]interface{}{MetaKeyDeferredLaunch: record}} +} + +func TestIsCeilingRecordKeyPartitionsByPrefix(t *testing.T) { + for _, key := range []string{CeilingDeferredKey, CeilingQueuedAtKey, CeilingLaunchKindKey, + CeilingLaunchPayloadKey, CeilingLaunchOriginKey, CeilingReasonCodeKey, + CeilingSurfaceWrittenAtKey, CeilingSurfaceAttemptCountKey, "ceiling_added_later"} { + if !IsCeilingRecordKey(key) { + t.Errorf("IsCeilingRecordKey(%q) = false, want true", key) + } + } + for _, key := range []string{DeferredLaunchStartWhenUnblockedKey, DeferredLaunchUserIDKey, + DeferredLaunchRecordRecentUseKey, "added_later"} { + if IsCeilingRecordKey(key) { + t.Errorf("IsCeilingRecordKey(%q) = true, want false: it is not the ceiling's to touch", key) + } + } +} + +func TestHasCeilingDeferredIntent(t *testing.T) { + for _, tc := range []struct { + name string + task *Task + want bool + }{ + {name: "nil task", task: nil, want: false}, + {name: "no metadata", task: &Task{}, want: false}, + {name: "no record", task: &Task{Metadata: map[string]interface{}{}}, want: false}, + {name: "record is not an object", task: &Task{Metadata: map[string]interface{}{MetaKeyDeferredLaunch: "scalar"}}, want: false}, + {name: "wip record only", task: ceilingTask(map[string]interface{}{DeferredLaunchUserIDKey: "u1"}), want: false}, + {name: "flag false", task: ceilingTask(map[string]interface{}{CeilingDeferredKey: false}), want: false}, + {name: "flag true", task: ceilingTask(map[string]interface{}{CeilingDeferredKey: true}), want: true}, + {name: "alongside a chain intent", task: ceilingTask(map[string]interface{}{ + CeilingDeferredKey: true, + DeferredLaunchStartWhenUnblockedKey: true, + }), want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := HasCeilingDeferredIntent(tc.task); got != tc.want { + t.Errorf("HasCeilingDeferredIntent = %v, want %v", got, tc.want) + } + }) + } +} + +// An unrelated WIP admission must not delete a ceiling deferral. Left unguarded it +// drops a launch that is never retried and never logged. +func TestDropWIPDeferredLaunchLeavesACeilingDeferral(t *testing.T) { + task := ceilingTask(map[string]interface{}{CeilingDeferredKey: true, CeilingLaunchKindKey: "start"}) + DropWIPDeferredLaunch(task) + if _, present := task.Metadata[MetaKeyDeferredLaunch]; !present { + t.Fatal("DropWIPDeferredLaunch deleted a ceiling deferral") + } +} + +func TestDropWIPDeferredLaunchStillGuardsAChainIntentAndStillDropsAPlainWIPRecord(t *testing.T) { + chain := ceilingTask(map[string]interface{}{DeferredLaunchStartWhenUnblockedKey: true}) + DropWIPDeferredLaunch(chain) + if _, present := chain.Metadata[MetaKeyDeferredLaunch]; !present { + t.Fatal("DropWIPDeferredLaunch deleted a chain intent") + } + + wip := ceilingTask(map[string]interface{}{DeferredLaunchUserIDKey: "u1"}) + DropWIPDeferredLaunch(wip) + if _, present := wip.Metadata[MetaKeyDeferredLaunch]; present { + t.Fatal("DropWIPDeferredLaunch left a plain WIP record in place: existing behaviour changed") + } +} + +// The replay payload carries the launch-scoped environment, the composed prompt and +// its attachments, so it must not reach an API client. The discriminators and +// bookkeeping stay projected: a client can still see that a deferral is pending, of +// which kind, since when and why. +func TestPublicTaskMetadataRedactsTheCeilingPayloadButKeepsTheDiscriminators(t *testing.T) { + metadata := map[string]interface{}{ + MetaKeyDeferredLaunch: map[string]interface{}{ + DeferredLaunchUserIDKey: "u1", + DeferredLaunchRecordRecentUseKey: true, + CeilingDeferredKey: true, + CeilingLaunchKindKey: "start", + CeilingQueuedAtKey: "2026-09-11T00:00:00Z", + CeilingReasonCodeKey: "ceiling", + CeilingSurfaceWrittenAtKey: "2026-09-11T00:00:01Z", + CeilingSurfaceAttemptCountKey: 2, + CeilingLaunchPayloadKey: map[string]interface{}{ + "env": map[string]interface{}{"SECRET_TOKEN": "hunter2"}, + "prompt": "the composed prompt", + }, + }, + } + + public := PublicTaskMetadata(metadata) + deferred, ok := public[MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok { + t.Fatal("public projection lost the deferred_launch record") + } + if _, leaked := deferred[CeilingLaunchPayloadKey]; leaked { + t.Error("public projection leaked the ceiling launch payload") + } + for _, key := range []string{CeilingDeferredKey, CeilingLaunchKindKey, CeilingQueuedAtKey, + CeilingReasonCodeKey, CeilingSurfaceWrittenAtKey, CeilingSurfaceAttemptCountKey} { + if _, present := deferred[key]; !present { + t.Errorf("public projection dropped %q, which AC-22(a)'s observable depends on", key) + } + } + + // The projection is detached: redacting must not mutate the caller's map. + original, _ := metadata[MetaKeyDeferredLaunch].(map[string]interface{}) + if _, present := original[CeilingLaunchPayloadKey]; !present { + t.Error("PublicTaskMetadata mutated the source record instead of projecting a copy") + } +} diff --git a/apps/backend/internal/task/models/ceiling_record.go b/apps/backend/internal/task/models/ceiling_record.go new file mode 100644 index 0000000000..c3d8ccc1cf --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_record.go @@ -0,0 +1,198 @@ +package models + +import ( + "encoding/json" + "fmt" + "time" +) + +// CeilingLaunchKind names which launch shape a deferred record replays. The set is +// closed: a record naming anything outside it is unreplayable, and guessing would +// answer one kind of launch with another. +type CeilingLaunchKind string + +const ( + CeilingLaunchStart CeilingLaunchKind = "start" + CeilingLaunchStartCreated CeilingLaunchKind = "start_created" + CeilingLaunchPromptEnsure CeilingLaunchKind = "prompt_ensure" + CeilingLaunchWorkflowStepEnsure CeilingLaunchKind = "workflow_step_ensure" + CeilingLaunchQueueDrainEnsure CeilingLaunchKind = "queue_drain_ensure" + CeilingLaunchResume CeilingLaunchKind = "resume" + CeilingLaunchDynamicRelaunch CeilingLaunchKind = "dynamic_relaunch" +) + +// ceilingLaunchKinds is an explicit membership map rather than a switch with a +// default arm, so a kind added to the type without being classified here is absent +// from the set instead of silently admitted to it. +var ceilingLaunchKinds = map[CeilingLaunchKind]bool{ + CeilingLaunchStart: true, + CeilingLaunchStartCreated: true, + CeilingLaunchPromptEnsure: true, + CeilingLaunchWorkflowStepEnsure: true, + CeilingLaunchQueueDrainEnsure: true, + CeilingLaunchResume: true, + CeilingLaunchDynamicRelaunch: true, +} + +// AllCeilingLaunchKinds is the canonical list. Code that must cover every kind +// should range over this rather than hand-writing its own literal. +var AllCeilingLaunchKinds = []CeilingLaunchKind{ + CeilingLaunchStart, + CeilingLaunchStartCreated, + CeilingLaunchPromptEnsure, + CeilingLaunchWorkflowStepEnsure, + CeilingLaunchQueueDrainEnsure, + CeilingLaunchResume, + CeilingLaunchDynamicRelaunch, +} + +// IsCeilingLaunchKind reports membership of the closed set. +func IsCeilingLaunchKind(kind CeilingLaunchKind) bool { + return ceilingLaunchKinds[kind] +} + +// CeilingDeferral is the ceiling's half of the shared deferred_launch record. +type CeilingDeferral struct { + Kind CeilingLaunchKind + Payload map[string]interface{} + Origin string + ReasonCode string + QueuedAt time.Time + + // Population, PopulationKnown and Ceiling are the admission controller's + // reading at the moment of this refusal (or of the most recent reason-code + // change). They are not part of the launch being replayed, so they are not + // part of CeilingDeferralsEquivalent's comparison — only the AC-49 card + // carrier reads them. + Population int + PopulationKnown bool + Ceiling int +} + +// CeilingRecordKeys builds the ceiling keys for a deferral. It returns only the +// keys the ceiling owns, so a caller merges them into whatever the record already +// holds rather than replacing it. +func CeilingRecordKeys(deferral CeilingDeferral) map[string]interface{} { + keys := map[string]interface{}{ + CeilingDeferredKey: true, + CeilingLaunchKindKey: string(deferral.Kind), + CeilingLaunchOriginKey: deferral.Origin, + CeilingReasonCodeKey: deferral.ReasonCode, + CeilingQueuedAtKey: deferral.QueuedAt.UTC().Format(time.RFC3339), + CeilingLaunchPayloadKey: deferral.Payload, + CeilingValueAtRefusalKey: deferral.Ceiling, + } + if deferral.Payload == nil { + keys[CeilingLaunchPayloadKey] = map[string]interface{}{} + } + if deferral.PopulationKnown { + keys[CeilingPopulationAtRefusalKey] = deferral.Population + } + return keys +} + +// MergeCeilingRecord folds a deferral's keys into a task's existing +// deferred_launch value and reports the record to store. +// +// Where the existing value is an object the ceiling keys are added and every key +// it does not own is left alone, so a dependency-chain intent and a ceiling +// deferral coexist on one record. Where the existing value is present but is not +// an object — a scalar, an array, or null — no reader can interpret it as an +// intent, so this replaces that one key's value and reports the discarded value so +// the caller can record it; every other key in the task's metadata is untouched +// either way. +func MergeCeilingRecord( + existing interface{}, deferral CeilingDeferral, +) (record map[string]interface{}, discarded interface{}, replaced bool) { + ceilingKeys := CeilingRecordKeys(deferral) + + current, isObject := existing.(map[string]interface{}) + if existing == nil { + return ceilingKeys, nil, false + } + if !isObject { + return ceilingKeys, existing, true + } + + merged := make(map[string]interface{}, len(current)+len(ceilingKeys)) + for key, value := range current { + merged[key] = value + } + for key, value := range ceilingKeys { + merged[key] = value + } + return merged, nil, false +} + +// ReadCeilingDeferral extracts the ceiling half of a record. +// +// A record whose kind is absent or outside the closed set is reported as +// unreplayable rather than defaulted: the one kind an older reader understands is +// "start", and answering a resume with a fresh task launch is the specific outcome +// that must not happen. +func ReadCeilingDeferral(record map[string]interface{}) (CeilingDeferral, error) { + if record == nil { + return CeilingDeferral{}, fmt.Errorf("deferred launch record is absent") + } + if flag, _ := record[CeilingDeferredKey].(bool); !flag { + return CeilingDeferral{}, fmt.Errorf("record carries no ceiling deferral") + } + + rawKind, _ := record[CeilingLaunchKindKey].(string) + kind := CeilingLaunchKind(rawKind) + if !IsCeilingLaunchKind(kind) { + return CeilingDeferral{}, &UnreplayableCeilingRecordError{Kind: rawKind} + } + + deferral := CeilingDeferral{Kind: kind} + deferral.Payload, _ = record[CeilingLaunchPayloadKey].(map[string]interface{}) + if deferral.Payload == nil { + deferral.Payload = map[string]interface{}{} + } + deferral.Origin, _ = record[CeilingLaunchOriginKey].(string) + deferral.ReasonCode, _ = record[CeilingReasonCodeKey].(string) + if stamp, ok := record[CeilingQueuedAtKey].(string); ok { + if parsed, err := time.Parse(time.RFC3339, stamp); err == nil { + deferral.QueuedAt = parsed.UTC() + } + } + deferral.Ceiling = CeilingRecordInt(record[CeilingValueAtRefusalKey]) + if population, ok := record[CeilingPopulationAtRefusalKey]; ok { + deferral.Population = CeilingRecordInt(population) + deferral.PopulationKnown = true + } + return deferral, nil +} + +// CeilingRecordInt reads a number stored on a deferred_launch record +// regardless of which numeric representation decoded it: json.Number for a +// caller using UseNumber (the repository's own read path), float64 for +// encoding/json's ordinary interface{} decoding (test fixtures and any other +// caller). +func CeilingRecordInt(value interface{}) int { + switch v := value.(type) { + case json.Number: + n, _ := v.Int64() + return int(n) + case float64: + return int(v) + case int: + return v + default: + return 0 + } +} + +// UnreplayableCeilingRecordError reports a record whose launch kind is absent or +// unrecognised. It is distinguishable from an ordinary read failure because the +// disposition differs: the record is dropped rather than retried. +type UnreplayableCeilingRecordError struct { + Kind string +} + +func (e *UnreplayableCeilingRecordError) Error() string { + if e.Kind == "" { + return "ceiling launch kind is absent" + } + return fmt.Sprintf("ceiling launch kind %q is outside the known set", e.Kind) +} diff --git a/apps/backend/internal/task/models/ceiling_record_test.go b/apps/backend/internal/task/models/ceiling_record_test.go new file mode 100644 index 0000000000..79e7093bb4 --- /dev/null +++ b/apps/backend/internal/task/models/ceiling_record_test.go @@ -0,0 +1,208 @@ +package models + +import ( + "errors" + "testing" + "time" +) + +func sampleDeferral() CeilingDeferral { + return CeilingDeferral{ + Kind: CeilingLaunchStartCreated, + Payload: map[string]interface{}{"session_id": "s1", "prompt": "do the thing"}, + Origin: "automatic", + ReasonCode: "ceiling", + QueuedAt: time.Date(2026, 9, 12, 8, 30, 0, 0, time.UTC), + } +} + +// TestCeilingLaunchKindsAreAClosedSet ranges over the canonical list so a kind +// added to the type without being classified is caught here rather than defaulting +// into the set. +func TestCeilingLaunchKindsAreAClosedSet(t *testing.T) { + if len(AllCeilingLaunchKinds) != 7 { + t.Fatalf("AllCeilingLaunchKinds has %d entries, want 7", len(AllCeilingLaunchKinds)) + } + seen := map[CeilingLaunchKind]bool{} + for _, kind := range AllCeilingLaunchKinds { + if seen[kind] { + t.Fatalf("%q appears twice in AllCeilingLaunchKinds", kind) + } + seen[kind] = true + if !IsCeilingLaunchKind(kind) { + t.Errorf("IsCeilingLaunchKind(%q) = false, want true", kind) + } + } + for _, outside := range []CeilingLaunchKind{"", "start_task", "START", "relaunch"} { + if IsCeilingLaunchKind(outside) { + t.Errorf("IsCeilingLaunchKind(%q) = true, want false", outside) + } + } +} + +// TestCeilingRecordKeysNestEveryReplayFieldUnderOneKey pins the confidentiality +// boundary: replay fields live under a single nested key, never spread across the +// record's top level, because that key is what the public projection redacts. +func TestCeilingRecordKeysNestEveryReplayFieldUnderOneKey(t *testing.T) { + keys := CeilingRecordKeys(sampleDeferral()) + + payload, ok := keys[CeilingLaunchPayloadKey].(map[string]interface{}) + if !ok { + t.Fatalf("payload not nested under %s: %+v", CeilingLaunchPayloadKey, keys) + } + if payload["prompt"] != "do the thing" || payload["session_id"] != "s1" { + t.Fatalf("payload lost its replay fields: %+v", payload) + } + for _, leaked := range []string{"prompt", "session_id"} { + if _, present := keys[leaked]; present { + t.Errorf("replay field %q leaked to the record's top level", leaked) + } + } + if keys[CeilingDeferredKey] != true { + t.Errorf("discriminator not set: %+v", keys) + } + if keys[CeilingLaunchKindKey] != string(CeilingLaunchStartCreated) { + t.Errorf("launch kind = %v, want %q", keys[CeilingLaunchKindKey], CeilingLaunchStartCreated) + } + if keys[CeilingReasonCodeKey] != "ceiling" || keys[CeilingLaunchOriginKey] != "automatic" { + t.Errorf("reason code or origin missing: %+v", keys) + } + if got := keys[CeilingQueuedAtKey]; got != "2026-09-12T08:30:00Z" { + t.Errorf("ceiling_queued_at = %v, want RFC 3339 UTC", got) + } + for key := range keys { + if !IsCeilingRecordKey(key) { + t.Errorf("%q is not a ceiling-prefixed key; it would collide with the other writer", key) + } + } +} + +// TestMergeCeilingRecordLeavesKeysItDoesNotOwn is the merge-not-replace rule, and +// the chain intent surviving it is the observable that matters. +func TestMergeCeilingRecordLeavesKeysItDoesNotOwn(t *testing.T) { + existing := map[string]interface{}{ + DeferredLaunchStartWhenUnblockedKey: true, + DeferredLaunchUserIDKey: "user-9", + DeferredLaunchRecordRecentUseKey: true, + "some_future_key": "keep me", + } + + record, discarded, replaced := MergeCeilingRecord(existing, sampleDeferral()) + if replaced || discarded != nil { + t.Fatalf("merging into an object must not replace it: replaced=%v discarded=%v", replaced, discarded) + } + for key, want := range existing { + if record[key] != want { + t.Errorf("merge dropped %q: got %v want %v", key, record[key], want) + } + } + if record[CeilingDeferredKey] != true { + t.Errorf("merge did not add the ceiling keys: %+v", record) + } + if !HasStartWhenUnblockedIntent(&Task{Metadata: map[string]interface{}{MetaKeyDeferredLaunch: record}}) { + t.Error("HasStartWhenUnblockedIntent no longer reports the chain intent after a ceiling merge") + } + if !HasCeilingDeferredIntent(&Task{Metadata: map[string]interface{}{MetaKeyDeferredLaunch: record}}) { + t.Error("HasCeilingDeferredIntent does not report the merged deferral") + } +} + +// TestMergeCeilingRecordReplacesANonObjectValue covers the three reachable +// malformed shapes. No reader can read an intent out of any of them, so replacing +// that one key is correct, and the discarded value is reported so it stays +// recoverable. +func TestMergeCeilingRecordReplacesANonObjectValue(t *testing.T) { + for name, existing := range map[string]interface{}{ + "scalar": "nonsense", + "array": []interface{}{1, 2}, + "number": float64(7), + } { + t.Run(name, func(t *testing.T) { + record, discarded, replaced := MergeCeilingRecord(existing, sampleDeferral()) + if !replaced { + t.Fatalf("replaced = false for a %s value", name) + } + if discarded == nil { + t.Error("the discarded value was not reported, so the corruption is unrecoverable from the log") + } + if record[CeilingDeferredKey] != true { + t.Fatalf("replacement is not a well-formed ceiling record: %+v", record) + } + }) + } +} + +func TestMergeCeilingRecordCreatesTheRecordWhenAbsent(t *testing.T) { + record, discarded, replaced := MergeCeilingRecord(nil, sampleDeferral()) + if replaced || discarded != nil { + t.Fatalf("an absent value is not a replacement: replaced=%v discarded=%v", replaced, discarded) + } + if record[CeilingLaunchKindKey] != string(CeilingLaunchStartCreated) { + t.Fatalf("record not created: %+v", record) + } +} + +func TestReadCeilingDeferralRoundTrips(t *testing.T) { + want := sampleDeferral() + record, _, _ := MergeCeilingRecord(nil, want) + + got, err := ReadCeilingDeferral(record) + if err != nil { + t.Fatalf("ReadCeilingDeferral: %v", err) + } + if got.Kind != want.Kind || got.Origin != want.Origin || got.ReasonCode != want.ReasonCode { + t.Fatalf("round trip lost discriminators: %+v", got) + } + if !got.QueuedAt.Equal(want.QueuedAt) { + t.Fatalf("queued_at = %v, want %v", got.QueuedAt, want.QueuedAt) + } + if got.Payload["prompt"] != "do the thing" { + t.Fatalf("round trip lost the payload: %+v", got.Payload) + } +} + +// TestReadCeilingDeferralRefusesAnUnknownKind is the criterion's point: an absent +// or unrecognised kind must not fall back to "start", which is the only kind an +// older reader understands, because that answers a resume with a fresh launch. +func TestReadCeilingDeferralRefusesAnUnknownKind(t *testing.T) { + for name, kind := range map[string]interface{}{ + "absent": nil, + "unknown": "teleport", + "wrong case": "START", + "wrong type": 42, + "future kind": "start_v2", + } { + t.Run(name, func(t *testing.T) { + record := map[string]interface{}{ + CeilingDeferredKey: true, + CeilingLaunchPayloadKey: map[string]interface{}{"session_id": "s1"}, + } + if kind != nil { + record[CeilingLaunchKindKey] = kind + } + + got, err := ReadCeilingDeferral(record) + if err == nil { + t.Fatalf("ReadCeilingDeferral accepted kind %v, returning %+v", kind, got) + } + var unreplayable *UnreplayableCeilingRecordError + if !errors.As(err, &unreplayable) { + t.Fatalf("error %v is not distinguishable as unreplayable, so it cannot be dropped rather than retried", err) + } + if got.Kind != "" { + t.Fatalf("a rejected record still yielded kind %q", got.Kind) + } + }) + } +} + +func TestReadCeilingDeferralRejectsARecordWithoutTheDiscriminator(t *testing.T) { + if _, err := ReadCeilingDeferral(map[string]interface{}{ + DeferredLaunchStartWhenUnblockedKey: true, + }); err == nil { + t.Fatal("a WIP-only record was read as a ceiling deferral") + } + if _, err := ReadCeilingDeferral(nil); err == nil { + t.Fatal("an absent record was read as a ceiling deferral") + } +} diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index ff3692a672..9addcca9ed 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -1195,7 +1195,7 @@ func HasStartWhenUnblockedIntent(task *Task) bool { // every dependency chain, because admission happens at create time for any task // entering a step with room. func DropWIPDeferredLaunch(task *Task) { - if task == nil || task.Metadata == nil || HasStartWhenUnblockedIntent(task) { + if task == nil || task.Metadata == nil || HasStartWhenUnblockedIntent(task) || HasCeilingDeferredIntent(task) { return } delete(task.Metadata, MetaKeyDeferredLaunch) diff --git a/apps/backend/internal/task/models/public_metadata.go b/apps/backend/internal/task/models/public_metadata.go index afa34b992f..b74d8bfde0 100644 --- a/apps/backend/internal/task/models/public_metadata.go +++ b/apps/backend/internal/task/models/public_metadata.go @@ -8,6 +8,12 @@ import "maps" // transient workflow-move marker carries encoded one-shot instructions and an // internal move ID; it is server-owned lifecycle state and must never reach a // public event or DTO. +// +// The session-ceiling replay payload is redacted for a stronger reason than +// tidiness: it carries the launch-scoped environment map, the composed prompt, its +// attachments and entity references. The record's discriminators, timestamps and +// reason code stay projected, so a client can still see that a deferral is pending, +// of which kind, since when and why. func PublicTaskMetadata(metadata map[string]interface{}) map[string]interface{} { if metadata == nil { return nil @@ -22,6 +28,7 @@ func PublicTaskMetadata(metadata map[string]interface{}) map[string]interface{} publicDeferred := maps.Clone(deferred) delete(publicDeferred, DeferredLaunchUserIDKey) delete(publicDeferred, DeferredLaunchRecordRecentUseKey) + delete(publicDeferred, CeilingLaunchPayloadKey) public[MetaKeyDeferredLaunch] = publicDeferred return public } diff --git a/apps/backend/internal/task/models/session_state_sets.go b/apps/backend/internal/task/models/session_state_sets.go index ab18b6eb7d..c5b68ca32f 100644 --- a/apps/backend/internal/task/models/session_state_sets.go +++ b/apps/backend/internal/task/models/session_state_sets.go @@ -20,3 +20,30 @@ func IsTaskLookupActiveSessionState(state TaskSessionState) bool { return false } } + +// admittedSessionStates classifies every TaskSessionState by whether a session in +// that state holds a unit of the instance-wide session ceiling. Every state is +// listed explicitly: a state added to AllTaskSessionStates without a decision here +// is absent from this map rather than defaulted into the counted set, and +// TestAdmittedSessionStatesCoverEveryState fails on the omission. +var admittedSessionStates = map[TaskSessionState]bool{ + TaskSessionStateCreated: false, + TaskSessionStateStarting: true, + TaskSessionStateRunning: true, + TaskSessionStateIdle: false, + TaskSessionStateWaitingForInput: false, + TaskSessionStateCompleted: false, + TaskSessionStateFailed: false, + TaskSessionStateCancelled: false, +} + +// IsAdmittedSessionState reports whether a session in this state is running or +// starting an agent process, and so occupies a unit of the instance-wide session +// ceiling. +// +// This is a different set from its neighbour IsTaskLookupActiveSessionState, which +// additionally counts Created and WaitingForInput: those hold a task's active-session +// slot and a worktree, but no process and no core. +func IsAdmittedSessionState(state TaskSessionState) bool { + return admittedSessionStates[state] +} diff --git a/apps/backend/internal/task/models/session_state_sets_test.go b/apps/backend/internal/task/models/session_state_sets_test.go index 684ee4dfe1..d19f78b45a 100644 --- a/apps/backend/internal/task/models/session_state_sets_test.go +++ b/apps/backend/internal/task/models/session_state_sets_test.go @@ -2,8 +2,11 @@ package models import "testing" -// TestIsTaskLookupActiveSessionState verifies the active-state contract used by -// GetActiveTaskSessionByTaskID. +// TestIsTaskLookupActiveSessionState pins each state's expected boolean against +// an independent, hand-written table rather than the predicate under test, so a +// semantic regression (for example flipping Completed to active) fails here +// instead of passing silently through callers that derive their own expectation +// from the same predicate. func TestIsTaskLookupActiveSessionState(t *testing.T) { want := map[TaskSessionState]bool{ TaskSessionStateCreated: true, @@ -16,13 +19,6 @@ func TestIsTaskLookupActiveSessionState(t *testing.T) { TaskSessionStateCancelled: false, } - // AllTaskSessionStates is the canonical list this test ranges over. The checks - // below assert it and want cover exactly the same set, with no duplicates — so - // extending AllTaskSessionStates without extending want, or a duplicated/dropped - // entry silently shrinking coverage, fails here. What this CANNOT catch: a new - // TaskSessionState constant added to models.go but never added to - // AllTaskSessionStates. Go does not enforce exhaustiveness and the exhaustive - // linter is not enabled in this repo. seen := make(map[TaskSessionState]bool, len(AllTaskSessionStates)) for _, state := range AllTaskSessionStates { if seen[state] { @@ -39,12 +35,50 @@ func TestIsTaskLookupActiveSessionState(t *testing.T) { } } - for _, state := range AllTaskSessionStates { - t.Run(string(state), func(t *testing.T) { - wantActive := want[state] - if got := IsTaskLookupActiveSessionState(state); got != wantActive { - t.Errorf("IsTaskLookupActiveSessionState(%q) = %v, want %v", state, got, wantActive) - } - }) + for state, wantActive := range want { + if got := IsTaskLookupActiveSessionState(state); got != wantActive { + t.Errorf("IsTaskLookupActiveSessionState(%q) = %v, want %v", state, got, wantActive) + } + } +} + +// TestAdmittedSessionStatesCoverEveryState fails when a member of +// AllTaskSessionStates has no explicit entry in admittedSessionStates. Go does not +// enforce switch/slice exhaustiveness, so without this a state added later is +// silently classified by the map's zero value instead of by a decision. +func TestAdmittedSessionStatesCoverEveryState(t *testing.T) { + for _, state := range AllTaskSessionStates { + if _, classified := admittedSessionStates[state]; !classified { + t.Errorf("state %q has no entry in admittedSessionStates: classify it as counted or not counted", state) + } + } + if len(admittedSessionStates) != len(AllTaskSessionStates) { + t.Errorf("admittedSessionStates has %d entries, AllTaskSessionStates has %d: the map classifies a state that is not in the canonical list", + len(admittedSessionStates), len(AllTaskSessionStates)) + } +} + +// TestIsAdmittedSessionStateCountsOnlyStartingAndRunning pins the counted set to +// the two states that hold an agent process. +func TestIsAdmittedSessionStateCountsOnlyStartingAndRunning(t *testing.T) { + for _, state := range AllTaskSessionStates { + want := state == TaskSessionStateStarting || state == TaskSessionStateRunning + if got := IsAdmittedSessionState(state); got != want { + t.Errorf("IsAdmittedSessionState(%q) = %v, want %v", state, got, want) + } + } +} + +// TestIsAdmittedSessionStateIsDistinctFromLookupActive guards the two predicates +// against being collapsed into one: Created and WaitingForInput are active for the +// task-lookup set and not counted here. +func TestIsAdmittedSessionStateIsDistinctFromLookupActive(t *testing.T) { + for _, state := range []TaskSessionState{TaskSessionStateCreated, TaskSessionStateWaitingForInput} { + if !IsTaskLookupActiveSessionState(state) { + t.Fatalf("precondition: IsTaskLookupActiveSessionState(%q) = false, want true", state) + } + if IsAdmittedSessionState(state) { + t.Errorf("IsAdmittedSessionState(%q) = true, want false: it holds no agent process", state) + } } } diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index 8878903137..b96d9620d7 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -68,6 +68,14 @@ type TaskRepository interface { // task.Position as given, for the generic task-update API's explicit // position field (predates REQ-TASKS-KANBAN-TASK-REORDERING-001). UpdateTaskWithExplicitPosition(ctx context.Context, task *models.Task) error + // UpdateTaskPreservingDeferredLaunch is UpdateTask for callers holding a + // task snapshot old enough to race the session ceiling's deferred_launch + // compare-and-set writers: deferred_launch in the write payload is + // replaced by the row's own current value at write time, so a stale + // snapshot can never resurrect or clobber a concurrent CAS write. Every + // other key keeps ordinary replace semantics, including deletion by + // omission. + UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error DeleteTask(ctx context.Context, id string) error ListTasks(ctx context.Context, workflowID string) ([]*models.Task, error) ListTasksByWorkspace(ctx context.Context, workspaceID, workflowID, repositoryID, query string, page, pageSize int, sort string, includeArchived, includeEphemeral, onlyEphemeral, excludeConfig bool) ([]*models.Task, int, error) @@ -108,6 +116,21 @@ type TaskRepository interface { // counterpart to an atomic remove: an editor must never re-create a key a // concurrent claim just consumed. SetTaskMetadataKeyIfPresent(ctx context.Context, taskID, key string, value interface{}) (bool, error) + // GetTaskDeferredLaunch reads a task's deferred_launch record and the + // opaque prior-state token a subsequent SetTaskDeferredLaunchIfUnchanged + // compares against. Unlike SetTaskMetadataKeyIfPresent, the comparison is + // over the whole stored value, not just the key's presence, so an editor + // that reads, patches one field and writes back cannot silently clobber a + // concurrent writer's change to a different field. + GetTaskDeferredLaunch(ctx context.Context, taskID string) (map[string]interface{}, interface{}, error) + // SetTaskDeferredLaunchIfUnchanged writes the deferred_launch record only + // when the stored value still matches prior (from GetTaskDeferredLaunch). + // A lost comparison is reported through lostCompare with a nil error: it + // is an ordinary, expected race whose handling is to re-read and re-apply, + // not a failure. + SetTaskDeferredLaunchIfUnchanged( + ctx context.Context, taskID string, prior interface{}, value map[string]interface{}, + ) (stored bool, lostCompare bool, err error) UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error // UpdateTaskStateIfSessionState atomically transitions task state only while // the named session remains in expectedSessionState and the task is not diff --git a/apps/backend/internal/task/repository/sqlite/deferred_launch_cas.go b/apps/backend/internal/task/repository/sqlite/deferred_launch_cas.go new file mode 100644 index 0000000000..bf0841336d --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/deferred_launch_cas.go @@ -0,0 +1,292 @@ +package sqlite + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "maps" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/task/models" +) + +// DeferredLaunchPrior is an opaque expected-prior-state token for +// SetTaskDeferredLaunchIfUnchanged, obtained from GetTaskDeferredLaunch or from +// AbsentDeferredLaunch. +// +// It is opaque, and it is produced by this package rather than supplied by the +// caller, for one reason: the comparison is over the whole stored deferred_launch +// value, and a caller holding a decoded map[string]interface{} cannot reconstruct +// those bytes reliably. Re-marshalling a decoded map turns every number into a +// float64, so a Unix-second ceiling_queued_at comes back as 1.7576064e+09 and the +// caller's compare-and-set can never win again. Deriving the token from the +// database's own bytes at read time removes that failure mode from the API. +// +// It is exported through GetTaskDeferredLaunch and SetTaskDeferredLaunchIfUnchanged +// as interface{} rather than as this named type, so a consumer package (the +// orchestrator's admission controller) can thread the token through a +// repository-shaped interface without importing this package — the same +// structural-interface boundary every other repoStore method already keeps. +type DeferredLaunchPrior struct { + present bool + canonical string +} + +// AbsentDeferredLaunch is the expected prior state meaning "the task carries no +// deferred_launch record". It is a legal prior, and it is what lets the first +// writer create the record: the create case — two concurrent first refusals each +// reading "no record" — is exactly the race this compare-and-set exists for, and +// the repository's stamp-based CAS primitives cannot express it at all. +func AbsentDeferredLaunch() interface{} { return DeferredLaunchPrior{} } + +// GetTaskDeferredLaunch reads a task's deferred_launch record and the prior-state +// token that a subsequent SetTaskDeferredLaunchIfUnchanged compares against. +// +// The returned record decodes numbers as json.Number rather than float64, so a +// caller that reads, edits one key and writes back does not silently rewrite the +// others. A record whose stored value is not a JSON object decodes to a nil map +// with a present prior, leaving the caller free to replace it. +func (r *Repository) GetTaskDeferredLaunch( + ctx context.Context, taskID string, +) (map[string]interface{}, interface{}, error) { + var raw sql.NullString + err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`SELECT metadata FROM tasks WHERE id = ?`), taskID).Scan(&raw) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, DeferredLaunchPrior{}, fmt.Errorf("task not found: %s", taskID) + } + return nil, DeferredLaunchPrior{}, err + } + metadata := "{}" + if raw.Valid { + metadata = raw.String + } + return decodeDeferredLaunch(metadata) +} + +// SetTaskDeferredLaunchIfUnchanged writes the deferred_launch record only when the +// stored value still matches prior. +// +// The transaction, row lock and update query are the ones setMetadataKeyIfStamp +// already uses: one BeginTxx, then lockMetadataRow (SELECT … FOR UPDATE on +// Postgres, a transaction-scoped read on SQLite), then the json_set / jsonb_set +// single-key update, then commit. Only the comparison predicate is new. +// +// A lost comparison is reported through lostCompare with a nil error, and that +// distinction is the point: a lost compare is an ordinary, expected race whose +// handling is to re-read and re-apply, whereas an error is a failure that +// escalates to a refusal the system could not persist. Returning the first as the +// second would surface a red card for a case the design handles. +func (r *Repository) SetTaskDeferredLaunchIfUnchanged( + ctx context.Context, taskID string, prior interface{}, value map[string]interface{}, +) (stored bool, lostCompare bool, err error) { + if value == nil { + return false, false, fmt.Errorf("deferred launch value must not be nil for task %s", taskID) + } + payload, err := json.Marshal(value) + if err != nil { + return false, false, fmt.Errorf("failed to serialize deferred launch record: %w", err) + } + + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return false, false, err + } + defer func() { _ = tx.Rollback() }() + + metadata, err := r.lockMetadataRow(ctx, tx, "tasks", "task", taskID) + if err != nil { + return false, false, err + } + _, found, err := decodeDeferredLaunch(metadata) + if err != nil { + return false, false, err + } + if found != prior { + // Commit rather than roll back: the read lock has nothing to undo, and + // committing releases it immediately so the winner is not held up. + if err := tx.Commit(); err != nil { + return false, false, err + } + return false, true, nil + } + + result, err := tx.ExecContext(ctx, + r.db.Rebind(metadataKeyUpdateQuery("tasks", r.db.DriverName())), + metadataKeyUpdateArgs(r.db.DriverName(), models.MetaKeyDeferredLaunch, string(payload), r.nowUTC(), taskID)...) + if err != nil { + return false, false, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, false, err + } + if rows == 0 { + return false, false, fmt.Errorf("task not found: %s", taskID) + } + if err := tx.Commit(); err != nil { + return false, false, err + } + return true, false, nil +} + +// decodeDeferredLaunch splits a task's whole metadata JSON into the decoded +// deferred_launch record and its prior-state token. Both callers go through it so +// the read side and the locked compare side canonicalize identically. +func decodeDeferredLaunch(metadataJSON string) (map[string]interface{}, DeferredLaunchPrior, error) { + trimmed := bytes.TrimSpace([]byte(metadataJSON)) + if len(trimmed) == 0 || string(trimmed) == jsonNull { + return nil, DeferredLaunchPrior{}, nil + } + var metadata map[string]json.RawMessage + if err := json.Unmarshal(trimmed, &metadata); err != nil { + return nil, DeferredLaunchPrior{}, fmt.Errorf("failed to parse metadata: %w", err) + } + canonical, err := models.CanonicalJSON(metadata[models.MetaKeyDeferredLaunch]) + if err != nil { + return nil, DeferredLaunchPrior{}, fmt.Errorf("failed to parse %s: %w", models.MetaKeyDeferredLaunch, err) + } + if canonical == nil { + return nil, DeferredLaunchPrior{}, nil + } + + prior := DeferredLaunchPrior{present: true, canonical: string(canonical)} + decoder := json.NewDecoder(bytes.NewReader(canonical)) + decoder.UseNumber() + var record map[string]interface{} + if err := decoder.Decode(&record); err != nil { + // Present but not an object. The caller decides what to do with it, + // and the prior still describes exactly what is stored. + return nil, prior, nil + } + return record, prior, nil +} + +// TakeTaskDeferredLaunchWIPKeys removes the launch-intent keys a direct start +// claims, and only those, returning what it took so a failed start can put them +// back. +// +// The deferred_launch record carries independent meanings that share one key, so +// claiming the whole key would delete a meaning this caller does not own. The +// partition is by prefix rather than by an enumerated list, so neither writer has +// to track the other's keys: everything named by IsCeilingRecordKey stays, and +// everything else is the claim's. +// +// When removing the claimed keys empties the object the whole key is deleted, +// which is the entire behaviour for a record that carries no ceiling keys. +// A record holding only ceiling keys is not this caller's to claim, so the take +// is inert and reports nothing claimed. +func (r *Repository) TakeTaskDeferredLaunchWIPKeys( + ctx context.Context, taskID string, +) (wip map[string]interface{}, claimed bool, err error) { + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return nil, false, err + } + defer func() { _ = tx.Rollback() }() + + metadata, err := r.lockMetadataRow(ctx, tx, "tasks", "task", taskID) + if err != nil { + return nil, false, err + } + record, prior, err := decodeDeferredLaunch(metadata) + if err != nil { + return nil, false, err + } + if !prior.present { + return nil, false, tx.Commit() + } + if record == nil { + return nil, false, fmt.Errorf("%s on task %s is not a JSON object", models.MetaKeyDeferredLaunch, taskID) + } + + claimedKeys := make(map[string]interface{}) + retained := make(map[string]interface{}) + for key, value := range record { + if models.IsCeilingRecordKey(key) { + retained[key] = value + continue + } + claimedKeys[key] = value + } + if len(claimedKeys) == 0 { + return nil, false, tx.Commit() + } + + if len(retained) == 0 { + if _, err := r.removeTaskMetadataKeyWithExecutor(ctx, tx, taskID, models.MetaKeyDeferredLaunch); err != nil { + return nil, false, err + } + return claimedKeys, true, tx.Commit() + } + if err := r.writeDeferredLaunchLocked(ctx, tx, taskID, retained); err != nil { + return nil, false, err + } + return claimedKeys, true, tx.Commit() +} + +// RestoreTaskDeferredLaunchWIPKeys puts previously claimed keys back, merging +// into whatever the record holds now rather than replacing it. +// +// Restoring the pre-claim snapshot wholesale would overwrite anything written to +// the record while the launch was in flight. Merging is what lets the two +// meanings coexist: this caller puts back exactly the keys it took and leaves +// every other key at its current value. +func (r *Repository) RestoreTaskDeferredLaunchWIPKeys( + ctx context.Context, taskID string, wip map[string]interface{}, +) error { + if len(wip) == 0 { + return nil + } + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + metadata, err := r.lockMetadataRow(ctx, tx, "tasks", "task", taskID) + if err != nil { + return err + } + record, _, err := decodeDeferredLaunch(metadata) + if err != nil { + return err + } + merged := make(map[string]interface{}, len(record)+len(wip)) + maps.Copy(merged, record) + maps.Copy(merged, wip) + + if err := r.writeDeferredLaunchLocked(ctx, tx, taskID, merged); err != nil { + return err + } + return tx.Commit() +} + +// writeDeferredLaunchLocked performs the single-key update inside a transaction +// that already holds the row. +func (r *Repository) writeDeferredLaunchLocked( + ctx context.Context, tx *sqlx.Tx, taskID string, record map[string]interface{}, +) error { + payload, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("failed to serialize deferred launch record: %w", err) + } + result, err := tx.ExecContext(ctx, + r.db.Rebind(metadataKeyUpdateQuery("tasks", r.db.DriverName())), + metadataKeyUpdateArgs(r.db.DriverName(), models.MetaKeyDeferredLaunch, string(payload), r.nowUTC(), taskID)...) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return fmt.Errorf("task not found: %s", taskID) + } + return nil +} diff --git a/apps/backend/internal/task/repository/sqlite/deferred_launch_cas_test.go b/apps/backend/internal/task/repository/sqlite/deferred_launch_cas_test.go new file mode 100644 index 0000000000..dc48b4212b --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/deferred_launch_cas_test.go @@ -0,0 +1,344 @@ +package sqlite + +import ( + "context" + "encoding/json" + "sync" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func newCeilingRecord(queuedAt int64) map[string]interface{} { + return map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingQueuedAtKey: queuedAt, + models.CeilingLaunchKindKey: "start_task", + } +} + +func readTaskMetadata(t *testing.T, repo *Repository, taskID string) map[string]interface{} { + t.Helper() + task, err := repo.GetTask(context.Background(), taskID) + if err != nil { + t.Fatalf("GetTask(%s): %v", taskID, err) + } + return task.Metadata +} + +// TestSetTaskDeferredLaunchCreatesFromAbsent covers AC-40's new part: absent is a +// legal expected prior state, so the first writer can create the record. +func TestSetTaskDeferredLaunchCreatesFromAbsent(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "cas-create", Title: "Create"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + record, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-create") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if record != nil { + t.Fatalf("GetTaskDeferredLaunch returned %v, want nil for a task with no record", record) + } + + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-create", prior, newCeilingRecord(1757606400)) + if err != nil { + t.Fatalf("SetTaskDeferredLaunchIfUnchanged: %v", err) + } + if !stored || lost { + t.Fatalf("SetTaskDeferredLaunchIfUnchanged = (stored=%v, lost=%v), want (true, false)", stored, lost) + } + + metadata := readTaskMetadata(t, repo, "cas-create") + written, ok := metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok { + t.Fatalf("deferred_launch not written: %+v", metadata) + } + if written[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling_deferred not set: %+v", written) + } +} + +// TestSetTaskDeferredLaunchReportsLostCompareDistinguishably covers AC-40b: an +// expected prior of absent against a present record is a lost comparison, not an +// error, so the caller enters AC-12e's re-read path instead of AC-45's failure path. +func TestSetTaskDeferredLaunchReportsLostCompareDistinguishably(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "cas-lost", Title: "Lost"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-lost") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + // Another writer lands first, using the same absent prior. + if stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-lost", prior, newCeilingRecord(111)); err != nil || !stored || lost { + t.Fatalf("first write = (%v, %v, %v), want (true, false, nil)", stored, lost, err) + } + + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-lost", prior, newCeilingRecord(222)) + if err != nil { + t.Fatalf("lost compare reported as an error: %v", err) + } + if stored || !lost { + t.Fatalf("second write = (stored=%v, lost=%v), want (false, true)", stored, lost) + } + + written := readTaskMetadata(t, repo, "cas-lost")[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if got := jsonNumberOf(t, written[models.CeilingQueuedAtKey]); got != "111" { + t.Fatalf("losing writer overwrote the stored record: ceiling_queued_at = %s, want 111", got) + } +} + +// TestSetTaskDeferredLaunchAcceptsAMatchingPresentPrior and its stale twin cover +// AC-40a: the comparison is over the whole deferred_launch value. +func TestSetTaskDeferredLaunchAcceptsAMatchingPresentPrior(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "cas-present", Title: "Present", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: newCeilingRecord(1757606400)}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + record, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-present") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if record == nil || record[models.CeilingDeferredKey] != true { + t.Fatalf("GetTaskDeferredLaunch returned %+v, want the stored ceiling record", record) + } + // A large integer must survive the read/compare round trip; a float64 + // round trip would render it in exponent form and the CAS could never win. + if got := jsonNumberOf(t, record[models.CeilingQueuedAtKey]); got != "1757606400" { + t.Fatalf("ceiling_queued_at read back as %s, want 1757606400", got) + } + + record[models.CeilingReasonCodeKey] = "ceiling" + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-present", prior, record) + if err != nil || !stored || lost { + t.Fatalf("SetTaskDeferredLaunchIfUnchanged = (%v, %v, %v), want (true, false, nil)", stored, lost, err) + } +} + +func TestSetTaskDeferredLaunchRejectsAStalePresentPrior(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "cas-stale", Title: "Stale", + Metadata: map[string]interface{}{models.MetaKeyDeferredLaunch: newCeilingRecord(1)}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + record, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-stale") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + // Somebody else advances the record after our read. + _, newPrior, err := repo.GetTaskDeferredLaunch(ctx, "cas-stale") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if stored, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-stale", newPrior, newCeilingRecord(2)); err != nil || !stored { + t.Fatalf("interleaved write failed: stored=%v err=%v", stored, err) + } + + record[models.CeilingReasonCodeKey] = "ceiling" + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-stale", prior, record) + if err != nil { + t.Fatalf("stale compare reported as an error: %v", err) + } + if stored || !lost { + t.Fatalf("stale write = (stored=%v, lost=%v), want (false, true)", stored, lost) + } +} + +// TestSetTaskDeferredLaunchLeavesOtherMetadataKeysAlone pins that the CAS writes +// one key: it must not become a whole-metadata replace. +func TestSetTaskDeferredLaunchLeavesOtherMetadataKeysAlone(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: "cas-neighbours", Title: "Neighbours", + Metadata: map[string]interface{}{"unrelated_key": "keep me"}, + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-neighbours") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if stored, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-neighbours", prior, newCeilingRecord(5)); err != nil || !stored { + t.Fatalf("write failed: stored=%v err=%v", stored, err) + } + + metadata := readTaskMetadata(t, repo, "cas-neighbours") + if metadata["unrelated_key"] != "keep me" { + t.Fatalf("CAS clobbered a neighbouring metadata key: %+v", metadata) + } +} + +// TestSetTaskDeferredLaunchReportsMissingTaskAsAnError keeps a genuine failure +// distinguishable from AC-40b's expected lost comparison. +func TestSetTaskDeferredLaunchReportsMissingTaskAsAnError(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + if _, _, err := repo.GetTaskDeferredLaunch(ctx, "cas-missing"); err == nil { + t.Fatal("GetTaskDeferredLaunch accepted a task that does not exist") + } + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-missing", AbsentDeferredLaunch(), newCeilingRecord(1)) + if err == nil { + t.Fatalf("SetTaskDeferredLaunchIfUnchanged = (%v, %v, nil) for a missing task, want an error", stored, lost) + } + if lost { + t.Fatal("a missing task was reported as a lost comparison") + } +} + +// TestSetTaskDeferredLaunchAdmitsExactlyOneConcurrentCreate is AC-12e's race: two +// automatic launches refused for the same task at the same moment both read "no +// record". Exactly one may store; the other must report a lost compare, not an +// error, and must not lose the stored payload. +func TestSetTaskDeferredLaunchAdmitsExactlyOneConcurrentCreate(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "cas-race", Title: "Race"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + const writers = 8 + var ( + wg sync.WaitGroup + readAll sync.WaitGroup + mu sync.Mutex + stores int + losses int + failures []error + ) + // Every writer must hold its prior BEFORE any write lands, or the race + // under test does not happen: a writer that reads after an earlier commit + // holds a present prior, and its compare-and-set then succeeds correctly + // against the state it actually read. Releasing the writers without this + // barrier measures scheduling order, not the lost-update the CAS prevents. + readAll.Add(writers) + start := make(chan struct{}) + for i := 0; i < writers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "cas-race") + readAll.Done() + <-start + if err == nil { + var stored, lost bool + stored, lost, err = repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-race", prior, newCeilingRecord(int64(i))) + mu.Lock() + switch { + case err != nil: + failures = append(failures, err) + case stored: + stores++ + case lost: + losses++ + } + mu.Unlock() + return + } + mu.Lock() + failures = append(failures, err) + mu.Unlock() + }(i) + } + readAll.Wait() + close(start) + wg.Wait() + + if len(failures) != 0 { + t.Fatalf("concurrent CAS reported errors: %v", failures) + } + if stores != 1 { + t.Fatalf("stores = %d, want exactly 1 (losses = %d)", stores, losses) + } + if losses != writers-1 { + t.Fatalf("losses = %d, want %d", losses, writers-1) + } + if _, ok := readTaskMetadata(t, repo, "cas-race")[models.MetaKeyDeferredLaunch].(map[string]interface{}); !ok { + t.Fatal("no record survived the race") + } +} + +func jsonNumberOf(t *testing.T, value interface{}) string { + t.Helper() + switch typed := value.(type) { + case json.Number: + return typed.String() + case string: + return typed + default: + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal %v: %v", value, err) + } + return string(encoded) + } +} + +// TestSetTaskDeferredLaunchLetsALosingWriterReReadAndWin covers the other half of +// the contract: a lost comparison is recoverable, not terminal. The writer that +// lost re-reads, re-applies its rule to what it found, and writes again against +// the fresh prior. This is why a lost compare must not be reported as an error. +func TestSetTaskDeferredLaunchLetsALosingWriterReReadAndWin(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "cas-reread", Title: "Reread"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + + _, stalePrior, err := repo.GetTaskDeferredLaunch(ctx, "cas-reread") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + // The winner lands first. + if stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-reread", AbsentDeferredLaunch(), newCeilingRecord(100)); err != nil || !stored || lost { + t.Fatalf("winner = (%v, %v, %v), want (true, false, nil)", stored, lost, err) + } + + // The loser discovers it lost... + if stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-reread", stalePrior, newCeilingRecord(200)); err != nil || stored || !lost { + t.Fatalf("loser = (%v, %v, %v), want (false, true, nil)", stored, lost, err) + } + + // ...re-reads, re-applies its rule to what is actually stored, and wins. + record, freshPrior, err := repo.GetTaskDeferredLaunch(ctx, "cas-reread") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if got := jsonNumberOf(t, record[models.CeilingQueuedAtKey]); got != "100" { + t.Fatalf("re-read saw ceiling_queued_at %s, want the winner's 100", got) + } + record[models.CeilingReasonCodeKey] = "ceiling_superseded" + stored, lost, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "cas-reread", freshPrior, record) + if err != nil || !stored || lost { + t.Fatalf("re-applied write = (%v, %v, %v), want (true, false, nil)", stored, lost, err) + } + + final, _, err := repo.GetTaskDeferredLaunch(ctx, "cas-reread") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + // AC-32: the first record's queued_at survives the loser's re-application. + if got := jsonNumberOf(t, final[models.CeilingQueuedAtKey]); got != "100" { + t.Fatalf("ceiling_queued_at = %s after re-apply, want the original 100 preserved", got) + } + if final[models.CeilingReasonCodeKey] != "ceiling_superseded" { + t.Fatalf("re-applied reason code missing: %+v", final) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/deferred_launch_wip_test.go b/apps/backend/internal/task/repository/sqlite/deferred_launch_wip_test.go new file mode 100644 index 0000000000..edd6e5118e --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/deferred_launch_wip_test.go @@ -0,0 +1,205 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func wipOnlyRecord() map[string]interface{} { + return map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + models.DeferredLaunchUserIDKey: "user-1", + } +} + +func mixedRecord() map[string]interface{} { + record := wipOnlyRecord() + record[models.CeilingDeferredKey] = true + record[models.CeilingLaunchKindKey] = "start_task" + return record +} + +func storedDeferredLaunch(t *testing.T, repo *Repository, taskID string) map[string]interface{} { + t.Helper() + record, _, err := repo.GetTaskDeferredLaunch(context.Background(), taskID) + if err != nil { + t.Fatalf("GetTaskDeferredLaunch(%s): %v", taskID, err) + } + return record +} + +func createTaskWithDeferredLaunch(t *testing.T, repo *Repository, taskID string, record map[string]interface{}) { + t.Helper() + task := &models.Task{ID: taskID, Title: taskID} + if record != nil { + task.Metadata = map[string]interface{}{models.MetaKeyDeferredLaunch: record} + } + if err := repo.CreateTask(context.Background(), task); err != nil { + t.Fatalf("CreateTask(%s): %v", taskID, err) + } +} + +// TestTakeWIPKeysLeavesCeilingKeysInPlace is the criterion this pair exists for: +// the WIP claim must not delete a ceiling record that landed beside it. Losing +// that record is a silently dropped launch, and it is unrecoverable rather than +// merely stale, because the sweep then has nothing to find. +func TestTakeWIPKeysLeavesCeilingKeysInPlace(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-mixed", mixedRecord()) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-mixed") + if err != nil { + t.Fatalf("TakeTaskDeferredLaunchWIPKeys: %v", err) + } + if !claimed { + t.Fatal("claimed = false, want true: the record carries WIP keys") + } + if wip[models.DeferredLaunchStartWhenUnblockedKey] != true || wip[models.DeferredLaunchUserIDKey] != "user-1" { + t.Fatalf("returned WIP keys = %+v, want both WIP keys", wip) + } + if _, leaked := wip[models.CeilingDeferredKey]; leaked { + t.Fatalf("ceiling key was taken as a WIP key: %+v", wip) + } + + remaining := storedDeferredLaunch(t, repo, "wip-mixed") + if remaining == nil { + t.Fatal("the whole deferred_launch record was deleted; the ceiling deferral is gone") + } + if remaining[models.CeilingDeferredKey] != true || remaining[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("ceiling keys did not survive the claim: %+v", remaining) + } + if _, still := remaining[models.DeferredLaunchStartWhenUnblockedKey]; still { + t.Fatalf("WIP key survived the claim: %+v", remaining) + } +} + +// TestTakeWIPKeysDeletesTheRecordWhenOnlyWIPKeysExisted preserves today's +// behaviour for every record that exists before this card. +func TestTakeWIPKeysDeletesTheRecordWhenOnlyWIPKeysExisted(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-only", wipOnlyRecord()) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-only") + if err != nil || !claimed { + t.Fatalf("take = (%+v, %v, %v), want claimed", wip, claimed, err) + } + if remaining := storedDeferredLaunch(t, repo, "wip-only"); remaining != nil { + t.Fatalf("record survived as %+v, want the key deleted entirely", remaining) + } +} + +func TestTakeWIPKeysIsInertWhenOnlyCeilingKeysExist(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-ceiling-only", map[string]interface{}{ + models.CeilingDeferredKey: true, + }) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-ceiling-only") + if err != nil { + t.Fatalf("TakeTaskDeferredLaunchWIPKeys: %v", err) + } + if claimed || len(wip) != 0 { + t.Fatalf("take = (%+v, %v), want inert: a ceiling deferral is not the WIP path's to consume", wip, claimed) + } + if remaining := storedDeferredLaunch(t, repo, "wip-ceiling-only"); remaining[models.CeilingDeferredKey] != true { + t.Fatalf("ceiling record disturbed: %+v", remaining) + } +} + +func TestTakeWIPKeysIsInertWhenNoRecordExists(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-none", nil) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-none") + if err != nil { + t.Fatalf("TakeTaskDeferredLaunchWIPKeys: %v", err) + } + if claimed || len(wip) != 0 { + t.Fatalf("take = (%+v, %v), want inert", wip, claimed) + } +} + +// TestRestoreWIPKeysMergesIntoACeilingRecordWrittenDuringTheLaunch is AC-59's +// other write: the release must not overwrite a ceiling record that appeared +// while the launch was in flight, which an unconditional restore of the +// pre-claim snapshot would do. +func TestRestoreWIPKeysMergesIntoACeilingRecordWrittenDuringTheLaunch(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-restore-merge", wipOnlyRecord()) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-restore-merge") + if err != nil || !claimed { + t.Fatalf("take failed: %v %v", claimed, err) + } + + // A ceiling refusal lands while the launch is in flight. + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "wip-restore-merge") + if err != nil { + t.Fatalf("GetTaskDeferredLaunch: %v", err) + } + if stored, _, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "wip-restore-merge", prior, map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }); err != nil || !stored { + t.Fatalf("ceiling write failed: stored=%v err=%v", stored, err) + } + + // The launch fails and the claim is released. + if err := repo.RestoreTaskDeferredLaunchWIPKeys(ctx, "wip-restore-merge", wip); err != nil { + t.Fatalf("RestoreTaskDeferredLaunchWIPKeys: %v", err) + } + + merged := storedDeferredLaunch(t, repo, "wip-restore-merge") + if merged[models.CeilingDeferredKey] != true || merged[models.CeilingLaunchKindKey] != "start_task" { + t.Fatalf("restore clobbered the ceiling record: %+v", merged) + } + if merged[models.DeferredLaunchStartWhenUnblockedKey] != true || merged[models.DeferredLaunchUserIDKey] != "user-1" { + t.Fatalf("restore did not put the WIP keys back: %+v", merged) + } +} + +func TestRestoreWIPKeysRecreatesADeletedRecord(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-restore-recreate", wipOnlyRecord()) + + wip, claimed, err := repo.TakeTaskDeferredLaunchWIPKeys(ctx, "wip-restore-recreate") + if err != nil || !claimed { + t.Fatalf("take failed: %v %v", claimed, err) + } + if err := repo.RestoreTaskDeferredLaunchWIPKeys(ctx, "wip-restore-recreate", wip); err != nil { + t.Fatalf("RestoreTaskDeferredLaunchWIPKeys: %v", err) + } + + restored := storedDeferredLaunch(t, repo, "wip-restore-recreate") + if restored[models.DeferredLaunchStartWhenUnblockedKey] != true || restored[models.DeferredLaunchUserIDKey] != "user-1" { + t.Fatalf("record not restored: %+v", restored) + } + if !models.HasStartWhenUnblockedIntent(&models.Task{Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: restored, + }}) { + t.Fatal("HasStartWhenUnblockedIntent no longer reports the restored intent") + } +} + +func TestRestoreWIPKeysIsANoOpForAnEmptyClaim(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + createTaskWithDeferredLaunch(t, repo, "wip-restore-empty", map[string]interface{}{ + models.CeilingDeferredKey: true, + }) + + if err := repo.RestoreTaskDeferredLaunchWIPKeys(ctx, "wip-restore-empty", nil); err != nil { + t.Fatalf("RestoreTaskDeferredLaunchWIPKeys: %v", err) + } + if record := storedDeferredLaunch(t, repo, "wip-restore-empty"); record[models.CeilingDeferredKey] != true { + t.Fatalf("empty restore disturbed the record: %+v", record) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/session_ceiling.go b/apps/backend/internal/task/repository/sqlite/session_ceiling.go new file mode 100644 index 0000000000..e774bf1cd1 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/session_ceiling.go @@ -0,0 +1,48 @@ +package sqlite + +import ( + "context" + + "github.com/kandev/kandev/internal/task/models" +) + +// ListAdmittedSessionIDs returns the ids of every session in the admitted +// population — state STARTING or RUNNING — across the entire instance: every +// workspace, every workflow, every board, Office and non-Office alike. +// +// The query deliberately has no joins and no filters. A session counts regardless +// of whether its task is archived, ephemeral or automation-origin, and regardless +// of config_mode or IsPassthrough on the session itself: each one runs an agent +// process and consumes a core. Adding any of the filters the neighbouring task +// queries carry would under-count exactly the work the ceiling exists to bound. +// +// The state literals here are written out rather than derived from +// models.IsAdmittedSessionState on purpose: deriving them would make the +// drift-guard test compare the predicate against itself. Keeping the two sides +// independent is what lets TestAdmittedSessionIDsMatchSQLFilter catch an edit to +// one that was not made to the other. +func (r *Repository) ListAdmittedSessionIDs(ctx context.Context) ([]string, error) { + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(` + SELECT s.id + FROM task_sessions s + WHERE s.state IN (?, ?) + ORDER BY s.id ASC + `), string(models.TaskSessionStateStarting), string(models.TaskSessionStateRunning)) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return ids, nil +} diff --git a/apps/backend/internal/task/repository/sqlite/session_ceiling_test.go b/apps/backend/internal/task/repository/sqlite/session_ceiling_test.go new file mode 100644 index 0000000000..1adc29eb4e --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/session_ceiling_test.go @@ -0,0 +1,274 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +func ceilingDeferredMetadata() map[string]interface{} { + return map[string]interface{}{ + models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.CeilingDeferredKey: true, + models.CeilingLaunchKindKey: "start_task", + }, + } +} + +func archiveTaskForCeilingTests(t *testing.T, repo *Repository, taskID string) { + t.Helper() + ctx := context.Background() + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE tasks SET archived_at = ? WHERE id = ?`), time.Now().UTC(), taskID); err != nil { + t.Fatalf("archive %s: %v", taskID, err) + } +} + +// TestAdmittedSessionIDsMatchSQLFilter cross-checks ListAdmittedSessionIDs's SQL +// state filter against models.IsAdmittedSessionState for every state in +// models.AllTaskSessionStates, the package's single canonical state list. Editing +// the SQL without updating the predicate (or vice versa) fails here instead of +// only drifting silently — but only for states already present in +// AllTaskSessionStates: Go does not enforce switch/slice exhaustiveness, so a new +// TaskSessionState constant added to models.go without also being added to +// AllTaskSessionStates is not caught by this test (or by any linter — the +// exhaustive linter is not enabled in this repo). AC-1, AC-3, AC-3a. +func TestAdmittedSessionIDsMatchSQLFilter(t *testing.T) { + for _, state := range models.AllTaskSessionStates { + t.Run(string(state), func(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + taskID := "task-admitted-" + string(state) + sessionID := "session-admitted-" + string(state) + + if err := repo.CreateTask(ctx, &models.Task{ID: taskID, Title: "Admitted state check"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: sessionID, + TaskID: taskID, + State: state, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + got, err := repo.ListAdmittedSessionIDs(ctx) + if err != nil { + t.Fatalf("ListAdmittedSessionIDs: %v", err) + } + + want := models.IsAdmittedSessionState(state) + found := false + for _, id := range got { + if id == sessionID { + found = true + } + } + if found != want { + t.Fatalf("ListAdmittedSessionIDs returned %v for state %q; found=%v want=%v", got, state, found, want) + } + }) + } +} + +// TestListAdmittedSessionIDsIgnoresTaskShape pins AC-2 and AC-2a: a session counts +// regardless of whether its task is archived, ephemeral, or automation-origin, and +// regardless of config-mode or passthrough on the session itself. +func TestListAdmittedSessionIDsIgnoresTaskShape(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + cases := []struct { + name string + task *models.Task + session *models.TaskSession + archive bool + }{ + { + name: "archived task", + task: &models.Task{ID: "task-ceil-archived", Title: "Archived"}, + session: &models.TaskSession{ID: "session-ceil-archived", TaskID: "task-ceil-archived", State: models.TaskSessionStateRunning}, + archive: true, + }, + { + name: "ephemeral task", + task: &models.Task{ID: "task-ceil-ephemeral", Title: "Ephemeral", IsEphemeral: true}, + session: &models.TaskSession{ID: "session-ceil-ephemeral", TaskID: "task-ceil-ephemeral", State: models.TaskSessionStateStarting}, + }, + { + name: "automation origin task", + task: &models.Task{ID: "task-ceil-automation", Title: "Automation", Origin: models.TaskOriginAutomationRun}, + session: &models.TaskSession{ID: "session-ceil-automation", TaskID: "task-ceil-automation", State: models.TaskSessionStateRunning}, + }, + { + name: "config mode session", + task: &models.Task{ID: "task-ceil-config", Title: "Config"}, + session: &models.TaskSession{ + ID: "session-ceil-config", TaskID: "task-ceil-config", State: models.TaskSessionStateRunning, + Metadata: map[string]interface{}{"config_mode": true}, + }, + }, + { + name: "passthrough session", + task: &models.Task{ID: "task-ceil-passthrough", Title: "Passthrough"}, + session: &models.TaskSession{ID: "session-ceil-passthrough", TaskID: "task-ceil-passthrough", State: models.TaskSessionStateStarting, IsPassthrough: true}, + }, + } + + for _, tc := range cases { + if err := repo.CreateTask(ctx, tc.task); err != nil { + t.Fatalf("CreateTask(%s): %v", tc.name, err) + } + if err := repo.CreateTaskSession(ctx, tc.session); err != nil { + t.Fatalf("CreateTaskSession(%s): %v", tc.name, err) + } + if tc.archive { + archiveTaskForCeilingTests(t, repo, tc.task.ID) + } + } + + got, err := repo.ListAdmittedSessionIDs(ctx) + if err != nil { + t.Fatalf("ListAdmittedSessionIDs: %v", err) + } + index := make(map[string]bool, len(got)) + for _, id := range got { + index[id] = true + } + for _, tc := range cases { + if !index[tc.session.ID] { + t.Errorf("ListAdmittedSessionIDs omitted %s (%s); got %v", tc.session.ID, tc.name, got) + } + } + if len(got) != len(cases) { + t.Fatalf("ListAdmittedSessionIDs returned %d ids, want %d: %v", len(got), len(cases), got) + } +} + +// TestListTasksWithCeilingDeferredAppliesNoneOfTheThreeFilters pins AC-50b: the +// sweep's lister must not inherit ListTasksWithMetadataKey's archived_at, +// is_ephemeral or automation-origin filters. Each one would strand records the +// sweep is required to find. +func TestListTasksWithCeilingDeferredAppliesNoneOfTheThreeFilters(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + tasks := []*models.Task{ + {ID: "task-defer-archived", Title: "Archived", Metadata: ceilingDeferredMetadata()}, + {ID: "task-defer-ephemeral", Title: "Ephemeral", IsEphemeral: true, Metadata: ceilingDeferredMetadata()}, + {ID: "task-defer-automation", Title: "Automation", Origin: models.TaskOriginAutomationRun, Metadata: ceilingDeferredMetadata()}, + {ID: "task-defer-plain", Title: "Plain", Metadata: ceilingDeferredMetadata()}, + } + for _, task := range tasks { + if err := repo.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask(%s): %v", task.ID, err) + } + } + archiveTaskForCeilingTests(t, repo, "task-defer-archived") + + got, err := repo.ListTasksWithCeilingDeferred(ctx) + if err != nil { + t.Fatalf("ListTasksWithCeilingDeferred: %v", err) + } + index := make(map[string]bool, len(got)) + for _, task := range got { + index[task.ID] = true + } + for _, task := range tasks { + if !index[task.ID] { + t.Errorf("ListTasksWithCeilingDeferred omitted %s; got %d tasks", task.ID, len(got)) + } + } + if len(got) != len(tasks) { + t.Fatalf("ListTasksWithCeilingDeferred returned %d tasks, want %d", len(got), len(tasks)) + } +} + +// TestListTasksWithCeilingDeferredOrdersByIDAscending pins AC-50c: ordering is by +// tasks.id ascending alone. Insertion order below is deliberately not id order, +// and updated_at is deliberately moved afterwards on the id-first task so a +// three-column sort copied from ListTasksWithMetadataKey would fail here. +func TestListTasksWithCeilingDeferredOrdersByIDAscending(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + for _, id := range []string{"ceil-c", "ceil-a", "ceil-b"} { + if err := repo.CreateTask(ctx, &models.Task{ID: id, Title: id, Metadata: ceilingDeferredMetadata()}); err != nil { + t.Fatalf("CreateTask(%s): %v", id, err) + } + } + // Push the id-first task to the newest updated_at. Under t.id ASC it stays + // first; under a leading t.updated_at ASC term it would sort last. + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE tasks SET updated_at = ? WHERE id = ?`), time.Now().UTC().Add(time.Hour), "ceil-a"); err != nil { + t.Fatalf("bump updated_at: %v", err) + } + + got, err := repo.ListTasksWithCeilingDeferred(ctx) + if err != nil { + t.Fatalf("ListTasksWithCeilingDeferred: %v", err) + } + var ids []string + for _, task := range got { + ids = append(ids, task.ID) + } + want := []string{"ceil-a", "ceil-b", "ceil-c"} + if len(ids) != len(want) { + t.Fatalf("ListTasksWithCeilingDeferred returned %v, want %v", ids, want) + } + for i := range want { + if ids[i] != want[i] { + t.Fatalf("ListTasksWithCeilingDeferred returned %v, want %v", ids, want) + } + } +} + +// TestListTasksWithCeilingDeferredSkipsOtherDeferredLaunchRecords pins the +// discriminator: deferred_launch is a shared record carrying three independent +// meanings, and only the ceiling_deferred one is this sweep's candidate. It also +// pins that the predicate matches on value equality, not mere key presence: a +// task carrying ceiling_deferred explicitly set to false must not be swept. +func TestListTasksWithCeilingDeferredSkipsOtherDeferredLaunchRecords(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + tasks := []*models.Task{ + {ID: "skip-no-metadata", Title: "None"}, + {ID: "skip-empty-record", Title: "Empty", Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: map[string]interface{}{}, + }}, + {ID: "skip-start-when-unblocked", Title: "Unblocked", Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.DeferredLaunchStartWhenUnblockedKey: true, + }, + }}, + {ID: "skip-ceiling-false", Title: "False", Metadata: map[string]interface{}{ + models.MetaKeyDeferredLaunch: map[string]interface{}{ + models.CeilingDeferredKey: false, + }, + }}, + {ID: "keep-ceiling-true", Title: "True", Metadata: ceilingDeferredMetadata()}, + } + for _, task := range tasks { + if err := repo.CreateTask(ctx, task); err != nil { + t.Fatalf("CreateTask(%s): %v", task.ID, err) + } + } + + got, err := repo.ListTasksWithCeilingDeferred(ctx) + if err != nil { + t.Fatalf("ListTasksWithCeilingDeferred: %v", err) + } + if len(got) != 1 || got[0].ID != "keep-ceiling-true" { + var ids []string + for _, task := range got { + ids = append(ids, task.ID) + } + t.Fatalf("ListTasksWithCeilingDeferred returned %v, want [keep-ceiling-true]", ids) + } + if !models.HasCeilingDeferredIntent(got[0]) { + t.Fatalf("scanned task lost its ceiling deferral: %+v", got[0].Metadata) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/step_transition_writers_pin_test.go b/apps/backend/internal/task/repository/sqlite/step_transition_writers_pin_test.go index a67ff62f3b..7109cf7bb7 100644 --- a/apps/backend/internal/task/repository/sqlite/step_transition_writers_pin_test.go +++ b/apps/backend/internal/task/repository/sqlite/step_transition_writers_pin_test.go @@ -47,7 +47,7 @@ import ( // registering it. var registeredStepMutators = []string{ "task/repository/sqlite/Repository.insertTaskTx", - "task/repository/sqlite/Repository.updateTaskTx", + "task/repository/sqlite/Repository.buildTaskUpdateQuery", "task/repository/sqlite/Repository.UpdateTaskIfWorkflowStepHasCapacity", "task/repository/sqlite/Repository.PromoteQueuedTaskIfWorkflowStepHasCapacity", "task/repository/sqlite/Repository.RestoreTaskMessageRollbackIfSessionState", diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index 6eee861cd1..aec2f46eb8 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -1004,6 +1004,24 @@ func (r *Repository) UpdateTaskPriority(ctx context.Context, taskID, priority st // upsert/clear on workflow_step_participants inside the same tx as the // task UPDATE. func (r *Repository) UpdateTask(ctx context.Context, task *models.Task) error { + return r.updateTaskCommit(ctx, task, "", true, false) +} + +// UpdateTaskPreservingDeferredLaunch is UpdateTask for callers that hold a +// task snapshot old enough to have missed a concurrent write to +// deferred_launch — the service layer's request-driven UpdateTask, which +// reads the task once at the start of a request and can commit after the +// session ceiling's own compare-and-set writers (SetTaskDeferredLaunchIfUnchanged) +// have moved that key on. It performs the same write as UpdateTask, except +// deferred_launch in the write payload is replaced by the row's own current +// value at write time, so a stale in-memory snapshot can never resurrect or +// clobber whatever those writers did to it in the meantime. Every other key +// keeps ordinary replace semantics, including deletion by omission. +func (r *Repository) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + return r.updateTaskCommit(ctx, task, "", true, true) +} + +func (r *Repository) updateTaskCommit(ctx context.Context, task *models.Task, expectedWorkflowID string, preservePosition, protectDeferredLaunch bool) error { metadata, err := json.Marshal(task.Metadata) if err != nil { metadata = []byte("{}") @@ -1015,7 +1033,7 @@ func (r *Repository) UpdateTask(ctx context.Context, task *models.Task) error { } defer func() { _ = tx.Rollback() }() - entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, "", true) + entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, expectedWorkflowID, preservePosition, protectDeferredLaunch) if err != nil { return err } @@ -1039,27 +1057,7 @@ func (r *Repository) UpdateTask(ctx context.Context, task *models.Task) error { // write. A mismatch returns repoerrors.ErrWorkflowResolutionConflict and the // transaction rolls back untouched. func (r *Repository) UpdateTaskIfWorkflowMatches(ctx context.Context, task *models.Task, expectedWorkflowID string) error { - metadata, err := json.Marshal(task.Metadata) - if err != nil { - metadata = []byte("{}") - } - - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, expectedWorkflowID, true) - if err != nil { - return err - } - - if err := tx.Commit(); err != nil { - return err - } - r.dispatchStepEntry(ctx, task.ID, task.WorkflowID, task.WorkflowStepID, entryID, markerEntryID) - return nil + return r.updateTaskCommit(ctx, task, expectedWorkflowID, true, false) } // UpdateTaskWithExplicitPosition performs the same write as UpdateTask @@ -1069,27 +1067,7 @@ func (r *Repository) UpdateTaskIfWorkflowMatches(ctx context.Context, task *mode // (REQ-TASKS-KANBAN-TASK-REORDERING-001.28/.31): the generic task-update // API's explicit position field, which predates that contract. func (r *Repository) UpdateTaskWithExplicitPosition(ctx context.Context, task *models.Task) error { - metadata, err := json.Marshal(task.Metadata) - if err != nil { - metadata = []byte("{}") - } - - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, "", false) - if err != nil { - return err - } - - if err := tx.Commit(); err != nil { - return err - } - r.dispatchStepEntry(ctx, task.ID, task.WorkflowID, task.WorkflowStepID, entryID, markerEntryID) - return nil + return r.updateTaskCommit(ctx, task, "", false, false) } // readTaskPositionInTx reads a task's currently-persisted position inside @@ -1114,6 +1092,68 @@ func (r *Repository) readTaskPositionInTx(ctx context.Context, tx *sql.Tx, taskI return position, true, nil } +// applyPreservedPositionInTx overwrites task.Position with the value read +// fresh inside this transaction when preservePosition is set, so a caller's +// task object read before the transaction began does not clobber a +// concurrent reorder or arrival's position with a stale one. +func (r *Repository) applyPreservedPositionInTx(ctx context.Context, tx *sql.Tx, task *models.Task, preservePosition bool) error { + if !preservePosition { + return nil + } + currentPosition, positionFound, err := r.readTaskPositionInTx(ctx, tx, task.ID) + if err != nil { + return err + } + if positionFound { + task.Position = currentPosition + } + return nil +} + +// buildTaskUpdateQuery builds updateTaskTx's UPDATE statement and the +// metadata payload it binds. protectDeferredLaunch strips deferred_launch +// from the payload regardless of which query shape below owns the write: a +// key absent from the patch document leaves the row's own current value in +// place for both merge mechanisms (json_patch/jsonb `||` in the +// title-pending branch below, and the explicit splice +// protectedTaskMetadataMergeExpression performs for the plain-replace +// branch), so a stale in-memory snapshot can never resurrect or clobber +// whatever the session ceiling's own CAS writers did to that key in the +// meantime. +func (r *Repository) buildTaskUpdateQuery( + task *models.Task, metadata []byte, protectDeferredLaunch bool, +) (query string, finalMetadata []byte, err error) { + metadataExpr := "?" + if protectDeferredLaunch { + stripped, marshalErr := json.Marshal(stripProtectedTaskMetadata(task.Metadata)) + if marshalErr != nil { + return "", nil, marshalErr + } + metadata = stripped + if !models.IsAgentTitlePending(task.Metadata) { + metadataExpr = protectedTaskMetadataMergeExpression(r.db.DriverName()) + } + } + updateQuery := fmt.Sprintf(` + UPDATE tasks SET workspace_id = ?, workflow_id = ?, workflow_step_id = ?, title = ?, description = ?, state = ?, priority = ?, position = ?, wip_admitted = ?, queued_for_step_id = ?, queued_at = ?, metadata = %s, parent_id = ?, updated_at = ?, origin = ?, project_id = ?, labels = ?, identifier = ?, assignee_user_id = ? + WHERE id = ? + `, metadataExpr) + if models.IsAgentTitlePending(task.Metadata) { + pending := agentTitlePendingPredicate(r.db.DriverName()) + metadataMerge := pendingTaskMetadataMergeExpression(r.db.DriverName()) + updateQuery = fmt.Sprintf(` + UPDATE tasks SET workspace_id = ?, workflow_id = ?, workflow_step_id = ?, + title = CASE WHEN %s THEN ? ELSE title END, + description = ?, state = ?, priority = ?, position = ?, wip_admitted = ?, + queued_for_step_id = ?, queued_at = ?, + metadata = %s, + parent_id = ?, updated_at = ?, origin = ?, project_id = ?, labels = ?, identifier = ?, assignee_user_id = ? + WHERE id = ? + `, pending, metadataMerge) + } + return updateQuery, metadata, nil +} + // updateTaskTx writes the full task row. preservePosition, true for every // caller except the one path that honors an explicit caller-supplied // position (UpdateTaskWithExplicitPosition), overwrites task.Position with @@ -1121,8 +1161,11 @@ func (r *Repository) readTaskPositionInTx(ctx context.Context, tx *sql.Tx, taskI // in the struct: a caller's task object was read before this transaction // began, and by the time this write commits a concurrent reorder or arrival // (AC-TASKS-KANBAN-TASK-REORDERING-001.28) may have moved the row to a -// different position that this write must not clobber. -func (r *Repository) updateTaskTx(ctx context.Context, tx *sql.Tx, task *models.Task, metadata []byte, expectedWorkflowID string, preservePosition bool) (entryID string, markerEntryID int64, err error) { +// different position that this write must not clobber. protectDeferredLaunch, +// set only by UpdateTaskPreservingDeferredLaunch, re-merges the row's own +// current deferred_launch value into the write so a stale in-memory snapshot +// can never resurrect or clobber a concurrent session-ceiling CAS write. +func (r *Repository) updateTaskTx(ctx context.Context, tx *sql.Tx, task *models.Task, metadata []byte, expectedWorkflowID string, preservePosition, protectDeferredLaunch bool) (entryID string, markerEntryID int64, err error) { fromWorkflowID, fromStepID, found, err := r.readTaskStepInTx(ctx, tx, task.ID) if err != nil { return "", 0, err @@ -1136,14 +1179,8 @@ func (r *Repository) updateTaskTx(ctx context.Context, tx *sql.Tx, task *models. // ladder over every other case (design's error-mapping table). return "", 0, fmt.Errorf("%w: %s", ErrTaskNotFound, task.ID) } - if preservePosition { - currentPosition, positionFound, posErr := r.readTaskPositionInTx(ctx, tx, task.ID) - if posErr != nil { - return "", 0, posErr - } - if positionFound { - task.Position = currentPosition - } + if err := r.applyPreservedPositionInTx(ctx, tx, task, preservePosition); err != nil { + return "", 0, err } if expectedWorkflowID != "" && fromWorkflowID != expectedWorkflowID { // Checked here, immediately before the UPDATE below and using the @@ -1164,22 +1201,9 @@ func (r *Repository) updateTaskTx(ctx context.Context, tx *sql.Tx, task *models. // was invisible until exercised against Postgres with real concurrency. task.UpdatedAt = r.nowUTC() - updateQuery := ` - UPDATE tasks SET workspace_id = ?, workflow_id = ?, workflow_step_id = ?, title = ?, description = ?, state = ?, priority = ?, position = ?, wip_admitted = ?, queued_for_step_id = ?, queued_at = ?, metadata = ?, parent_id = ?, updated_at = ?, origin = ?, project_id = ?, labels = ?, identifier = ?, assignee_user_id = ? - WHERE id = ? - ` - if models.IsAgentTitlePending(task.Metadata) { - pending := agentTitlePendingPredicate(r.db.DriverName()) - metadataMerge := pendingTaskMetadataMergeExpression(r.db.DriverName()) - updateQuery = fmt.Sprintf(` - UPDATE tasks SET workspace_id = ?, workflow_id = ?, workflow_step_id = ?, - title = CASE WHEN %s THEN ? ELSE title END, - description = ?, state = ?, priority = ?, position = ?, wip_admitted = ?, - queued_for_step_id = ?, queued_at = ?, - metadata = %s, - parent_id = ?, updated_at = ?, origin = ?, project_id = ?, labels = ?, identifier = ?, assignee_user_id = ? - WHERE id = ? - `, pending, metadataMerge) + updateQuery, metadata, err := r.buildTaskUpdateQuery(task, metadata, protectDeferredLaunch) + if err != nil { + return "", 0, err } result, err := tx.ExecContext(ctx, r.db.Rebind(updateQuery), task.WorkspaceID, task.WorkflowID, task.WorkflowStepID, task.Title, task.Description, task.State, task.Priority, task.Position, dialect.BoolToInt(task.WIPAdmitted), task.QueuedForStepID, task.QueuedAt, string(metadata), task.ParentID, task.UpdatedAt, task.Origin, task.ProjectID, task.Labels, task.Identifier, task.AssigneeUserID, task.ID) if err != nil { @@ -1343,7 +1367,7 @@ func (r *Repository) MarkDeferredMoveAppliedForSession( if err != nil { return false, err } - if _, _, err := r.updateTaskTx(ctx, tx, task, metadata, "", true); err != nil { + if _, _, err := r.updateTaskTx(ctx, tx, task, metadata, "", true, false); err != nil { return false, err } if err := r.deleteDeferredMoveGuardTx(ctx, tx, record); err != nil { @@ -1712,7 +1736,7 @@ func (r *Repository) updateTaskWithWorkflowStepAdmissionAttempt( if err != nil { metadata = []byte("{}") } - entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, expectedWorkflowID, false) + entryID, markerEntryID, err := r.updateTaskTx(ctx, tx, task, metadata, expectedWorkflowID, false, false) if err != nil { return false, false, err } @@ -2213,6 +2237,71 @@ func pendingTaskMetadataMergeExpression(driver string) string { return "json_patch(CASE WHEN metadata IS NULL OR metadata = 'null' OR metadata = '' THEN '{}' ELSE metadata END, json_remove(?, '$.agent_title_pending', '$.agent_title_owner_session_id'))" } +// stripProtectedTaskMetadata returns a shallow clone of metadata with +// deferred_launch removed, for updateTaskTx's protectDeferredLaunch mode +// (UpdateTaskPreservingDeferredLaunch), applied to the write payload +// regardless of which of the two query shapes below owns the write. The key +// is owned by its own compare-and-set writers (the session ceiling's +// admission machinery) and can change between the moment a request-scoped +// caller read this snapshot and the moment this write commits. A key absent +// from the payload leaves the row's own current value in place under both +// merge mechanisms: protectedTaskMetadataMergeExpression's explicit splice +// for the plain-replace shape, and pendingTaskMetadataMergeExpression's +// json_patch/jsonb `||` merge for the agent-title-pending shape. Either way +// a stale snapshot can never resurrect or clobber whatever the ceiling's own +// writers did to the key in between. +// +// step_handoff_carry is deliberately NOT included here even though +// service_task_metadata.go's protectedTaskMetadataUpdate treats it the same +// way deferred_launch is treated at the HTTP PATCH boundary: unlike +// deferred_launch, step_handoff_carry has a legitimate direct write path +// through the ordinary in-memory task.Metadata + UpdateTask sequence +// (event_handlers_workflow.go's step-transition handling), not only a CAS +// primitive, so protecting it here would silently drop that write. +func stripProtectedTaskMetadata(metadata map[string]interface{}) map[string]interface{} { + // Always returns a non-nil map, even for nil input: json.Marshal of a nil + // map produces the JSON scalar `null`, and Postgres's jsonb `||` merge + // expression below concatenates a scalar with an object into a + // two-element array instead of merging, corrupting the metadata column. + // A nil range is a no-op, so this still yields "{}" for nil input. + cloned := make(map[string]interface{}, len(metadata)) + for key, value := range metadata { + if key == models.MetaKeyDeferredLaunch { + continue + } + cloned[key] = value + } + return cloned +} + +// protectedTaskMetadataMergeExpression is updateTaskTx's metadata write when +// protectDeferredLaunch is set (UpdateTaskPreservingDeferredLaunch). The +// payload (?, already stripped of deferred_launch by +// stripProtectedTaskMetadata) is the write's target — every other key +// behaves exactly like the historical plain replace, including deletion by +// omission — and only deferred_launch is patched back in from the row's own +// pre-write value, so a stale in-memory snapshot can never resurrect or +// clobber whatever the row's own CAS writers (the session ceiling's +// admission machinery) did to it in the meantime. A protected key absent +// from the current row must end up absent from the result too, not present +// with a JSON null — SQLite's json_patch treats a null overlay value as +// "remove if present, otherwise no-op" (RFC 7396), which is exactly that; +// Postgres has no such rule for its jsonb `||` operator, so +// jsonb_strip_nulls first removes the key jsonb_build_object had to +// represent as JSON null because the row does not have it. +func protectedTaskMetadataMergeExpression(driver string) string { + if dialect.IsPostgres(driver) { + return fmt.Sprintf( + "(?::jsonb || jsonb_strip_nulls(jsonb_build_object('%s', (%s)->'%s')))::text", + models.MetaKeyDeferredLaunch, postgresMetadataObject, models.MetaKeyDeferredLaunch, + ) + } + return fmt.Sprintf( + "json_patch(?, json_object('%s', json_extract(%s, '$.%s')))", + models.MetaKeyDeferredLaunch, sqliteMetadataObject, models.MetaKeyDeferredLaunch, + ) +} + // DetachTask clears only the hierarchy fields involved in detachment. Keeping // this as a targeted update prevents concurrent task edits from being replaced // by a stale full-row write. @@ -3215,6 +3304,42 @@ func (r *Repository) ListTasksWithMetadataKey(ctx context.Context, key string) ( return r.scanTasks(rows) } +// ListTasksWithCeilingDeferred returns every task carrying a ceiling_deferred +// record — archived, ephemeral and automation-origin tasks included, because +// AC-17b requires the sweep to find and drop archived/cancelled records and +// AC-2/AC-2a require ephemeral and automation-origin tasks to participate on +// the same terms as any other. It deliberately does not reuse +// ListTasksWithMetadataKey, whose WHERE excludes exactly those tasks. +// Ordered by id ascending alone (AC-50b, AC-50c) — no timestamp term, so an +// unrelated write to a deferred task cannot move it in the drain order +// between ticks, unlike ListTasksWithMetadataKey's three-column sort. +// +// The predicate matches on value equality, not mere key presence: a +// key-presence test would also match a record left behind with the flag +// explicitly false. +func (r *Repository) ListTasksWithCeilingDeferred(ctx context.Context) ([]*models.Task, error) { + var predicate string + var args []interface{} + if dialect.IsPostgres(r.ro.DriverName()) { + predicate = "jsonb_extract_path(CASE WHEN t.metadata IS NULL OR t.metadata = 'null' OR t.metadata = '' THEN '{}'::jsonb ELSE t.metadata::jsonb END, ?, ?) = 'true'::jsonb" + args = []interface{}{models.MetaKeyDeferredLaunch, models.CeilingDeferredKey} + } else { + predicate = "json_type(CASE WHEN t.metadata IS NULL OR t.metadata = 'null' OR t.metadata = '' THEN '{}' ELSE t.metadata END, ?) = 'true'" + args = []interface{}{jsonPath(models.MetaKeyDeferredLaunch + "." + models.CeilingDeferredKey)} + } + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(` + SELECT `+taskSelectColumns("t")+` + FROM tasks t + WHERE `+predicate+` + ORDER BY t.id ASC + `), args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + return r.scanTasks(rows) +} + // ListTasksByWorkspace returns paginated tasks for a workspace with total count // If query is non-empty, filters by task title, description, repository name, or repository path // If includeArchived is false, archived tasks are excluded. If onlyArchived diff --git a/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_postgres_test.go b/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_postgres_test.go new file mode 100644 index 0000000000..dd131e86df --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_postgres_test.go @@ -0,0 +1,69 @@ +package sqlite + +// Postgres parity coverage for updateTaskTx's protectDeferredLaunch metadata +// merge (UpdateTaskPreservingDeferredLaunch), whose `?::jsonb || +// jsonb_strip_nulls(...)` expression is dialect-sensitive: the JSON payload +// marshaled from a nil in-memory Task.Metadata is the literal scalar `null`, +// and Postgres's jsonb `||` concatenates a scalar with an object into a +// two-element array rather than merging into one, corrupting the metadata +// column. Skips unless KANDEV_TEST_POSTGRES_DSN is set; CI runs these in +// postgres-boot. + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresUpdateTaskPreservingDeferredLaunchWithNilMetadataStaysAnObject +// pins the Postgres-only counterpart of +// TestUpdateTaskPreservingDeferredLaunchWithNilMetadataStaysAnObject: a task +// update carrying nil in-memory Metadata must still leave the row's metadata +// column a well-formed JSON object with the row's own deferred_launch value +// intact, not a corrupted two-element array. +func TestPostgresUpdateTaskPreservingDeferredLaunchWithNilMetadataStaysAnObject(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + const taskID = "task-nil-metadata-pg" + seedPostgresTask(t, repo, taskID) + + stored, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, AbsentDeferredLaunch(), + map[string]interface{}{"prompt": "original"}) + if err != nil || lostCompare || !stored { + t.Fatalf("seed deferred launch: stored=%v lostCompare=%v err=%v", stored, lostCompare, err) + } + + task, err := repo.GetTask(ctx, taskID) + if err != nil { + t.Fatalf("load task: %v", err) + } + task.Metadata = nil + task.Description = "updated with nil metadata" + task.UpdatedAt = time.Now().UTC() + if err := repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { + t.Fatalf("UpdateTaskPreservingDeferredLaunch with nil metadata: %v", err) + } + + current, err := repo.GetTask(ctx, taskID) + if err != nil { + t.Fatalf("reload task: %v", err) + } + if current.Description != "updated with nil metadata" { + t.Fatalf("description = %q, want the nil-metadata update to still apply", current.Description) + } + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok { + t.Fatalf("deferred_launch missing or wrong shape after a nil-metadata update (metadata column corrupted?): %#v", + current.Metadata) + } + if deferred["prompt"] != "original" { + t.Fatalf("deferred_launch = %#v, want the row's own value preserved", deferred) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_test.go b/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_test.go new file mode 100644 index 0000000000..415cf0311c --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/task_deferred_launch_protected_test.go @@ -0,0 +1,219 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestUpdateTaskPreservesWinningDeferredLaunchAgainstStaleUpdate pins the +// same race TestUpdateTaskPreservesWinningTitleAgainstStaleUpdate already +// pins for agent_title_pending, but for deferred_launch: the session ceiling's +// own compare-and-set writers (SetTaskDeferredLaunchIfUnchanged) own that key, +// so the service layer's request-driven UpdateTaskPreservingDeferredLaunch — +// started before their write and committed after it, still holding the +// pre-CAS in-memory snapshot — must not resurrect or clobber whatever they +// wrote. Before this fix, that path wrote the whole metadata column verbatim +// from the stale snapshot. Plain UpdateTask deliberately keeps its old +// unprotected behavior: many orchestrator-internal callers read a task fresh +// immediately before mutating deferred_launch themselves and writing it back, +// and are not racing the ceiling's CAS writers the way a request-scoped +// snapshot can. +func TestUpdateTaskPreservesWinningDeferredLaunchAgainstStaleUpdate(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-deferred-race") + if _, err := repo.db.ExecContext(ctx, ` + UPDATE tasks SET metadata = ? WHERE id = ? + `, `{"deferred_launch":{"prompt":"original","ceiling_launch_kind":"start_created"}}`, "task-deferred-race"); err != nil { + t.Fatalf("seed deferred launch: %v", err) + } + + stale, err := repo.GetTask(ctx, "task-deferred-race") + if err != nil { + t.Fatalf("load stale task: %v", err) + } + + // Models the ceiling machinery's own CAS write landing after the stale + // read above but before the ordinary update below commits. + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "task-deferred-race") + if err != nil { + t.Fatalf("read deferred launch prior: %v", err) + } + stored, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "task-deferred-race", prior, + map[string]interface{}{"prompt": "original", "ceiling_launch_kind": "start_created", "ceiling_deferred": true}) + if err != nil || lostCompare || !stored { + t.Fatalf("simulate concurrent ceiling write: stored=%v lostCompare=%v err=%v", stored, lostCompare, err) + } + + // This models a request-scoped task update that started before the CAS + // write and commits after it, still holding the pre-CAS deferred_launch + // value. + stale.Description = "updated concurrently" + stale.Metadata["stale_change"] = "retained" + if err := repo.UpdateTaskPreservingDeferredLaunch(ctx, stale); err != nil { + t.Fatalf("stale UpdateTaskPreservingDeferredLaunch: %v", err) + } + + current, err := repo.GetTask(ctx, "task-deferred-race") + if err != nil { + t.Fatalf("reload task: %v", err) + } + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok { + t.Fatalf("deferred_launch missing or wrong shape: %#v", current.Metadata[models.MetaKeyDeferredLaunch]) + } + if deferred["ceiling_deferred"] != true { + t.Fatalf("deferred_launch = %#v, want the concurrent CAS winner retained", deferred) + } + if current.Description != "updated concurrently" { + t.Fatalf("description = %q, want stale update to retain its unrelated change", current.Description) + } + if current.Metadata["stale_change"] != "retained" { + t.Fatalf("metadata = %#v, want unrelated stale metadata change retained", current.Metadata) + } +} + +// TestUpdateTaskPreservesWinningDeferredLaunchAgainstStaleUpdateWhileTitlePending +// pins the same race as TestUpdateTaskPreservesWinningDeferredLaunchAgainstStaleUpdate, +// but for a task that also carries agent_title_pending — the normal shape of a +// freshly created task whose auto-start immediately hit the session ceiling. +// updateTaskTx has a second query shape for this case (the title-pending +// merge-patch), and it must protect deferred_launch exactly like the plain +// case: the ceiling's own CAS write (here, a replay stripping ceiling_deferred +// entirely) must survive a concurrently-committing stale snapshot that still +// holds the pre-strip value. +func TestUpdateTaskPreservesWinningDeferredLaunchAgainstStaleUpdateWhileTitlePending(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-deferred-title-pending-race") + if _, err := repo.db.ExecContext(ctx, ` + UPDATE tasks SET metadata = ? WHERE id = ? + `, `{"agent_title_pending":true,"agent_title_owner_session_id":"session-owner","deferred_launch":{"prompt":"original","ceiling_launch_kind":"start_created","ceiling_deferred":true}}`, "task-deferred-title-pending-race"); err != nil { + t.Fatalf("seed pending title + deferred launch: %v", err) + } + + stale, err := repo.GetTask(ctx, "task-deferred-title-pending-race") + if err != nil { + t.Fatalf("load stale task: %v", err) + } + + // Models a replay attempt: the ceiling's own CAS writer strips + // ceiling_deferred (key removed, not set false) before re-attempting the + // launch. + _, prior, err := repo.GetTaskDeferredLaunch(ctx, "task-deferred-title-pending-race") + if err != nil { + t.Fatalf("read deferred launch prior: %v", err) + } + stored, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "task-deferred-title-pending-race", prior, + map[string]interface{}{"prompt": "original", "ceiling_launch_kind": "start_created"}) + if err != nil || lostCompare || !stored { + t.Fatalf("simulate concurrent ceiling replay strip: stored=%v lostCompare=%v err=%v", stored, lostCompare, err) + } + + // The stale request-scoped update commits after the strip, still holding + // agent_title_pending:true (routing it through the title-pending query + // shape) and the pre-strip deferred_launch value. + stale.Description = "updated concurrently" + if err := repo.UpdateTaskPreservingDeferredLaunch(ctx, stale); err != nil { + t.Fatalf("stale UpdateTaskPreservingDeferredLaunch: %v", err) + } + + current, err := repo.GetTask(ctx, "task-deferred-title-pending-race") + if err != nil { + t.Fatalf("reload task: %v", err) + } + deferred, _ := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if deferred["ceiling_deferred"] == true { + t.Fatalf("deferred_launch = %#v, want ceiling_deferred to stay stripped, not resurrected by the stale title-pending write", deferred) + } + if current.Description != "updated concurrently" { + t.Fatalf("description = %q, want stale update to retain its unrelated change", current.Description) + } +} + +// TestUpdateTaskStillDeletesUnrelatedMetadataKeysByOmission guards the fix +// above against the regression it could easily introduce: protecting +// deferred_launch must not turn UpdateTaskPreservingDeferredLaunch's metadata +// write into a general merge. Every other key must keep behaving like a +// plain replace, including a caller deleting one by simply omitting it — +// the pattern service_tasks.go and the workflow engine rely on throughout +// (agent_title_pending, workflow_move_pending, queued_move_exit_pending, …). +func TestUpdateTaskStillDeletesUnrelatedMetadataKeysByOmission(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-omission-delete") + if _, err := repo.db.ExecContext(ctx, ` + UPDATE tasks SET metadata = ? WHERE id = ? + `, `{"some_pending_marker":true,"keep":"value"}`, "task-omission-delete"); err != nil { + t.Fatalf("seed metadata: %v", err) + } + + task, err := repo.GetTask(ctx, "task-omission-delete") + if err != nil { + t.Fatalf("load task: %v", err) + } + delete(task.Metadata, "some_pending_marker") + if err := repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { + t.Fatalf("UpdateTaskPreservingDeferredLaunch: %v", err) + } + + current, err := repo.GetTask(ctx, "task-omission-delete") + if err != nil { + t.Fatalf("reload task: %v", err) + } + if _, present := current.Metadata["some_pending_marker"]; present { + t.Fatalf("metadata = %#v, want the omitted key deleted, not preserved", current.Metadata) + } + if current.Metadata["keep"] != "value" { + t.Fatalf("metadata = %#v, want the untouched key preserved", current.Metadata) + } +} + +// TestUpdateTaskPreservingDeferredLaunchWithNilMetadataStaysAnObject pins the +// nil in-memory Task.Metadata edge case: stripProtectedTaskMetadata used to +// return a literal nil map for a nil input, which json.Marshal encodes as the +// JSON scalar `null` rather than `{}`. On the Postgres dialect the merge +// expression's `?::jsonb || jsonb_strip_nulls(...)` concatenates a jsonb +// scalar null with an object, producing a two-element JSON array +// (`[null, {...}]`) instead of an object — corrupting the metadata column. +// SQLite's json_patch tolerates a null target and was never actually broken, +// but this pins the shared code path on the dialect that is exercised in +// every environment, so a regression here is caught before the +// Postgres-only counterpart even runs. +func TestUpdateTaskPreservingDeferredLaunchWithNilMetadataStaysAnObject(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-nil-metadata") + if _, err := repo.db.ExecContext(ctx, ` + UPDATE tasks SET metadata = ? WHERE id = ? + `, `{"deferred_launch":{"prompt":"original"}}`, "task-nil-metadata"); err != nil { + t.Fatalf("seed deferred launch: %v", err) + } + + task, err := repo.GetTask(ctx, "task-nil-metadata") + if err != nil { + t.Fatalf("load task: %v", err) + } + task.Metadata = nil + task.Description = "updated with nil metadata" + if err := repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { + t.Fatalf("UpdateTaskPreservingDeferredLaunch with nil metadata: %v", err) + } + + current, err := repo.GetTask(ctx, "task-nil-metadata") + if err != nil { + t.Fatalf("reload task: %v", err) + } + if current.Description != "updated with nil metadata" { + t.Fatalf("description = %q, want the nil-metadata update to still apply", current.Description) + } + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + if !ok { + t.Fatalf("deferred_launch missing or wrong shape after a nil-metadata update: %#v", current.Metadata) + } + if deferred["prompt"] != "original" { + t.Fatalf("deferred_launch = %#v, want the row's own value preserved", deferred) + } +} diff --git a/apps/backend/internal/task/service/handoff_cascade.go b/apps/backend/internal/task/service/handoff_cascade.go index 7ba68cbd4b..60a63dbf2a 100644 --- a/apps/backend/internal/task/service/handoff_cascade.go +++ b/apps/backend/internal/task/service/handoff_cascade.go @@ -1326,6 +1326,18 @@ func (s *HandoffService) finalizeActiveSessions( if publisher, ok := s.eventPublisher.(taskSessionCancellationPublisher); ok { publisher.PublishTaskSessionsCancelled(finalizeCtx, taskID, cancelled, reason) } + // Bulk writer: every returned row is released unconditionally rather than + // branched on state, since RETURNING reports post-update CANCELLED for all + // of them (AC-51e). Releasing an id that held no reservation is a defined + // no-op. + if s.sessionCeilingReleaser != nil { + for _, session := range cancelled { + if session == nil || session.ID == "" { + continue + } + s.sessionCeilingReleaser.ReleaseCeilingReservation(session.ID) + } + } return nil } diff --git a/apps/backend/internal/task/service/handoff_service.go b/apps/backend/internal/task/service/handoff_service.go index 17228507e6..a6a07b4d55 100644 --- a/apps/backend/internal/task/service/handoff_service.go +++ b/apps/backend/internal/task/service/handoff_service.go @@ -260,26 +260,27 @@ type RelatedTasks struct { // than reaching into the repos directly so document writes still go // through DocumentService and emit the same revision/event side effects. type HandoffService struct { - tasks repository.TaskRepository - docs *DocumentService - docsRepo repository.DocumentRepository - blockers BlockerRepository - wsGroups WorkspaceGroupRepo - sessions SessionWorktreeReader - cleaner WorkspaceCleaner - runCanceller RunCanceller - gitArchiveCapture GitArchiveCapture - eventPublisher TaskEventPublisher - vacancyReconciler VacatedStepReconciler - resourceCleaner TaskResourceCleaner - taskAccessCheck func(ctx context.Context, taskID string) error - comments CommentReader - logger *logger.Logger - parentLock parentMutex - archiveCascadeLock parentMutex - partialArchiveMu sync.Mutex - partialArchiveIDs map[string]string - workspaceGroupLock parentMutex + tasks repository.TaskRepository + docs *DocumentService + docsRepo repository.DocumentRepository + blockers BlockerRepository + wsGroups WorkspaceGroupRepo + sessions SessionWorktreeReader + cleaner WorkspaceCleaner + runCanceller RunCanceller + gitArchiveCapture GitArchiveCapture + eventPublisher TaskEventPublisher + vacancyReconciler VacatedStepReconciler + resourceCleaner TaskResourceCleaner + taskAccessCheck func(ctx context.Context, taskID string) error + comments CommentReader + sessionCeilingReleaser SessionCeilingReleaser + logger *logger.Logger + parentLock parentMutex + archiveCascadeLock parentMutex + partialArchiveMu sync.Mutex + partialArchiveIDs map[string]string + workspaceGroupLock parentMutex } // TaskEventPublisher abstracts the side-effect of broadcasting task @@ -356,6 +357,13 @@ func (s *HandoffService) SetVacatedStepReconciler(r VacatedStepReconciler) { s.vacancyReconciler = r } +// SetSessionCeilingReleaser wires session-ceiling release (orchestrator) for +// finalizeActiveSessions' bulk cancellation, which never passes through the +// orchestrator's own persistence funnels. +func (s *HandoffService) SetSessionCeilingReleaser(releaser SessionCeilingReleaser) { + s.sessionCeilingReleaser = releaser +} + // TaskResourceCleaner tears down a task's runtime resources (container, // sandbox, worktree, executor_running rows, quick-chat dir, task_environment // row) AFTER the cascade has stamped the task's DB row. The cascade paths diff --git a/apps/backend/internal/task/service/handoff_workspace_test.go b/apps/backend/internal/task/service/handoff_workspace_test.go index 9ff5fbd261..aa1cc10b8f 100644 --- a/apps/backend/internal/task/service/handoff_workspace_test.go +++ b/apps/backend/internal/task/service/handoff_workspace_test.go @@ -694,6 +694,20 @@ func (r *phase4TaskRepo) UpdateTaskWithExplicitPosition(context.Context, *models r.panicNotUsed("UpdateTaskWithExplicitPosition") return nil } +func (r *phase4TaskRepo) UpdateTaskPreservingDeferredLaunch(context.Context, *models.Task) error { + r.panicNotUsed("UpdateTaskPreservingDeferredLaunch") + return nil +} +func (r *phase4TaskRepo) GetTaskDeferredLaunch(context.Context, string) (map[string]interface{}, interface{}, error) { + r.panicNotUsed("GetTaskDeferredLaunch") + return nil, nil, nil +} +func (r *phase4TaskRepo) SetTaskDeferredLaunchIfUnchanged( + context.Context, string, interface{}, map[string]interface{}, +) (bool, bool, error) { + r.panicNotUsed("SetTaskDeferredLaunchIfUnchanged") + return false, false, nil +} func (r *phase4TaskRepo) DeleteTask(context.Context, string) error { r.panicNotUsed("DeleteTask") return nil diff --git a/apps/backend/internal/task/service/service.go b/apps/backend/internal/task/service/service.go index 9a307be5b1..76e402ef40 100644 --- a/apps/backend/internal/task/service/service.go +++ b/apps/backend/internal/task/service/service.go @@ -149,6 +149,15 @@ type ParkedProjectionCanceller interface { ClearParkedProjectionOnSessionTerminated(ctx context.Context, taskID, sessionID string, newState models.TaskSessionState) } +// SessionCeilingReleaser releases the orchestrator's session-ceiling +// reservation for a session cancelled through a task service-owned bulk path +// (archive's batch session cancellation, delete's cascaded session removal), +// neither of which passes through the orchestrator's own persistence +// funnels. Releasing a session that held no reservation is a defined no-op. +type SessionCeilingReleaser interface { + ReleaseCeilingReservation(sessionID string) +} + // TaskRowLivenessProber classifies an executors_running row's backing-process // liveness in a runtime-aware way (a local process check is never applied to a // remote/SSH row). It is optional and satisfied by the lifecycle adapter. When @@ -422,6 +431,7 @@ type Service struct { executionStopper TaskExecutionStopper clarificationCanceller TerminalClarificationCanceller parkedProjectionCanceller ParkedProjectionCanceller + sessionCeilingReleaser SessionCeilingReleaser rowLivenessProber TaskRowLivenessProber contextWindowResetter func(context.Context, string) error cleanupActivity TaskResourceCleanupActivityGate @@ -775,6 +785,12 @@ func (s *Service) SetParkedProjectionCanceller(canceller ParkedProjectionCancell s.parkedProjectionCanceller = canceller } +// SetSessionCeilingReleaser wires session-ceiling release (orchestrator) for +// the same task service-owned bulk paths SetParkedProjectionCanceller covers. +func (s *Service) SetSessionCeilingReleaser(releaser SessionCeilingReleaser) { + s.sessionCeilingReleaser = releaser +} + // SetRowLivenessProber wires the runtime-aware executors_running liveness probe // (satisfied by the lifecycle adapter). It is optional; when unwired, cleanup // treats every row as Unknown. diff --git a/apps/backend/internal/task/service/service_deferred_launch.go b/apps/backend/internal/task/service/service_deferred_launch.go index a3fd54c53a..4f9b0f8cdf 100644 --- a/apps/backend/internal/task/service/service_deferred_launch.go +++ b/apps/backend/internal/task/service/service_deferred_launch.go @@ -9,6 +9,12 @@ import ( "github.com/kandev/kandev/internal/task/models" ) +// deferredLaunchPromptCASRetryBudget bounds the read-compare-write retry for +// a concurrent, legitimate deferred_launch writer (the session ceiling's own +// admission machinery) mutating a different field while this edit is +// in flight. Mirrors the orchestrator's own deferredLaunchCASRetryBudget. +const deferredLaunchPromptCASRetryBudget = 3 + var ( // ErrNoPendingDeferredLaunch means the task carries no deferred launch // record to edit. Either it never had one, or it has already been consumed @@ -38,7 +44,7 @@ var ( // the record itself is gone (consumed by the gate or by a direct start), or a // session already exists on the task. // -// # Why the write is a conditional single-key patch +// # Why the write is a value-compared, retried single-key patch // // The obvious implementation — read the task, check it has no session, write // the task back — reintroduces the very bug this file exists to fix. A start @@ -47,11 +53,19 @@ var ( // metadata it read before, RESURRECTS the key. The gate fires a second session // on a task that is already running, which is exactly the production symptom. // -// So the write goes through SetTaskMetadataKeyIfPresent, which patches the one -// key and only while it is still there. A concurrent start wins the race and -// this returns ErrDeferredLaunchAlreadyStarted, with nothing written. The -// session check stays because it gives the common case a precise error instead -// of a bare conflict; it is not what makes the update safe. +// So the write goes through GetTaskDeferredLaunch/SetTaskDeferredLaunchIfUnchanged, +// the session ceiling's own compare-and-set protocol, rather than +// SetTaskMetadataKeyIfPresent: presence-only is not enough, because the ceiling's +// admission machinery (for example the replay sweeper) can legitimately rewrite +// other fields on this same record while this edit is in flight, and a +// presence-only write would silently overwrite that change with the stale copy +// this function read earlier. A lost comparison against an unchanged prior is +// retried, bounded by deferredLaunchPromptCASRetryBudget; a lost comparison +// against a record the retry finds gone means a concurrent start consumed it, +// which reports the same ErrDeferredLaunchAlreadyStarted a presence-only write +// would have. The session check stays because it gives the common case a +// precise error instead of a bare conflict; it is not what makes the update +// safe. func (s *Service) UpdateDeferredLaunchPrompt(ctx context.Context, taskID, prompt string) (*models.Task, error) { if err := s.authorizeTaskID(ctx, taskID); err != nil { return nil, err @@ -59,14 +73,18 @@ func (s *Service) UpdateDeferredLaunchPrompt(ctx context.Context, taskID, prompt if strings.TrimSpace(prompt) == "" { return nil, ErrDeferredLaunchPromptEmpty } - task, err := s.tasks.GetTask(ctx, taskID) + launch, prior, err := s.tasks.GetTaskDeferredLaunch(ctx, taskID) if err != nil { return nil, err } - launch, ok := pendingDeferredLaunch(task) - if !ok { + if launch == nil { return nil, ErrNoPendingDeferredLaunch } + // The session check runs after establishing the record is present, so a + // concurrent start's atomic claim landing during this call (rather than + // before this function started) is caught below by the compare-and-set + // losing against the prior this function already captured, not + // misreported as "never had one". started, err := s.taskHasAnySession(ctx, taskID) if err != nil { return nil, err @@ -75,36 +93,43 @@ func (s *Service) UpdateDeferredLaunchPrompt(ctx context.Context, taskID, prompt return nil, ErrDeferredLaunchAlreadyStarted } - updated := make(map[string]interface{}, len(launch)+1) - maps.Copy(updated, launch) - updated["prompt"] = prompt - written, err := s.tasks.SetTaskMetadataKeyIfPresent(ctx, taskID, models.MetaKeyDeferredLaunch, updated) - if err != nil { - return nil, err + for attempt := 0; attempt < deferredLaunchPromptCASRetryBudget; attempt++ { + updated := make(map[string]interface{}, len(launch)+1) + maps.Copy(updated, launch) + updated["prompt"] = prompt + if payload, ok := launch[models.CeilingLaunchPayloadKey].(map[string]interface{}); ok { + updatedPayload := make(map[string]interface{}, len(payload)+1) + maps.Copy(updatedPayload, payload) + updatedPayload["prompt"] = prompt + updated[models.CeilingLaunchPayloadKey] = updatedPayload + } + stored, lostCompare, err := s.tasks.SetTaskDeferredLaunchIfUnchanged(ctx, taskID, prior, updated) + if err != nil { + return nil, err + } + if stored { + // Re-read rather than returning the patched in-memory copy: the row + // now carries whatever else changed while this ran, and handing + // back a stale snapshot is how the resurrection bug got in. + task, err := s.tasks.GetTask(ctx, taskID) + if err != nil { + return nil, err + } + s.PublishTaskUpdated(ctx, task) + return task, nil + } + if !lostCompare { + return nil, ErrDeferredLaunchAlreadyStarted + } + launch, prior, err = s.tasks.GetTaskDeferredLaunch(ctx, taskID) + if err != nil { + return nil, err + } + if launch == nil { + return nil, ErrDeferredLaunchAlreadyStarted + } } - if !written { - return nil, ErrDeferredLaunchAlreadyStarted - } - - // Re-read rather than returning the patched in-memory copy: the row now - // carries whatever else changed while this ran, and handing back a stale - // snapshot is how the resurrection bug got in. - stored, err := s.tasks.GetTask(ctx, taskID) - if err != nil { - return nil, err - } - s.PublishTaskUpdated(ctx, stored) - return stored, nil -} - -// pendingDeferredLaunch returns the task's deferred launch record when it is -// present and well-formed. -func pendingDeferredLaunch(task *models.Task) (map[string]interface{}, bool) { - if task == nil || task.Metadata == nil { - return nil, false - } - launch, ok := task.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) - return launch, ok + return nil, ErrDeferredLaunchAlreadyStarted } // taskHasAnySession reports whether any session row exists for the task, diff --git a/apps/backend/internal/task/service/service_deferred_launch_test.go b/apps/backend/internal/task/service/service_deferred_launch_test.go index 416ee1305d..f32a367793 100644 --- a/apps/backend/internal/task/service/service_deferred_launch_test.go +++ b/apps/backend/internal/task/service/service_deferred_launch_test.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "fmt" "testing" "github.com/kandev/kandev/internal/auth/authn" @@ -201,6 +202,30 @@ func TestUpdateDeferredLaunchPromptReplacesOnlyThePrompt(t *testing.T) { } } +func TestUpdateDeferredLaunchPromptPatchesNestedCeilingPayload(t *testing.T) { + svc, _, repo := createTestService(t) + seedDeferredLaunchTask(t, repo) + task, err := repo.GetTask(context.Background(), deferredLaunchTaskID) + require.NoError(t, err) + launch := task.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + launch[models.CeilingDeferredKey] = true + launch[models.CeilingLaunchKindKey] = string(models.CeilingLaunchStart) + launch[models.CeilingLaunchPayloadKey] = map[string]interface{}{ + "prompt": "old nested prompt", + "agent_profile_id": "profile-1", + } + require.NoError(t, repo.UpdateTask(context.Background(), task)) + + _, err = svc.UpdateDeferredLaunchPrompt(context.Background(), deferredLaunchTaskID, "new nested prompt") + require.NoError(t, err) + + launch = storedLaunch(t, repo) + nested, ok := launch[models.CeilingLaunchPayloadKey].(map[string]interface{}) + require.True(t, ok, "ceiling launch payload must remain an object") + require.Equal(t, "new nested prompt", nested["prompt"]) + require.Equal(t, "new nested prompt", launch["prompt"], "legacy top-level prompt remains updated for older readers") +} + // Once the task has a session it is running, and a launch prompt nothing will // read must not be silently accepted. func TestUpdateDeferredLaunchPromptRejectsAStartedTask(t *testing.T) { @@ -302,6 +327,77 @@ func (r *racingSessionRepo) ListTaskSessions(ctx context.Context, taskID string) return sessions, nil } +// racingDeferredLaunchCASRepo injects a concurrent, successful +// SetTaskDeferredLaunchIfUnchanged write between the first +// GetTaskDeferredLaunch read UpdateDeferredLaunchPrompt performs and its own +// compare-and-set write — modelling the session ceiling's own CAS writers +// (for example the replay sweeper) mutating the record while the prompt +// edit is mid-flight. Deterministic: the interleaving is the seam, not a +// sleep. +type racingDeferredLaunchCASRepo struct { + repository.TaskRepository + taskID string + fired bool +} + +func (r *racingDeferredLaunchCASRepo) GetTaskDeferredLaunch( + ctx context.Context, taskID string, +) (map[string]interface{}, interface{}, error) { + launch, prior, err := r.TaskRepository.GetTaskDeferredLaunch(ctx, taskID) + if err != nil || r.fired || taskID != r.taskID { + return launch, prior, err + } + r.fired = true + concurrent := make(map[string]interface{}, len(launch)+1) + for k, v := range launch { + concurrent[k] = v + } + concurrent["ceiling_deferred"] = true + if _, lostCompare, casErr := r.SetTaskDeferredLaunchIfUnchanged( + ctx, taskID, prior, concurrent); casErr != nil || lostCompare { + return nil, nil, fmt.Errorf( + "failed to inject concurrent deferred launch write: lostCompare=%v err=%v", lostCompare, casErr) + } + // Return the pre-race snapshot: this call models the read a caller already + // performed before the concurrent write landed. + return launch, prior, nil +} + +// The ceiling-CAS-bypass race F2 exists to fix: a concurrent, legitimate +// write to deferred_launch (the session ceiling's own CAS writers) landing +// between UpdateDeferredLaunchPrompt's read and its write must not be +// silently clobbered by a stale read-modify-write. Before the fix, the write +// went through SetTaskMetadataKeyIfPresent — a presence-only predicate that +// only checks the key still exists, not that its value is unchanged — so it +// happily overwrote the concurrent writer's change with a stale copy plus +// just the new prompt. +func TestUpdateDeferredLaunchPromptSurvivesAConcurrentCeilingWrite(t *testing.T) { + svc, _, repo := createTestService(t) + seedDeferredLaunchTask(t, repo) + + racing := &racingDeferredLaunchCASRepo{TaskRepository: repo, taskID: deferredLaunchTaskID} + svc.tasks = racing + + if _, err := svc.UpdateDeferredLaunchPrompt( + context.Background(), deferredLaunchTaskID, "what we actually learned"); err != nil { + t.Fatalf("UpdateDeferredLaunchPrompt: %v", err) + } + if !racing.fired { + t.Fatal("the fixture never injected the concurrent ceiling write; the test proved nothing") + } + + launch := storedLaunch(t, repo) + if launch["prompt"] != "what we actually learned" { + t.Fatalf("prompt = %v, want the replacement", launch["prompt"]) + } + if launch["agent_profile_id"] != "profile-1" { + t.Fatalf("agent_profile_id = %v, want the original record preserved", launch["agent_profile_id"]) + } + if launch["ceiling_deferred"] != true { + t.Fatal("the concurrent ceiling write must survive; a stale read-modify-write clobbered it") + } +} + // The resurrection race, reported independently by three reviewers on #2660. // // A read-modify-write prompt edit re-creates the metadata key when a start diff --git a/apps/backend/internal/task/service/service_task_priority_test.go b/apps/backend/internal/task/service/service_task_priority_test.go index 3ae45ca55f..1f4b501952 100644 --- a/apps/backend/internal/task/service/service_task_priority_test.go +++ b/apps/backend/internal/task/service/service_task_priority_test.go @@ -27,6 +27,11 @@ func (r *recordingPriorityTaskRepository) UpdateTask(ctx context.Context, task * return r.TaskRepository.UpdateTask(ctx, task) } +func (r *recordingPriorityTaskRepository) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + r.fullWrites++ + return r.TaskRepository.UpdateTaskPreservingDeferredLaunch(ctx, task) +} + func TestUpdateTask_PriorityOnlyUsesFieldScopedRepositoryWrite(t *testing.T) { svc, eventBus, repo := createTestService(t) ctx := context.Background() diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index 46a7a9b488..6c10728f32 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -2031,7 +2031,7 @@ func (s *Service) UpdateTask(ctx context.Context, id string, req *UpdateTaskRequ if req.Position != nil { updateErr = s.tasks.UpdateTaskWithExplicitPosition(updateCtx, task) } else { - updateErr = s.tasks.UpdateTask(updateCtx, task) + updateErr = s.tasks.UpdateTaskPreservingDeferredLaunch(updateCtx, task) } if updateErr != nil { s.logger.Error("failed to update task", zap.String("task_id", id), zap.Error(updateErr)) @@ -2685,6 +2685,19 @@ func (s *Service) finalizeCancelledSessions( cancelParked() } } + // CancelActiveTaskSessionsByTaskID is a bulk writer: RETURNING reports every + // row's post-update CANCELLED state, not which were in an AC-1 state before + // the update, so every returned id is released unconditionally rather than + // branched on state (AC-51e). Releasing an id that held no reservation is a + // defined no-op. + if s.sessionCeilingReleaser != nil { + for _, session := range cancelledSessions { + if session == nil || session.ID == "" { + continue + } + s.sessionCeilingReleaser.ReleaseCeilingReservation(session.ID) + } + } s.publishSessionsCancelled(detachedCtx, taskID, activeSessions, cancelledSessions, models.SessionArchiveCancelReason) } diff --git a/apps/backend/internal/task/service/service_workflow.go b/apps/backend/internal/task/service/service_workflow.go index f126522d4d..44a20254f4 100644 --- a/apps/backend/internal/task/service/service_workflow.go +++ b/apps/backend/internal/task/service/service_workflow.go @@ -393,14 +393,22 @@ func (s *Service) UpdateTaskMetadata(ctx context.Context, id string, metadata ma } task.UpdatedAt = time.Now().UTC() - if err := s.tasks.UpdateTask(ctx, task); err != nil { + if err := s.tasks.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { s.logger.Error("failed to update task metadata", zap.String("task_id", id), zap.Error(err)) return nil, err } - s.PublishTaskUpdated(ctx, task) + // Reload rather than publish/return the pre-write snapshot: the + // preserving update can win a concurrent ceiling CAS on deferred_launch, + // and that field lives only in the database row from this point on. + current, err := s.tasks.GetTask(ctx, id) + if err != nil { + return nil, err + } + + s.PublishTaskUpdated(ctx, current) s.logger.Debug("task metadata updated", zap.String("task_id", id), zap.Any("metadata", metadata)) - return task, nil + return current, nil } // MoveTaskResult contains the result of a MoveTask operation. @@ -1316,7 +1324,7 @@ func (s *Service) updateMovedTaskSameStep(ctx context.Context, task *models.Task } return task.WIPAdmitted, nil } - if err := s.tasks.UpdateTask(ctx, task); err != nil { + if err := s.tasks.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil { return false, err } return task.WIPAdmitted, nil diff --git a/apps/backend/internal/task/service/service_workflow_deferred_launch_race_test.go b/apps/backend/internal/task/service/service_workflow_deferred_launch_race_test.go new file mode 100644 index 0000000000..aa227cf2be --- /dev/null +++ b/apps/backend/internal/task/service/service_workflow_deferred_launch_race_test.go @@ -0,0 +1,136 @@ +package service + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +// ceilingWriteRaceRepo wraps the real repository and, on the first GetTask +// call for a watched task, writes a fresh deferred_launch record through the +// repository's own CAS primitive before returning the pre-race snapshot — +// modeling the session ceiling's admission controller mutating +// deferred_launch in the window between a read-modify-write caller's own +// GetTask and its later write-back. +type ceilingWriteRaceRepo struct { + repository.TaskRepository + t *testing.T + watchTaskID string + fired bool +} + +func (r *ceilingWriteRaceRepo) GetTask(ctx context.Context, id string) (*models.Task, error) { + task, err := r.TaskRepository.GetTask(ctx, id) + if id == r.watchTaskID && !r.fired { + r.fired = true + _, prior, priorErr := r.GetTaskDeferredLaunch(ctx, id) + require.NoError(r.t, priorErr) + stored, lostCompare, setErr := r.SetTaskDeferredLaunchIfUnchanged(ctx, id, prior, + map[string]interface{}{"prompt": "original", "ceiling_deferred": true}) + require.NoError(r.t, setErr) + require.False(r.t, lostCompare) + require.True(r.t, stored) + } + return task, err +} + +// TestUpdateTaskMetadataSurvivesAConcurrentCeilingWrite pins RV3-B at +// UpdateTaskMetadata: the session ceiling admission controller's own CAS +// write to deferred_launch, landing between this method's GetTask and its +// own write-back, must survive. Before the fix, the plain UpdateTask call +// wrote the stale in-memory metadata verbatim and silently erased the +// concurrently-written ceiling_deferred record. +func TestUpdateTaskMetadataSurvivesAConcurrentCeilingWrite(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + require.NoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-meta-race", Name: "Race"})) + require.NoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-meta-race", WorkspaceID: "ws-meta-race", Name: "flow"})) + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: "task-meta-race", WorkspaceID: "ws-meta-race", WorkflowID: "wf-meta-race", + Title: "Race", State: v1.TaskStateCreated, + })) + _, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "task-meta-race", + sqliterepo.AbsentDeferredLaunch(), map[string]interface{}{"prompt": "original"}) + require.NoError(t, err) + require.False(t, lostCompare) + + raceRepo := &ceilingWriteRaceRepo{TaskRepository: repo, t: t, watchTaskID: "task-meta-race"} + svc.tasks = raceRepo + + returned, err := svc.UpdateTaskMetadata(ctx, "task-meta-race", map[string]interface{}{"ordinary": "value"}) + require.NoError(t, err) + require.True(t, raceRepo.fired, "the concurrent race must have actually been injected") + + // The returned task must be the reloaded, authoritative row, not the + // pre-write in-memory snapshot: that snapshot was read before the + // concurrent ceiling CAS landed and never had deferred_launch merged + // into it locally, so it would report the race write as lost even + // though the database preserved it. + require.Equal(t, "value", returned.Metadata["ordinary"], "the caller's own metadata update must still apply") + returnedDeferred, ok := returned.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "returned task's deferred_launch missing or wrong shape: %#v", returned.Metadata[models.MetaKeyDeferredLaunch]) + require.Equal(t, true, returnedDeferred["ceiling_deferred"], + "UpdateTaskMetadata must return a task reflecting the concurrent ceiling CAS write") + + published := eventBus.GetPublishedEvents() + require.NotEmpty(t, published, "UpdateTaskMetadata must publish a task.updated event") + last := published[len(published)-1] + require.Equal(t, events.TaskUpdated, last.Type) + data, ok := last.Data.(map[string]interface{}) + require.True(t, ok, "event data missing or wrong shape: %#v", last.Data) + require.Equal(t, returned.UpdatedAt.Format(time.RFC3339Nano), data["updated_at"], + "the published event must describe the same reloaded task that was returned, not the stale pre-write snapshot") + + current, err := repo.GetTask(ctx, "task-meta-race") + require.NoError(t, err) + require.Equal(t, "value", current.Metadata["ordinary"], "the caller's own metadata update must still apply") + + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "deferred_launch missing or wrong shape: %#v", current.Metadata[models.MetaKeyDeferredLaunch]) + require.Equal(t, true, deferred["ceiling_deferred"], + "UpdateTaskMetadata must not clobber a concurrent ceiling CAS write") +} + +// TestMoveTaskWithOptionsSameStepSurvivesAConcurrentCeilingWrite pins RV3-B +// at updateMovedTaskSameStep's plain-UpdateTask branch (no ExpectedWorkflowID, +// the ordinary reorder/same-step-move path): the same ceiling CAS race must +// survive a same-step move just as it does an ordinary metadata edit. +func TestMoveTaskWithOptionsSameStepSurvivesAConcurrentCeilingWrite(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + seedMoveWorkflows(t, ctx, repo) + seedMoveSteps(svc) + createMoveTask(t, ctx, repo, "task-move-race", "wf-source", "step-source", nil) + _, lostCompare, err := repo.SetTaskDeferredLaunchIfUnchanged(ctx, "task-move-race", + sqliterepo.AbsentDeferredLaunch(), map[string]interface{}{"prompt": "original"}) + require.NoError(t, err) + require.False(t, lostCompare) + + raceRepo := &ceilingWriteRaceRepo{TaskRepository: repo, t: t, watchTaskID: "task-move-race"} + svc.tasks = raceRepo + + _, err = svc.MoveTaskWithOptions(ctx, "task-move-race", "wf-source", "step-source", 1, MoveTaskOptions{}) + require.NoError(t, err) + require.True(t, raceRepo.fired, "the concurrent race must have actually been injected") + + current, err := repo.GetTask(ctx, "task-move-race") + require.NoError(t, err) + // A same-step move ignores the caller-supplied position literal + // (REQ-TASKS-KANBAN-TASK-REORDERING-001.28: see + // TestService_MoveTaskAllowsSameStepReorderWhenStepAlreadyOverLimit), so + // "the caller's own move must still apply" is pinned by the absence of + // an error and the unchanged position below, not by position becoming 1. + require.Equal(t, 0, current.Position, "a same-step move must leave position unchanged") + + deferred, ok := current.Metadata[models.MetaKeyDeferredLaunch].(map[string]interface{}) + require.True(t, ok, "deferred_launch missing or wrong shape: %#v", current.Metadata[models.MetaKeyDeferredLaunch]) + require.Equal(t, true, deferred["ceiling_deferred"], + "a same-step move must not clobber a concurrent ceiling CAS write") +} diff --git a/apps/backend/internal/task/service/service_workflow_event_source_test.go b/apps/backend/internal/task/service/service_workflow_event_source_test.go index c9c564f80d..d3aa65356d 100644 --- a/apps/backend/internal/task/service/service_workflow_event_source_test.go +++ b/apps/backend/internal/task/service/service_workflow_event_source_test.go @@ -19,12 +19,21 @@ type moveTaskUpdateHookRepository struct { } func (r *moveTaskUpdateHookRepository) UpdateTask(ctx context.Context, task *models.Task) error { + r.fireBeforeUpdate() + return r.Repository.UpdateTask(ctx, task) +} + +func (r *moveTaskUpdateHookRepository) UpdateTaskPreservingDeferredLaunch(ctx context.Context, task *models.Task) error { + r.fireBeforeUpdate() + return r.Repository.UpdateTaskPreservingDeferredLaunch(ctx, task) +} + +func (r *moveTaskUpdateHookRepository) fireBeforeUpdate() { r.updateOnce.Do(func() { if r.beforeUpdate != nil { r.beforeUpdate() } }) - return r.Repository.UpdateTask(ctx, task) } // TestService_MoveTaskEventsUseTransactionalSourceWorkflow covers a stale diff --git a/docs/plans/session-concurrency-ceiling/plan.md b/docs/plans/session-concurrency-ceiling/plan.md new file mode 100644 index 0000000000..3ca81e96bb --- /dev/null +++ b/docs/plans/session-concurrency-ceiling/plan.md @@ -0,0 +1,49 @@ +--- +status: done +created: 2026-09-15 +requirements: + - REQ-AGENTS-SESSION-CEILING-001 +system_design: + - ../../specs/agents/system-design/session-concurrency-ceiling.md +--- + +# Session Concurrency Ceiling Delivery Plan + +## Overview + +Add a shared instance-wide admission ceiling for agent sessions. Preserve the +existing orchestrator launch seams, durable task metadata, and executor +callbacks. Close the recovery, replay, stale-callback, and queue ownership +gaps found during review. + +## Delivery flow + +```text +launch seam -> admission controller -> reservation or deferred_launch CAS + -> executor callback -> confirm/release -> sweep replay -> task update +``` + +The controller owns live admission state. The task repository owns the durable +retry record. The executor owns process identity. These ownership boundaries +remain separate during repair. + +## Verification strategy + +- Run focused orchestrator tests for cold resume, callback identity, Office + replay, dynamic relaunch outcomes, queue retry, and deferred-record conflicts. +- Run focused task-service tests for nested ceiling prompt edits. +- Run `go test -race` for the orchestrator package because reservations and + callback release are concurrent. +- Run the documentation coverage evaluator through the PR workflow after push. + +## Work orders + +- [x] [Task 01: Preserve ceiling launch ownership](task-01-session-concurrency-ceiling.md) + +## Risks + +- A repository read failure before a process callback is treated as not owned, + so a reservation can wait for the sweep instead of being released by an + ambiguous callback. +- The startup ceiling is environment-only. Operators must restart the backend + after changing it. diff --git a/docs/plans/session-concurrency-ceiling/task-01-session-concurrency-ceiling.md b/docs/plans/session-concurrency-ceiling/task-01-session-concurrency-ceiling.md new file mode 100644 index 0000000000..099c80f642 --- /dev/null +++ b/docs/plans/session-concurrency-ceiling/task-01-session-concurrency-ceiling.md @@ -0,0 +1,80 @@ +--- +id: "01-session-concurrency-ceiling" +title: "Preserve session ceiling launch ownership" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-AGENTS-SESSION-CEILING-001 +acceptance_criteria: + - AC-AGENTS-SESSION-CEILING-001.1 + - AC-AGENTS-SESSION-CEILING-001.2 + - AC-AGENTS-SESSION-CEILING-001.3 + - AC-AGENTS-SESSION-CEILING-001.4 + - AC-AGENTS-SESSION-CEILING-001.5 + - AC-AGENTS-SESSION-CEILING-001.6 + - AC-AGENTS-SESSION-CEILING-001.7 +system_design: + - ../../specs/agents/system-design/session-concurrency-ceiling.md +--- + +# Task 01: Preserve Session Ceiling Launch Ownership + +## Outcome + +Keep every accepted or deferred launch owned by one component across refusal, +callback, replay, and prompt-edit paths. + +## In scope + +- Return the existing resume attempt when seam-3 admission refuses a cold + resume. +- Stop a successful early prompt-fallback result before callback preparation. +- Fence process callbacks by the current session execution identity. +- Preserve manual origin for the compound resume-and-prompt operation. +- Distinguish dynamic relaunch success, deferral, and failure. +- Keep Office start records out of workflow auto-start drop checks. +- Report a different pending payload as an explicit conflict. +- Requeue ordinary messages after seam-3 refusal. +- Update nested ceiling launch payload prompts. + +## Exclusions + +- New launch seams, new storage tables, or a frontend settings surface. +- Changes to provider protocols or the task card presentation. + +## Acceptance + +1. Automatic refusals either retain their exact durable replay record or return + an explicit conflict. Manual actions remain admitted and audited. +2. A stale execution callback cannot release a successor reservation, and a + deferred detached relaunch does not finalize its automation run. +3. Cold resume, queue retry, Office replay, dynamic outcome, and nested prompt + regressions pass in focused tests. + +## Verification + +```bash +cd apps/backend && go test ./internal/orchestrator ./internal/task/service -count=1 +cd apps/backend && go test -race ./internal/orchestrator -count=1 +git diff --check +``` + +## Likely files + +- `apps/backend/internal/orchestrator/task_operations.go` +- `apps/backend/internal/orchestrator/dynamic_launch.go` +- `apps/backend/internal/orchestrator/ceiling_defer.go` +- `apps/backend/internal/orchestrator/ceiling_replay.go` +- `apps/backend/internal/orchestrator/ceiling_release.go` +- `apps/backend/internal/orchestrator/event_handlers_agent.go` +- `apps/backend/internal/task/service/service_deferred_launch.go` +- matching regression tests + +## Results + +- Focused orchestrator regressions pass for cold-resume ownership, stale + callback fencing, Office replay, duplicate conflicts, queue retry, and the + dynamic deferred outcome. +- Focused task-service regressions pass for top-level and nested prompt edits. diff --git a/docs/specs/agents/requirements/session-concurrency-ceiling.md b/docs/specs/agents/requirements/session-concurrency-ceiling.md new file mode 100644 index 0000000000..e4e9fd77f8 --- /dev/null +++ b/docs/specs/agents/requirements/session-concurrency-ceiling.md @@ -0,0 +1,65 @@ +--- +status: active +system: agents +created: 2026-09-15 +owners: + - kandev +--- + +# Session Concurrency Ceiling Requirements + +## Overview + +One Kandev instance can start more agent sessions than its host can serve. A +shared ceiling must limit new automatic starts while preserving explicit user +actions and accepted work. This contract belongs to the agent system because it +controls admission of agent executions. Tasks own their durable deferral data. + +## Terminology + +- **Session ceiling:** The maximum number of sessions in `STARTING` or `RUNNING` + state, including launches admitted in the current process. +- **Deferred launch:** A durable task record that contains the complete launch + kind and payload for an automatic request that the ceiling refused. +- **Manual origin:** A direct user action. It can use a manual override when the + ceiling is full. + +## Requirements + +### REQ-AGENTS-SESSION-CEILING-001: Bound instance agent sessions + +**Intent:** Keep automatic agent starts within the instance capacity while +keeping accepted work and manual recovery available. + +**User story:** As an operator, I want automatic agent starts to respect a +bounded instance capacity, so that one installation does not overload its host. + +#### Acceptance criteria + +- **AC-AGENTS-SESSION-CEILING-001.1:** When an automatic launch would exceed the + configured ceiling, the system shall refuse the launch and persist its kind, + replay payload, origin, reason, and enqueue time on the task. +- **AC-AGENTS-SESSION-CEILING-001.2:** When a manual launch would exceed the + ceiling, the system shall admit it and record that it used a manual override. +- **AC-AGENTS-SESSION-CEILING-001.3:** The ceiling shall count sessions in + `STARTING` and `RUNNING` state together with in-flight reservations, and shall + release a reservation only after its launch reaches a counted state or fails. +- **AC-AGENTS-SESSION-CEILING-001.4:** A deferred launch shall retain its first + payload when a different launch for the same task arrives, and the second + caller shall receive an explicit conflict so it retains ownership. +- **AC-AGENTS-SESSION-CEILING-001.5:** A retry sweep shall replay deferred + launches by their stored kind. It shall preserve a record when capacity is + still full or replay fails for a non-ceiling reason. +- **AC-AGENTS-SESSION-CEILING-001.6:** A callback from an older execution shall + not release or confirm a reservation held by a successor execution for the + same session. +- **AC-AGENTS-SESSION-CEILING-001.7:** The startup environment variable + `KANDEV_MAX_CONCURRENT_SESSIONS` shall accept a non-negative integer, use zero + for unlimited, and use a derived default with a minimum of two when unset or + invalid. + +## Out of scope + +- Per-workspace or per-user ceilings. +- A frontend settings control for this startup-only value. +- Replacing the orchestrator's existing launch seams or task repository. diff --git a/docs/specs/agents/system-design/session-concurrency-ceiling.md b/docs/specs/agents/system-design/session-concurrency-ceiling.md new file mode 100644 index 0000000000..c99ae79a11 --- /dev/null +++ b/docs/specs/agents/system-design/session-concurrency-ceiling.md @@ -0,0 +1,98 @@ +--- +status: current +system: agents +requirements: + - REQ-AGENTS-SESSION-CEILING-001 +--- + +# Session Concurrency Ceiling System Design + +## Purpose and boundaries + +The orchestrator owns one admission controller shared by manual starts, +resumes, workflow starts, queue drains, and dynamic relaunches. The controller +does not own task metadata or provider execution. It returns a reservation or a +typed refusal. The task repository owns the durable deferred-launch record. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-AGENTS-SESSION-CEILING-001` | [Admission and replay](#admission-and-replay), [Failure and recovery](#failure-and-recovery) | + +## Components and responsibilities + +| Component | Responsibility | +| --- | --- | +| `sessionCeilingController` | Counts persisted `STARTING`/`RUNNING` sessions and process-local reservations. It admits, refuses, confirms, and releases launches. | +| Orchestrator launch seams | Pass the launch origin and complete replay payload to the controller. They consume a reservation only after launch success. | +| `deferCeilingRefusal` | Merges the ceiling-owned keys into task `deferred_launch` with compare-and-set semantics. | +| Ceiling sweep | Lists tasks with ceiling records, validates eligibility, and dispatches the stored launch kind. | +| Task repository and service | Store and update the shared deferred record. Prompt edits update both legacy top-level data and the nested ceiling payload. | +| Executor callbacks | Confirm or release only when the callback execution still owns the session row. | + +## Data and contracts + +The shared record uses `ceiling_deferred`, `ceiling_launch_kind`, +`ceiling_launch_payload`, `ceiling_launch_origin`, `ceiling_reason_code`, and +`ceiling_queued_at`. The payload is nested so replay does not confuse a resume, +prompt ensure, or dynamic relaunch with a task start. A different pending +payload returns `ErrCeilingLaunchConflict`; the existing record stays unchanged. + +The controller resolves `KANDEV_MAX_CONCURRENT_SESSIONS` once at startup. An +unset or invalid value uses half the host CPU count, with a minimum of two. Zero +means unlimited. The value is an environment-only startup setting. + +## Admission and replay + +The flow is: + +```text +launch request + -> orchestrator seam + -> sessionCeilingController.admit + -> reservation, or typed refusal + -> deferCeilingRefusal(task CAS) when automatic refusal + -> executor launch and callback + -> confirm/release reservation + -> ceiling sweep replays the stored kind when capacity is available +``` + +Manual origins bypass refusal and write a manual-override audit entry. Automatic +origins never become manual during replay. The sweep clears a record only after +successful dispatch. A repeated refusal leaves the original timestamp and +payload in place. + +## Failure and recovery + +Reservation ownership is session-keyed. A stale process-start callback first +checks the persisted session execution identity; it cannot release a successor's +reservation. A dynamic relaunch reports three outcomes: succeeded, deferred, or +failed. A deferred detached relaunch leaves the automation run and its durable +record for the sweep. Queue dispatch treats a seam-3 refusal as retryable so the +original message remains in the queue. + +Office automatic starts do not use workflow-step auto-start eligibility when the +sweep evaluates a `start` record. This keeps Office scheduling ownership in the +Office path. + +## Persistence + +`deferred_launch` is updated with a read-compare-write retry loop. The ceiling +keys share the task record with other launch intents and are removed as a group +after replay or a terminal drop. A prompt edit preserves unrelated keys and +updates the nested replay payload. + +## Security + +The controller receives launch data from trusted backend paths. The nested +payload is excluded from plugin host data. No prompt or environment value is +executed during replay; it is passed to the same existing launch seam after +task and session eligibility checks. + +## Observability + +Admission decisions, refusal reasons, reservation expiry, replay drops, and +manual overrides use the existing orchestrator logs and task status messages. +The stored reason and population snapshot make a refusal diagnosable after a +restart.