From f7f1d1099682f53f4b5d9f449afb3a4e72cf04b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Almeida?= <57680069+zeval@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:23:46 +0100 Subject: [PATCH] feat: add parent-controlled task stopping via MCP (#1803) * feat: add parent-controlled task stopping via MCP * fix: harden parent task stop lifecycle * fix: preserve rejected task rollback state --- apps/backend/config/prompts/kandev-context.md | 2 +- apps/backend/internal/agent/runtime/facade.go | 7 +- .../runtime/lifecycle/manager_interaction.go | 2 +- .../lifecycle/manager_interaction_test.go | 8 + .../backend/internal/agent/runtime/runtime.go | 2 +- .../agent/runtime/runtime_contract_test.go | 14 + apps/backend/internal/backendapp/adapters.go | 9 + apps/backend/internal/backendapp/helpers.go | 1 + .../backendapp/mcp_stop_composition_test.go | 101 +++ .../internal/gateway/websocket/client.go | 7 + .../websocket/client_mcp_actions_test.go | 129 +++ .../integration/mcp_integration_test.go | 61 +- .../integration/mcp_stop_task_test.go | 334 ++++++++ .../internal/integration/test_server_test.go | 9 + .../mcp/handlers/clarification_pause_test.go | 242 ++++++ .../backend/internal/mcp/handlers/handlers.go | 255 +++++- .../internal/mcp/handlers/handlers_test.go | 6 + .../mcp/handlers/message_task_test.go | 79 ++ .../internal/mcp/handlers/stop_task.go | 116 +++ .../internal/mcp/handlers/stop_task_test.go | 271 ++++++ apps/backend/internal/mcp/server/handlers.go | 21 + .../internal/mcp/server/handlers_test.go | 127 ++- apps/backend/internal/mcp/server/server.go | 22 +- .../internal/mcp/server/server_test.go | 11 +- .../mcp/server/sysprompt_sync_test.go | 59 ++ .../coordinator_stop_edges_test.go | 366 ++++++++ .../orchestrator/coordinator_stop_test.go | 798 ++++++++++++++++++ .../orchestrator/event_handlers_agent.go | 266 +++++- .../event_handlers_clarification.go | 40 +- .../event_handlers_clarification_test.go | 111 +++ .../event_handlers_runtime_state_race_test.go | 66 ++ .../event_handlers_step_completion.go | 4 + .../event_handlers_step_completion_test.go | 60 ++ .../orchestrator/event_handlers_streaming.go | 434 ++++++++-- .../event_handlers_streaming_test.go | 204 +++++ .../orchestrator/event_handlers_test.go | 185 +++- .../orchestrator/event_handlers_transient.go | 17 +- .../event_handlers_transient_test.go | 139 +++ .../orchestrator/event_handlers_workflow.go | 127 ++- .../execution_teardown_ownership_test.go | 199 +++++ .../orchestrator/executor/executor.go | 77 ++ .../orchestrator/executor/executor_execute.go | 273 +++++- .../executor/executor_interaction.go | 181 +++- .../executor/executor_interaction_test.go | 540 ++++++++++++ .../executor/executor_mocks_test.go | 56 +- .../orchestrator/executor/executor_office.go | 36 +- .../executor/executor_office_test.go | 65 ++ .../orchestrator/executor/executor_resume.go | 31 +- .../executor/executor_resume_test.go | 47 ++ .../orchestrator/executor/executor_test.go | 369 ++++++++ .../orchestrator/scheduler/scheduler.go | 9 + .../orchestrator/scheduler/scheduler_test.go | 10 + apps/backend/internal/orchestrator/service.go | 13 + .../internal/orchestrator/task_operations.go | 252 +++++- .../orchestrator/task_operations_test.go | 185 +++- apps/backend/internal/sysprompt/sysprompt.go | 59 +- .../internal/sysprompt/sysprompt_test.go | 15 + .../task/handlers/message_handlers.go | 11 +- .../task/handlers/message_handlers_test.go | 67 ++ .../task/handlers/process_handlers_test.go | 5 + .../internal/task/repository/interface.go | 9 + .../task/repository/sqlite/session.go | 153 +++- .../task/repository/sqlite/session_test.go | 96 +++ .../internal/task/repository/sqlite/task.go | 146 ++++ .../repository/sqlite/task_state_cas_test.go | 187 ++++ .../task/service/handoff_workspace_test.go | 6 + .../task/service/message_rollback_test.go | 45 + .../task/service/resource_cleanup_jobs.go | 10 +- .../service/resource_cleanup_jobs_test.go | 1 + apps/backend/internal/task/service/service.go | 3 + .../internal/task/service/service_tasks.go | 170 ++++ .../task/service/service_tasks_stop_test.go | 121 ++- .../internal/task/service/service_test.go | 63 ++ .../internal/task/service/service_workflow.go | 30 + apps/backend/pkg/websocket/actions.go | 1 + .../office/onboarding-task-launch.spec.ts | 35 +- ...-19-reject-mcp-actions-on-raw-websocket.md | 56 ++ docs/decisions/INDEX.md | 1 + docs/plans/parent-child-task-stop/plan.md | 433 ++++++++++ .../task-01-execution-stop-semantics.md | 76 ++ .../task-02-coordinator-stop-operation.md | 73 ++ .../task-03-mcp-stop-handler.md | 80 ++ .../task-04-task-mcp-tool.md | 68 ++ .../task-05-integration-regression.md | 62 ++ .../task-06-public-docs.md | 67 ++ docs/public/automation-and-mcp.md | 10 +- docs/public/coordination.md | 24 +- docs/public/coverage.json | 23 +- docs/public/feature-status.md | 4 +- docs/public/sessions-and-review.md | 7 +- docs/public/websocket-api.md | 9 +- docs/specs/INDEX.md | 1 + .../tasks/parent-child-message-interrupt.md | 6 +- docs/specs/tasks/parent-child-task-stop.md | 240 ++++++ 94 files changed, 9158 insertions(+), 370 deletions(-) create mode 100644 apps/backend/internal/backendapp/mcp_stop_composition_test.go create mode 100644 apps/backend/internal/gateway/websocket/client_mcp_actions_test.go create mode 100644 apps/backend/internal/integration/mcp_stop_task_test.go create mode 100644 apps/backend/internal/mcp/handlers/stop_task.go create mode 100644 apps/backend/internal/mcp/handlers/stop_task_test.go create mode 100644 apps/backend/internal/orchestrator/coordinator_stop_edges_test.go create mode 100644 apps/backend/internal/orchestrator/coordinator_stop_test.go create mode 100644 apps/backend/internal/orchestrator/event_handlers_runtime_state_race_test.go create mode 100644 apps/backend/internal/orchestrator/execution_teardown_ownership_test.go create mode 100644 apps/backend/internal/orchestrator/executor/executor_interaction_test.go create mode 100644 apps/backend/internal/task/service/message_rollback_test.go create mode 100644 docs/decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md create mode 100644 docs/plans/parent-child-task-stop/plan.md create mode 100644 docs/plans/parent-child-task-stop/task-01-execution-stop-semantics.md create mode 100644 docs/plans/parent-child-task-stop/task-02-coordinator-stop-operation.md create mode 100644 docs/plans/parent-child-task-stop/task-03-mcp-stop-handler.md create mode 100644 docs/plans/parent-child-task-stop/task-04-task-mcp-tool.md create mode 100644 docs/plans/parent-child-task-stop/task-05-integration-regression.md create mode 100644 docs/plans/parent-child-task-stop/task-06-public-docs.md create mode 100644 docs/specs/tasks/parent-child-task-stop.md diff --git a/apps/backend/config/prompts/kandev-context.md b/apps/backend/config/prompts/kandev-context.md index 5062e72283..5e6360c297 100644 --- a/apps/backend/config/prompts/kandev-context.md +++ b/apps/backend/config/prompts/kandev-context.md @@ -17,6 +17,6 @@ Available tools: - create_task_kandev: Create a new task or subtask. Required params: title. For subtasks, set parent_id to the literal string "self" (the MCP server expands it to your current task ID) and omit workspace_id/workflow_id/workflow_step_id; they inherit from the parent. Pass workspace_id/workflow_id on a subtask only when deliberately targeting another task workspace/workflow; any supplied workflow_id must belong to the effective workspace_id. MCP subtasks reuse the parent's materialized workspace by default; set workspace_mode to "new_workspace" only when the subtask should launch in its own worktree/materialized workspace. For top-level tasks, provide workspace_id/workflow_id unless each can be auto-resolved uniquely. workflow_step_id is optional. - update_task_kandev: Update a task. Required params: task_id. - spawn_session_kandev: Spawn an ADDITIONAL agent session on your current task (no new task is created — it runs alongside your session in the same workspace). Required params: prompt (the new session's ONLY initial context). Optional: agent_profile_id (defaults to your profile; specify a different one to spawn a different agent), name (session tab label, e.g. "reviewer"), task_id (defaults to your task). Returns the new session_id. -- message_task_kandev: Message another task's agent, or a specific session via the optional session_id param — including a sibling session on your OWN task (e.g. one you spawned). Required params: task_id, prompt. +- message_task_kandev: Message another task's agent, or a specific session via optional session_id — including a sibling session on your OWN task. Required params: task_id, prompt.{coordinator_task_control_section} IMPORTANT: You MUST use these MCP tools when instructed to create plans, ask questions, or interact with the Kandev platform. Do not skip them. diff --git a/apps/backend/internal/agent/runtime/facade.go b/apps/backend/internal/agent/runtime/facade.go index cf0c03a05e..c952c86635 100644 --- a/apps/backend/internal/agent/runtime/facade.go +++ b/apps/backend/internal/agent/runtime/facade.go @@ -2,6 +2,7 @@ package runtime import ( "context" + "errors" "fmt" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" @@ -53,7 +54,11 @@ func (f *facade) Stop(ctx context.Context, executionID string, reason string) er if executionID == "" { return fmt.Errorf("runtime: executionID is required") } - return f.backend.StopAgentWithReason(ctx, executionID, reason, false) + err := f.backend.StopAgentWithReason(ctx, executionID, reason, false) + if errors.Is(err, lifecycle.ErrExecutionNotFound) { + return errors.Join(ErrNotFound, err) + } + return err } // GetExecution returns a snapshot view of an execution. diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go index e4d1cb750f..bd13aca6aa 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go @@ -534,7 +534,7 @@ func (m *Manager) StopAgent(ctx context.Context, executionID string, force bool) func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, reason string, force bool) error { execution, exists := m.executionStore.Get(executionID) if !exists { - return fmt.Errorf("execution %q not found", executionID) + return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) } activityLease, err := m.acquireActivity(ctx, activity.KindExecutionStopping) if err != nil { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go index 94e9b4ddbb..6449c31d39 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go @@ -35,6 +35,14 @@ type restartMockAgentctlServer struct { failSessionNew bool } +func TestStopAgentWithReason_MissingExecutionIsClassified(t *testing.T) { + mgr := &Manager{executionStore: NewExecutionStore(), logger: newTestLogger().WithFields()} + + err := mgr.StopAgentWithReason(context.Background(), "missing", "cleanup", true) + + require.ErrorIs(t, err, ErrExecutionNotFound) +} + func newRestartMockAgentctlServer(t *testing.T, failStop, failSessionNew bool) *restartMockAgentctlServer { t.Helper() diff --git a/apps/backend/internal/agent/runtime/runtime.go b/apps/backend/internal/agent/runtime/runtime.go index 7ef4ad5910..b9474c452d 100644 --- a/apps/backend/internal/agent/runtime/runtime.go +++ b/apps/backend/internal/agent/runtime/runtime.go @@ -131,7 +131,7 @@ type Execution struct { // callers don't need to import a sibling package for a tiny struct. type Event = streams.AgentEvent -// ErrNotFound is returned by GetExecution when no execution matches. +// ErrNotFound reports that an execution lookup or stop found no live runtime. var ErrNotFound = errors.New("runtime: execution not found") // ErrUnsupported is returned by methods that have not yet been wired up diff --git a/apps/backend/internal/agent/runtime/runtime_contract_test.go b/apps/backend/internal/agent/runtime/runtime_contract_test.go index f94e5c13ce..7123db777f 100644 --- a/apps/backend/internal/agent/runtime/runtime_contract_test.go +++ b/apps/backend/internal/agent/runtime/runtime_contract_test.go @@ -265,6 +265,20 @@ func TestRuntime_Stop_DelegatesToBackend(t *testing.T) { } } +func TestRuntime_Stop_ClassifiesMissingExecution(t *testing.T) { + t.Parallel() + backend := newFakeBackend() + backend.stopErr = lifecycle.ErrExecutionNotFound + rt := agentruntime.New(backend) + err := rt.Stop(context.Background(), "missing", "cleanup") + if !errors.Is(err, agentruntime.ErrNotFound) { + t.Fatalf("Stop error = %v, want runtime.ErrNotFound", err) + } + if !errors.Is(err, lifecycle.ErrExecutionNotFound) { + t.Fatalf("Stop error = %v, want lifecycle.ErrExecutionNotFound", err) + } +} + func TestRuntime_Stop_RequiresExecutionID(t *testing.T) { t.Parallel() rt := agentruntime.New(newFakeBackend()) diff --git a/apps/backend/internal/backendapp/adapters.go b/apps/backend/internal/backendapp/adapters.go index 7a4c298a1f..6488fcbdb4 100644 --- a/apps/backend/internal/backendapp/adapters.go +++ b/apps/backend/internal/backendapp/adapters.go @@ -67,6 +67,15 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived( return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state) } +func (a *taskRepositoryAdapter) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + return a.svc.UpdateTaskStateIfSessionState(ctx, taskID, sessionID, expectedSessionState, state) +} + // lifecycleAdapter adapts the lifecycle manager as an AgentManagerClient type lifecycleAdapter struct { mgr *lifecycle.Manager diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 0fee686bf1..a82734e43f 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -1183,6 +1183,7 @@ func registerMCPAndDebugRoutes( mcpHandlers.SetConfigDeps(p.services.Workflow, p.agentSettingsController, p.mcpConfigSvc) mcpHandlers.SetClarificationInputPauser(p.orchestratorSvc) mcpHandlers.SetPromptReferenceResolver(p.services.Prompts) + mcpHandlers.SetTaskStopper(p.orchestratorSvc) // Enrich list_tasks responses with associated GitHub PRs (link, title, // number, state) when the github service is available. diff --git a/apps/backend/internal/backendapp/mcp_stop_composition_test.go b/apps/backend/internal/backendapp/mcp_stop_composition_test.go new file mode 100644 index 0000000000..2ba6ec4b37 --- /dev/null +++ b/apps/backend/internal/backendapp/mcp_stop_composition_test.go @@ -0,0 +1,101 @@ +package backendapp + +import ( + "context" + "testing" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events/bus" + gateways "github.com/kandev/kandev/internal/gateway/websocket" + "github.com/kandev/kandev/internal/orchestrator" + taskservice "github.com/kandev/kandev/internal/task/service" + ws "github.com/kandev/kandev/pkg/websocket" + "github.com/stretchr/testify/require" +) + +func TestRegisterMCPAndDebugRoutesWiresCoordinatorTaskStopper(t *testing.T) { + gin.SetMode(gin.TestMode) + harness := newBootStateTestHarness(t) + ctx := context.Background() + + workspaces, err := harness.taskSvc.ListWorkspaces(ctx) + require.NoError(t, err) + require.NotEmpty(t, workspaces) + workflows, err := harness.taskSvc.ListWorkflows(ctx, workspaces[0].ID, true) + require.NoError(t, err) + require.NotEmpty(t, workflows) + steps, err := harness.workflowSvc.ListStepsByWorkflow(ctx, workflows[0].ID) + require.NoError(t, err) + require.NotEmpty(t, steps) + parent, err := harness.taskSvc.CreateTask(ctx, &taskservice.CreateTaskRequest{ + WorkspaceID: workspaces[0].ID, + WorkflowID: workflows[0].ID, + WorkflowStepID: steps[0].ID, + Title: "Stop coordinator", + }) + require.NoError(t, err) + childID, err := harness.taskSvc.CreateChildTask(ctx, parent, taskservice.ChildTaskSpec{ + Title: "Unneeded child", + }) + require.NoError(t, err) + + log, err := logger.NewLogger(logger.LoggingConfig{ + Level: "error", Format: "console", OutputPath: "stdout", + }) + require.NoError(t, err) + eventBus := bus.NewMemoryEventBus(log) + taskRepoAdapter := &taskRepositoryAdapter{repo: harness.taskRepo, svc: harness.taskSvc} + orchestratorSvc := orchestrator.NewService( + orchestrator.DefaultServiceConfig(), eventBus, nil, + taskRepoAdapter, harness.taskRepo, nil, nil, nil, log, + ) + lifecycleMgr := lifecycle.NewManager( + nil, eventBus, nil, nil, nil, nil, + lifecycle.ExecutorFallbackDeny, t.TempDir(), log, + ) + registerMCPStopTestCleanup(t, "lifecycle manager", lifecycleMgr.Stop) + gateway := gateways.NewGateway(log) + registerMCPAndDebugRoutes(routeParams{ + router: gin.New(), + gateway: gateway, + taskSvc: harness.taskSvc, + taskRepo: harness.taskRepo, + orchestratorSvc: orchestratorSvc, + lifecycleMgr: lifecycleMgr, + eventBus: eventBus, + services: &Services{Workflow: harness.workflowSvc}, + log: log, + addCleanup: func(cleanup func() error) { + registerMCPStopTestCleanup(t, "MCP server", cleanup) + }, + }, nil, nil, nil, nil, nil) + + require.True(t, gateway.Dispatcher.HasHandler(ws.ActionMCPStopTask), + "registerMCPAndDebugRoutes did not register mcp.stop_task") + request, err := ws.NewRequest("stop-1", ws.ActionMCPStopTask, map[string]string{ + "task_id": childID, + "sender_task_id": parent.ID, + }) + require.NoError(t, err) + response, err := gateway.Dispatcher.Dispatch(ctx, request) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type, "payload=%s", response.Payload) + var payload struct { + TaskID string `json:"task_id"` + Status orchestrator.CoordinatorTaskStopStatus `json:"status"` + } + require.NoError(t, response.ParsePayload(&payload)) + require.Equal(t, childID, payload.TaskID) + require.Equal(t, orchestrator.CoordinatorTaskStopStatusNotRunning, payload.Status) +} + +func registerMCPStopTestCleanup(t *testing.T, name string, cleanup func() error) { + t.Helper() + t.Cleanup(func() { + if err := cleanup(); err != nil { + t.Errorf("stop %s: %v", name, err) + } + }) +} diff --git a/apps/backend/internal/gateway/websocket/client.go b/apps/backend/internal/gateway/websocket/client.go index 3dfd4b98fc..32a80fdeb0 100644 --- a/apps/backend/internal/gateway/websocket/client.go +++ b/apps/backend/internal/gateway/websocket/client.go @@ -3,6 +3,7 @@ package websocket import ( "context" "encoding/json" + "strings" "sync" "time" @@ -121,6 +122,12 @@ func (c *Client) handleMessage(msg *ws.Message) { zap.String("action", msg.Action), zap.String("id", msg.ID)) + if strings.HasPrefix(msg.Action, "mcp.") { + c.sendError(msg.ID, msg.Action, ws.ErrorCodeForbidden, + "MCP actions are not available over the raw WebSocket", nil) + return + } + // Handle subscription actions specially (they need access to the client) switch msg.Action { case ws.ActionTaskSubscribe: diff --git a/apps/backend/internal/gateway/websocket/client_mcp_actions_test.go b/apps/backend/internal/gateway/websocket/client_mcp_actions_test.go new file mode 100644 index 0000000000..7fd400cd7d --- /dev/null +++ b/apps/backend/internal/gateway/websocket/client_mcp_actions_test.go @@ -0,0 +1,129 @@ +package websocket + +import ( + "context" + "encoding/json" + "testing" + + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestClient_HandleMessageRejectsMCPNamespaceBeforeDispatch(t *testing.T) { + for _, action := range []string{"mcp.stop_task", "mcp.future_action"} { + t.Run(action, func(t *testing.T) { + dispatcher := ws.NewDispatcher() + handlerCalls := 0 + dispatcher.RegisterFunc(action, func(_ context.Context, msg *ws.Message) (*ws.Message, error) { + handlerCalls++ + return ws.NewResponse(msg.ID, msg.Action, map[string]bool{"called": true}) + }) + + hub := newTestHub(t) + hub.dispatcher = dispatcher + client := newTestClient("raw-client") + client.hub = hub + + request, err := ws.NewRequest("request-1", action, map[string]string{ + "task_id": "child-task", + "sender_task_id": "forged-parent-task", + }) + if err != nil { + t.Fatalf("build request: %v", err) + } + + client.handleMessage(request) + + response := receiveClientMessage(t, client) + if response.Type != ws.MessageTypeError { + t.Fatalf("response type = %q, want %q", response.Type, ws.MessageTypeError) + } + if response.ID != request.ID || response.Action != request.Action { + t.Fatalf("response correlation = (%q, %q), want (%q, %q)", + response.ID, response.Action, request.ID, request.Action) + } + + var payload ws.ErrorPayload + if err := response.ParsePayload(&payload); err != nil { + t.Fatalf("parse error payload: %v", err) + } + if payload.Code != ws.ErrorCodeForbidden { + t.Fatalf("error code = %q, want %q", payload.Code, ws.ErrorCodeForbidden) + } + if handlerCalls != 0 { + t.Fatalf("dispatcher handler calls = %d, want 0", handlerCalls) + } + }) + } +} + +func TestClient_HandleMessageStillDispatchesPublicActions(t *testing.T) { + const action = "test.public" + + dispatcher := ws.NewDispatcher() + handlerCalls := 0 + dispatcher.RegisterFunc(action, func(_ context.Context, msg *ws.Message) (*ws.Message, error) { + handlerCalls++ + return ws.NewResponse(msg.ID, msg.Action, map[string]bool{"ok": true}) + }) + + hub := newTestHub(t) + hub.dispatcher = dispatcher + client := newTestClient("raw-client") + client.hub = hub + + request, err := ws.NewRequest("request-1", action, nil) + if err != nil { + t.Fatalf("build request: %v", err) + } + client.handleMessage(request) + + response := receiveClientMessage(t, client) + if response.Type != ws.MessageTypeResponse { + t.Fatalf("response type = %q, want %q", response.Type, ws.MessageTypeResponse) + } + if handlerCalls != 1 { + t.Fatalf("dispatcher handler calls = %d, want 1", handlerCalls) + } +} + +func TestDispatcher_DirectMCPDispatchRemainsAvailable(t *testing.T) { + const action = "mcp.stop_task" + + dispatcher := ws.NewDispatcher() + handlerCalls := 0 + dispatcher.RegisterFunc(action, func(_ context.Context, msg *ws.Message) (*ws.Message, error) { + handlerCalls++ + return ws.NewResponse(msg.ID, msg.Action, map[string]bool{"stopped": true}) + }) + + request, err := ws.NewRequest("request-1", action, nil) + if err != nil { + t.Fatalf("build request: %v", err) + } + response, err := dispatcher.Dispatch(context.Background(), request) + if err != nil { + t.Fatalf("direct dispatch: %v", err) + } + if response == nil || response.Type != ws.MessageTypeResponse { + t.Fatalf("direct dispatch response = %#v, want response", response) + } + if handlerCalls != 1 { + t.Fatalf("dispatcher handler calls = %d, want 1", handlerCalls) + } +} + +func receiveClientMessage(t *testing.T, client *Client) *ws.Message { + t.Helper() + + select { + case data := <-client.send: + var message ws.Message + if err := json.Unmarshal(data, &message); err != nil { + t.Fatalf("decode client message: %v", err) + } + return &message + default: + t.Fatal("client produced no response") + return nil + } +} diff --git a/apps/backend/internal/integration/mcp_integration_test.go b/apps/backend/internal/integration/mcp_integration_test.go index a1073cb5fc..c73e8cd0de 100644 --- a/apps/backend/internal/integration/mcp_integration_test.go +++ b/apps/backend/internal/integration/mcp_integration_test.go @@ -70,15 +70,29 @@ func setupMCPTestServer(t *testing.T) (*TestServer, string, string, string, stri return ts, parentTaskID, workspaceID, workflowID, workflowStepID } +// dispatchTrustedMCP exercises the in-process dispatcher path used by the +// authenticated agent stream. Raw /ws intentionally rejects the internal +// mcp.* namespace (ADR-2026-07-19-reject-mcp-actions-on-raw-websocket). +func dispatchTrustedMCP( + t *testing.T, + ts *TestServer, + requestID, action string, + payload map[string]interface{}, +) (*ws.Message, error) { + t.Helper() + request, err := ws.NewRequest(requestID, action, payload) + if err != nil { + return nil, err + } + return ts.Gateway.Dispatcher.Dispatch(context.Background(), request) +} + func TestMCPCreateTask_SubtaskInheritsFromParent(t *testing.T) { ts, parentTaskID, workspaceID, workflowID, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // Create subtask with title, description, and parent_id — workspace/workflow/step should be inherited - resp, err := client.SendRequest("subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ "parent_id": parentTaskID, "title": "Subtask via MCP", "description": "Implement the subtask feature", @@ -118,7 +132,7 @@ func TestMCPCreateTask_SubtaskVisibleInListTasks(t *testing.T) { workflowID := parentPayload["workflow_id"].(string) // Create subtask - resp, err := client.SendRequest("subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ "parent_id": parentTaskID, "title": "Visible Subtask", "description": "Implement the visible subtask feature", @@ -127,7 +141,7 @@ func TestMCPCreateTask_SubtaskVisibleInListTasks(t *testing.T) { require.Equal(t, ws.MessageTypeResponse, resp.Type) // List tasks — should contain both parent and subtask - listResp, err := client.SendRequest("list-1", ws.ActionMCPListTasks, map[string]interface{}{ + listResp, err := dispatchTrustedMCP(t, ts, "list-1", ws.ActionMCPListTasks, map[string]interface{}{ "workflow_id": workflowID, }) require.NoError(t, err) @@ -153,11 +167,8 @@ func TestMCPCreateTask_TopLevelWithoutParent(t *testing.T) { ts, _, workspaceID, workflowID, workflowStepID := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // Create top-level task (no parent_id, all IDs provided) - resp, err := client.SendRequest("top-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "top-1", ws.ActionMCPCreateTask, map[string]interface{}{ "workspace_id": workspaceID, "workflow_id": workflowID, "workflow_step_id": workflowStepID, @@ -180,11 +191,8 @@ func TestMCPCreateTask_NoParentNoIDs_ReturnsError(t *testing.T) { ts, _, _, _, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // No parent_id and no workspace/workflow -> should fail - resp, err := client.SendRequest("fail-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "fail-1", ws.ActionMCPCreateTask, map[string]interface{}{ "title": "Orphan Task", }) require.NoError(t, err) @@ -195,10 +203,7 @@ func TestMCPCreateTask_InvalidParentID_ReturnsError(t *testing.T) { ts, _, _, _, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - - resp, err := client.SendRequest("bad-parent", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "bad-parent", ws.ActionMCPCreateTask, map[string]interface{}{ "parent_id": "nonexistent-task-id", "title": "Bad Parent", }) @@ -210,11 +215,8 @@ func TestMCPCreateTask_StartAgentFalse_DoesNotRequireDescription(t *testing.T) { ts, parentTaskID, _, _, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // With start_agent=false, description should NOT be required for subtasks - resp, err := client.SendRequest("subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ "parent_id": parentTaskID, "title": "Subtask without description", "start_agent": false, // Don't auto-start, so no description needed @@ -229,11 +231,8 @@ func TestMCPCreateTask_StartAgentTrue_RequiresDescription(t *testing.T) { ts, parentTaskID, _, _, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // With start_agent=true (default), description IS required for subtasks - resp, err := client.SendRequest("subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "subtask-1", ws.ActionMCPCreateTask, map[string]interface{}{ "parent_id": parentTaskID, "title": "Subtask without description", // start_agent defaults to true, description is required @@ -248,12 +247,9 @@ func TestMCPCreateTask_SourceTaskID_TopLevel_Succeeds(t *testing.T) { ts, parentTaskID, workspaceID, workflowID, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // source_task_id is set by agentctl to the current task; verify the path succeeds // and the task is created (even though parentTaskID has no repositories). - resp, err := client.SendRequest("top-source", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "top-source", ws.ActionMCPCreateTask, map[string]interface{}{ "workspace_id": workspaceID, "workflow_id": workflowID, "title": "Top Level with Source Task", @@ -271,13 +267,10 @@ func TestMCPCreateTask_SourceTaskID_NotFound_StillCreatesTask(t *testing.T) { ts, _, workspaceID, workflowID, _ := setupMCPTestServer(t) defer ts.Close() - client := NewWSClient(t, ts.Server.URL) - defer client.Close() - // Non-existent source_task_id must silently fall through (Warn log only), // not cause a validation error. This covers the error-swallow branch at // resolveTaskRepositories:422. - resp, err := client.SendRequest("top-notfound", ws.ActionMCPCreateTask, map[string]interface{}{ + resp, err := dispatchTrustedMCP(t, ts, "top-notfound", ws.ActionMCPCreateTask, map[string]interface{}{ "workspace_id": workspaceID, "workflow_id": workflowID, "title": "Top Level with Missing Source Task", diff --git a/apps/backend/internal/integration/mcp_stop_task_test.go b/apps/backend/internal/integration/mcp_stop_task_test.go new file mode 100644 index 0000000000..307889650e --- /dev/null +++ b/apps/backend/internal/integration/mcp_stop_task_test.go @@ -0,0 +1,334 @@ +package integration + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + mcphandlers "github.com/kandev/kandev/internal/mcp/handlers" + "github.com/kandev/kandev/internal/orchestrator" + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + taskservice "github.com/kandev/kandev/internal/task/service" + "github.com/kandev/kandev/pkg/acp/protocol" + v1 "github.com/kandev/kandev/pkg/api/v1" + ws "github.com/kandev/kandev/pkg/websocket" +) + +const mcpStopResponseDeadline = time.Second + +type mcpStopRecordingAgentManager struct { + *SimulatedAgentManagerClient + promptCalls atomic.Int32 + stopCalls atomic.Int32 + stopEntered chan struct{} + allowStop chan struct{} + stopFinished chan struct{} + enterOnce sync.Once + finishOnce sync.Once + releaseOnce sync.Once +} + +func newMCPStopRecordingAgentManager(eventBus bus.EventBus, log *logger.Logger) *mcpStopRecordingAgentManager { + return &mcpStopRecordingAgentManager{ + SimulatedAgentManagerClient: NewSimulatedAgentManager(eventBus, log), + stopEntered: make(chan struct{}), + allowStop: make(chan struct{}), + stopFinished: make(chan struct{}), + } +} + +func (m *mcpStopRecordingAgentManager) StartAgentProcess( + ctx context.Context, + executionID string, +) error { + if err := m.SimulatedAgentManagerClient.StartAgentProcess(ctx, executionID); err != nil { + return err + } + m.mu.Lock() + instance := m.instances[executionID] + m.mu.Unlock() + if instance != nil { + instance.statusMu.Lock() + instance.status = v1.AgentStatusRunning + instance.statusMu.Unlock() + } + return nil +} + +func (m *mcpStopRecordingAgentManager) PromptAgent( + ctx context.Context, + executionID, prompt string, + attachments []v1.MessageAttachment, + dispatchOnly bool, +) (*executor.PromptResult, error) { + m.promptCalls.Add(1) + return m.SimulatedAgentManagerClient.PromptAgent( + ctx, executionID, prompt, attachments, dispatchOnly, + ) +} + +func (m *mcpStopRecordingAgentManager) StopAgentWithReason( + ctx context.Context, + executionID, reason string, + force bool, +) error { + m.stopCalls.Add(1) + m.enterOnce.Do(func() { close(m.stopEntered) }) + <-m.allowStop + err := m.SimulatedAgentManagerClient.StopAgentWithReason(ctx, executionID, reason, force) + m.finishOnce.Do(func() { close(m.stopFinished) }) + return err +} + +func (m *mcpStopRecordingAgentManager) IsAgentRunningForSession( + _ context.Context, + sessionID string, +) bool { + status, ok := m.runtimeStatus(sessionID) + return ok && status == v1.AgentStatusRunning +} + +func (m *mcpStopRecordingAgentManager) IsAgentReadyForPrompt( + ctx context.Context, + sessionID string, +) bool { + return m.IsAgentRunningForSession(ctx, sessionID) +} + +func (m *mcpStopRecordingAgentManager) runtimeStatus(sessionID string) (v1.AgentStatus, bool) { + m.mu.Lock() + var instance *simulatedInstance + for _, candidate := range m.instances { + if candidate.sessionID == sessionID { + instance = candidate + break + } + } + m.mu.Unlock() + if instance == nil { + return "", false + } + instance.statusMu.Lock() + defer instance.statusMu.Unlock() + return instance.status, true +} + +func (m *mcpStopRecordingAgentManager) releaseStop() { + m.releaseOnce.Do(func() { close(m.allowStop) }) +} + +type mcpStopDispatchOutcome struct { + response *ws.Message + err error + elapsed time.Duration +} + +func TestMCPStopTask_DirectParentStopsLongRunningChild(t *testing.T) { + ts, parentTaskID, _, _, _ := setupMCPTestServer(t) + defer ts.Close() + + manager := newMCPStopRecordingAgentManager(ts.EventBus, ts.Logger) + manager.SetLaunchDelay(10 * time.Millisecond) + manager.SetExecutionTime(30 * time.Second) + manager.SetACPMessageFn(func(taskID, _ string) []protocol.Message { + return []protocol.Message{{ + Type: protocol.MessageTypeProgress, TaskID: taskID, Timestamp: time.Now(), + Data: map[string]interface{}{ + "type": "tool_call", + "tool_call_id": "long-running-work", + "tool_name": "integration_wait", + "tool_title": "Long-running integration work", + "tool_status": "running", + }, + }} + }) + + cfg := orchestrator.DefaultServiceConfig() + cfg.Scheduler.ProcessInterval = 50 * time.Millisecond + taskRepoAdapter := &taskRepositoryAdapter{repo: ts.TaskRepo, svc: ts.TaskSvc} + orchestratorSvc := orchestrator.NewService( + cfg, ts.EventBus, manager, taskRepoAdapter, ts.TaskRepo, nil, nil, nil, ts.Logger, + ) + orchestratorSvc.SetMessageCreator(&testMessageCreatorAdapter{svc: ts.TaskSvc}) + orchestratorSvc.SetTurnService(&testTurnServiceAdapter{svc: ts.TaskSvc}) + orchestratorCtx, cancelOrchestrator := context.WithCancel(context.Background()) + require.NoError(t, orchestratorSvc.Start(orchestratorCtx)) + defer func() { + if err := orchestratorSvc.Stop(); err != nil { + t.Errorf("stop orchestrator: %v", err) + } + manager.Close() + cancelOrchestrator() + }() + defer manager.releaseStop() + + // Register the real backend MCP handler on the shared dispatcher. Calls below + // dispatch the internal action directly: local task-mode MCP transport is + // intentionally outside this integration test and has separate adapter tests. + mcpHandlers := mcphandlers.NewHandlers( + ts.TaskSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, + orchestratorSvc, orchestratorSvc.GetMessageQueue(), ts.Logger, + ) + mcpHandlers.RegisterHandlers(ts.Gateway.Dispatcher) + + parent, err := ts.TaskRepo.GetTask(context.Background(), parentTaskID) + require.NoError(t, err) + child, err := ts.TaskSvc.CreateTask(context.Background(), &taskservice.CreateTaskRequest{ + WorkspaceID: parent.WorkspaceID, WorkflowID: parent.WorkflowID, WorkflowStepID: parent.WorkflowStepID, + Title: "Long-running child", Description: "Keep working until the parent stops this task.", + Priority: "medium", ParentID: parentTaskID, + }) + require.NoError(t, err) + + running := make(chan struct{}) + var runningOnce sync.Once + subscription, err := ts.EventBus.Subscribe(events.TaskSessionStateChanged, func(_ context.Context, event *bus.Event) error { + data, ok := event.Data.(map[string]interface{}) + if ok && data["task_id"] == child.ID && data["new_state"] == string(models.TaskSessionStateRunning) { + runningOnce.Do(func() { close(running) }) + } + return nil + }) + require.NoError(t, err) + defer func() { + if err := subscription.Unsubscribe(); err != nil { + t.Errorf("unsubscribe session-state observer: %v", err) + } + }() + + launch, err := orchestratorSvc.LaunchSession(context.Background(), &orchestrator.LaunchSessionRequest{ + TaskID: child.ID, Intent: orchestrator.IntentStart, AgentProfileID: "integration-agent", + Prompt: "Keep working until explicitly stopped.", + }) + require.NoError(t, err) + require.True(t, launch.Success) + require.NotEmpty(t, launch.SessionID) + mcpStopAwaitSignal(t, running, 2*time.Second, "child RUNNING state") + + sessionBefore, err := ts.TaskRepo.GetTaskSession(context.Background(), launch.SessionID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, sessionBefore.State) + taskBefore, err := ts.TaskRepo.GetTask(context.Background(), child.ID) + require.NoError(t, err) + require.Equal(t, v1.TaskStateInProgress, taskBefore.State) + require.True(t, manager.IsAgentRunningForSession(context.Background(), launch.SessionID)) + promptsBefore := manager.promptCalls.Load() + messagesBefore, err := ts.TaskRepo.ListMessages(context.Background(), launch.SessionID) + require.NoError(t, err) + + stopRequest, err := ws.NewRequest("mcp-stop-1", ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": child.ID, "sender_task_id": parentTaskID, + }) + require.NoError(t, err) + dispatchDone := make(chan mcpStopDispatchOutcome, 1) + dispatchStarted := time.Now() + go func() { + response, dispatchErr := ts.Gateway.Dispatcher.Dispatch(context.Background(), stopRequest) + dispatchDone <- mcpStopDispatchOutcome{ + response: response, err: dispatchErr, elapsed: time.Since(dispatchStarted), + } + }() + + mcpStopAwaitSignal(t, manager.stopEntered, mcpStopResponseDeadline, "asynchronous runtime teardown") + first := mcpStopAwaitDispatch(t, dispatchDone, mcpStopResponseDeadline) + require.NoError(t, first.err) + require.Less(t, first.elapsed, mcpStopResponseDeadline) + require.Equal(t, ws.MessageTypeResponse, first.response.Type) + firstPayload := mcpStopParseResponse(t, first.response) + require.Equal(t, child.ID, firstPayload.TaskID) + require.Equal(t, orchestrator.CoordinatorTaskStopStatusStopped, firstPayload.Status) + + // The response lands while simulated teardown is deliberately blocked, + // proving it confirms logical cancellation rather than process exit. + require.True(t, manager.IsAgentRunningForSession(context.Background(), launch.SessionID)) + stoppedSession, err := ts.TaskRepo.GetTaskSession(context.Background(), launch.SessionID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, stoppedSession.State) + stoppedTask, err := ts.TaskRepo.GetTask(context.Background(), child.ID) + require.NoError(t, err) + require.Equal(t, v1.TaskStateReview, stoppedTask.State) + require.Equal(t, promptsBefore, manager.promptCalls.Load(), "stop must not dispatch a replacement prompt") + require.Zero(t, orchestratorSvc.GetMessageQueue().GetStatus(context.Background(), launch.SessionID).Count) + + manager.releaseStop() + mcpStopAwaitSignal(t, manager.stopFinished, 2*time.Second, "simulated runtime stop") + status, exists := manager.runtimeStatus(launch.SessionID) + require.True(t, exists) + require.Equal(t, v1.AgentStatusStopped, status) + require.False(t, manager.IsAgentRunningForSession(context.Background(), launch.SessionID)) + + messagesAfterStop, err := ts.TaskRepo.ListMessages(context.Background(), launch.SessionID) + require.NoError(t, err) + require.Len(t, messagesAfterStop, len(messagesBefore)) + launchesAfterStop := manager.GetLaunchCount() + promptsAfterStop := manager.promptCalls.Load() + stopCallsAfterStop := manager.stopCalls.Load() + + repeatRequest, err := ws.NewRequest("mcp-stop-2", ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": child.ID, "sender_task_id": parentTaskID, + }) + require.NoError(t, err) + repeatResponse, err := ts.Gateway.Dispatcher.Dispatch(context.Background(), repeatRequest) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, repeatResponse.Type) + repeatPayload := mcpStopParseResponse(t, repeatResponse) + require.Equal(t, child.ID, repeatPayload.TaskID) + require.Equal(t, orchestrator.CoordinatorTaskStopStatusNotRunning, repeatPayload.Status) + + repeatedSession, err := ts.TaskRepo.GetTaskSession(context.Background(), launch.SessionID) + require.NoError(t, err) + repeatedTask, err := ts.TaskRepo.GetTask(context.Background(), child.ID) + require.NoError(t, err) + require.Equal(t, stoppedSession.UpdatedAt, repeatedSession.UpdatedAt) + require.Equal(t, stoppedTask.UpdatedAt, repeatedTask.UpdatedAt) + require.Equal(t, launchesAfterStop, manager.GetLaunchCount()) + require.Equal(t, promptsAfterStop, manager.promptCalls.Load()) + require.Equal(t, stopCallsAfterStop, manager.stopCalls.Load()) + repeatedMessages, err := ts.TaskRepo.ListMessages(context.Background(), launch.SessionID) + require.NoError(t, err) + require.Len(t, repeatedMessages, len(messagesAfterStop)) +} + +type mcpStopResponsePayload struct { + TaskID string `json:"task_id"` + Status orchestrator.CoordinatorTaskStopStatus `json:"status"` +} + +func mcpStopParseResponse(t *testing.T, response *ws.Message) mcpStopResponsePayload { + t.Helper() + var payload mcpStopResponsePayload + require.NoError(t, response.ParsePayload(&payload)) + return payload +} + +func mcpStopAwaitSignal(t *testing.T, signal <-chan struct{}, timeout time.Duration, description string) { + t.Helper() + select { + case <-signal: + case <-time.After(timeout): + t.Fatalf("timed out waiting for %s", description) + } +} + +func mcpStopAwaitDispatch( + t *testing.T, + done <-chan mcpStopDispatchOutcome, + timeout time.Duration, +) mcpStopDispatchOutcome { + t.Helper() + select { + case outcome := <-done: + return outcome + case <-time.After(timeout): + t.Fatal("coordinator stop waited for simulated runtime teardown") + return mcpStopDispatchOutcome{} + } +} diff --git a/apps/backend/internal/integration/test_server_test.go b/apps/backend/internal/integration/test_server_test.go index 5564a03074..771e2ff1a3 100644 --- a/apps/backend/internal/integration/test_server_test.go +++ b/apps/backend/internal/integration/test_server_test.go @@ -79,6 +79,15 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived( return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state) } +func (a *taskRepositoryAdapter) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + return a.svc.UpdateTaskStateIfSessionState(ctx, taskID, sessionID, expectedSessionState, state) +} + // testMessageCreatorAdapter adapts the task service to the orchestrator.MessageCreator interface for tests type testMessageCreatorAdapter struct { svc *taskservice.Service diff --git a/apps/backend/internal/mcp/handlers/clarification_pause_test.go b/apps/backend/internal/mcp/handlers/clarification_pause_test.go index eeaa2e3be9..b4d5ef3296 100644 --- a/apps/backend/internal/mcp/handlers/clarification_pause_test.go +++ b/apps/backend/internal/mcp/handlers/clarification_pause_test.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/clarification" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/task/service" + v1 "github.com/kandev/kandev/pkg/api/v1" ws "github.com/kandev/kandev/pkg/websocket" "github.com/stretchr/testify/require" ) @@ -35,6 +36,247 @@ func (c *recordingSessionCanceller) DetachSessionAndNotify(_ context.Context, se return c.count } +type immediateClarificationService struct { + response *clarification.Response + waitErr error +} + +func (s *immediateClarificationService) CreateRequest(*clarification.Request) (string, bool) { + return "pending-race", true +} + +func (s *immediateClarificationService) WaitForResponse(context.Context, string) (*clarification.Response, error) { + return s.response, s.waitErr +} + +func (s *immediateClarificationService) CancelRequest(string) bool { return true } + +// clarificationCoordinatorStopRaceRepo simulates the coordinator committing a +// stop immediately before one clarification state write. The legacy +// unconditional method demonstrates the stale-writer bug by restoring the +// target state; the conditional method preserves CANCELLED. +type clarificationCoordinatorStopRaceRepo struct { + SessionRepository + taskRepo TaskRepository + targetState models.TaskSessionState + stopped bool +} + +func (r *clarificationCoordinatorStopRaceRepo) stopBeforeTarget( + ctx context.Context, + sessionID string, + state models.TaskSessionState, +) error { + if r.stopped || state != r.targetState { + return nil + } + r.stopped = true + session, err := r.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if err := r.SessionRepository.UpdateTaskSessionState( + ctx, + sessionID, + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + ); err != nil { + return err + } + return r.taskRepo.UpdateTaskState(ctx, session.TaskID, v1.TaskStateReview) +} + +func (r *clarificationCoordinatorStopRaceRepo) UpdateTaskSessionState( + ctx context.Context, + sessionID string, + state models.TaskSessionState, + errorMessage string, +) error { + if err := r.stopBeforeTarget(ctx, sessionID, state); err != nil { + return err + } + return r.SessionRepository.UpdateTaskSessionState(ctx, sessionID, state, errorMessage) +} + +func (r *clarificationCoordinatorStopRaceRepo) UpdateTaskSessionStateIfCurrent( + ctx context.Context, + sessionID string, + expected, state models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + if err := r.stopBeforeTarget(ctx, sessionID, state); err != nil { + return false, time.Time{}, err + } + updater := r.SessionRepository.(interface { + UpdateTaskSessionStateIfCurrent( + context.Context, + string, + models.TaskSessionState, + models.TaskSessionState, + string, + ) (bool, time.Time, error) + }) + return updater.UpdateTaskSessionStateIfCurrent(ctx, sessionID, expected, state, errorMessage) +} + +// clarificationTaskStateRaceRepo commits coordinator cancellation after the +// clarification session CAS has restored RUNNING but before its task-state +// write. The underlying session-owned task CAS must reject IN_PROGRESS. +type clarificationTaskStateRaceRepo struct { + TaskRepository + sessionRepo SessionRepository + stopped bool +} + +func (r *clarificationTaskStateRaceRepo) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (v1.TaskState, bool, error) { + if !r.stopped { + r.stopped = true + if err := r.sessionRepo.UpdateTaskSessionState( + ctx, + sessionID, + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + ); err != nil { + return "", false, err + } + if err := r.UpdateTaskState(ctx, taskID, v1.TaskStateReview); err != nil { + return "", false, err + } + } + updater := r.TaskRepository.(sessionOwnedTaskStateUpdater) + return updater.UpdateTaskStateIfSessionState( + ctx, + taskID, + sessionID, + expectedSessionState, + state, + ) +} + +func askUserQuestionRaceMessage(t *testing.T, taskID, sessionID string) *ws.Message { + t.Helper() + return makeWSMessage(t, ws.ActionMCPAskUserQuestion, map[string]interface{}{ + "session_id": sessionID, + "task_id": taskID, + "questions": []map[string]interface{}{ + { + "prompt": "What colour?", + "options": []map[string]interface{}{ + {"label": "Red", "description": "R"}, + {"label": "Blue", "description": "B"}, + }, + }, + }, + }) +} + +func TestHandleAskUserQuestion_CoordinatorStopWinsWaitingTransition(t *testing.T) { + _, repo := newTestTaskService(t) + const taskID = "task-clarification-wait-race" + const sessionID = "session-clarification-wait-race" + seedMCPHandlerSession(t, repo, taskID, sessionID, models.TaskSessionStateRunning) + + racingRepo := &clarificationCoordinatorStopRaceRepo{ + SessionRepository: repo, + taskRepo: repo, + targetState: models.TaskSessionStateWaitingForInput, + } + eventBus := &mcpRecordingEventBus{} + h := &Handlers{ + clarificationSvc: &immediateClarificationService{waitErr: errors.New("wait cancelled")}, + sessionRepo: racingRepo, + taskRepo: repo, + eventBus: eventBus, + logger: testLogger(t).WithFields(), + } + + _, err := h.handleAskUserQuestion(context.Background(), askUserQuestionRaceMessage(t, taskID, sessionID)) + require.NoError(t, err) + + session, err := repo.GetTaskSession(context.Background(), sessionID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + task, err := repo.GetTask(context.Background(), taskID) + require.NoError(t, err) + require.Equal(t, "REVIEW", string(task.State)) + require.Empty(t, eventBus.events, "rejected WAITING_FOR_INPUT transition must not publish") +} + +func TestHandleAskUserQuestion_CoordinatorStopWinsRunningTransition(t *testing.T) { + _, repo := newTestTaskService(t) + const taskID = "task-clarification-resume-race" + const sessionID = "session-clarification-resume-race" + seedMCPHandlerSession(t, repo, taskID, sessionID, models.TaskSessionStateRunning) + + racingRepo := &clarificationCoordinatorStopRaceRepo{ + SessionRepository: repo, + taskRepo: repo, + targetState: models.TaskSessionStateRunning, + } + eventBus := &mcpRecordingEventBus{} + h := &Handlers{ + clarificationSvc: &immediateClarificationService{response: &clarification.Response{}}, + sessionRepo: racingRepo, + taskRepo: repo, + eventBus: eventBus, + logger: testLogger(t).WithFields(), + } + + _, err := h.handleAskUserQuestion(context.Background(), askUserQuestionRaceMessage(t, taskID, sessionID)) + require.NoError(t, err) + + session, err := repo.GetTaskSession(context.Background(), sessionID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + task, err := repo.GetTask(context.Background(), taskID) + require.NoError(t, err) + require.Equal(t, "REVIEW", string(task.State)) + require.Len(t, eventBus.events, 1, "only accepted WAITING_FOR_INPUT transition may publish") + eventData := eventBus.events[0].Data.(map[string]interface{}) + require.Equal(t, string(models.TaskSessionStateWaitingForInput), eventData["new_state"]) +} + +func TestHandleAskUserQuestion_CoordinatorStopWinsAfterRunningTransition(t *testing.T) { + _, repo := newTestTaskService(t) + const taskID = "task-clarification-task-race" + const sessionID = "session-clarification-task-race" + seedMCPHandlerSession(t, repo, taskID, sessionID, models.TaskSessionStateRunning) + + racingTaskRepo := &clarificationTaskStateRaceRepo{ + TaskRepository: repo, + sessionRepo: repo, + } + eventBus := &mcpRecordingEventBus{} + h := &Handlers{ + clarificationSvc: &immediateClarificationService{response: &clarification.Response{}}, + sessionRepo: repo, + taskRepo: racingTaskRepo, + eventBus: eventBus, + logger: testLogger(t).WithFields(), + } + + _, err := h.handleAskUserQuestion( + context.Background(), + askUserQuestionRaceMessage(t, taskID, sessionID), + ) + require.NoError(t, err) + + session, err := repo.GetTaskSession(context.Background(), sessionID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + task, err := repo.GetTask(context.Background(), taskID) + require.NoError(t, err) + require.Equal(t, v1.TaskStateReview, task.State) + require.Len(t, eventBus.events, 1, "stale RUNNING transition must not publish") + eventData := eventBus.events[0].Data.(map[string]interface{}) + require.Equal(t, string(models.TaskSessionStateWaitingForInput), eventData["new_state"]) +} + func TestHandleClarificationTimeout_UsesHardPauser(t *testing.T) { pauser := &recordingClarificationInputPauser{count: 2} h := &Handlers{logger: testLogger(t).WithFields()} diff --git a/apps/backend/internal/mcp/handlers/handlers.go b/apps/backend/internal/mcp/handlers/handlers.go index 2646775e3f..1b2e80784b 100644 --- a/apps/backend/internal/mcp/handlers/handlers.go +++ b/apps/backend/internal/mcp/handlers/handlers.go @@ -70,11 +70,37 @@ type SessionRepository interface { SetSessionMetadataKey(ctx context.Context, sessionID, key string, value interface{}) error } +// conditionalSessionStateUpdater is implemented by repositories that can +// reject stale session-state writers. Keeping it optional preserves existing +// handler fakes and alternate repositories while production SQLite gets CAS +// protection against coordinator stops. +type conditionalSessionStateUpdater interface { + UpdateTaskSessionStateIfCurrent( + ctx context.Context, + sessionID string, + expected, state models.TaskSessionState, + errorMessage string, + ) (bool, time.Time, error) +} + // TaskRepository interface for updating task state. type TaskRepository interface { UpdateTaskState(ctx context.Context, taskID string, state v1.TaskState) error } +// sessionOwnedTaskStateUpdater atomically guards a task-state write with the +// current state of its owning session. Production SQLite implements this so a +// clarification answer cannot restore IN_PROGRESS after coordinator stop has +// already committed CANCELLED. +type sessionOwnedTaskStateUpdater interface { + UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, + ) (v1.TaskState, bool, error) +} + // EventBus interface for publishing events. type EventBus interface { Publish(ctx context.Context, topic string, event *bus.Event) error @@ -105,6 +131,13 @@ type SessionLauncher interface { RenameSession(ctx context.Context, sessionID, name string) error } +// TaskStopper exposes the narrow coordinator halt operation used by +// stop_task_kandev. The MCP layer owns authorization; lifecycle semantics stay +// in the orchestrator. +type TaskStopper interface { + StopTaskForCoordinator(ctx context.Context, taskID string) (orchestrator.CoordinatorTaskStopResult, error) +} + // MessageQueuer queues a prompt message for delivery to a session on its next turn. // TakeQueued is exposed so move_task can roll back the hand-off prompt when the // underlying MoveTask call fails — without it, a queued "you were moved..." @@ -136,6 +169,8 @@ type Handlers struct { planService *service.PlanService walkthroughService *service.WalkthroughService sessionLauncher SessionLauncher + taskStopper TaskStopper + stopTaskGetter func(context.Context, string) (*models.Task, error) messageQueue MessageQueuer promptResolver PromptReferenceResolver logger *logger.Logger @@ -171,7 +206,7 @@ func NewHandlers( messageQueue MessageQueuer, log *logger.Logger, ) *Handlers { - return &Handlers{ + h := &Handlers{ taskSvc: taskSvc, workflowCtrl: workflowCtrl, clarificationSvc: clarificationSvc, @@ -186,6 +221,13 @@ func NewHandlers( messageQueue: messageQueue, logger: log.WithFields(zap.String("component", "mcp-handlers")), } + if taskSvc != nil { + h.stopTaskGetter = taskSvc.GetTask + } + if stopper, ok := sessionLauncher.(TaskStopper); ok { + h.taskStopper = stopper + } + return h } // SetClarificationInputPauser wires the orchestrator-owned hard pause used when @@ -198,6 +240,11 @@ func (h *Handlers) SetPromptReferenceResolver(resolver PromptReferenceResolver) h.promptResolver = resolver } +// SetTaskStopper wires the orchestrator-owned halt operation. +func (h *Handlers) SetTaskStopper(stopper TaskStopper) { + h.taskStopper = stopper +} + // SetConfigDeps sets the config-mode dependencies for agent-native configuration handlers. // These are optional and only needed when config-mode MCP sessions are used. func (h *Handlers) SetConfigDeps( @@ -224,6 +271,7 @@ func (h *Handlers) RegisterHandlers(d *ws.Dispatcher) { d.RegisterFunc(ws.ActionMCPUpdateRepositoryBaseBranch, h.handleUpdateRepositoryBaseBranch) d.RegisterFunc(ws.ActionMCPStepComplete, h.handleStepComplete) d.RegisterFunc(ws.ActionMCPMessageTask, h.handleMessageTask) + d.RegisterFunc(ws.ActionMCPStopTask, h.handleStopTask) d.RegisterFunc(ws.ActionMCPSpawnSession, h.handleSpawnSession) d.RegisterFunc(ws.ActionMCPGetTaskConversation, h.handleGetTaskConversation) d.RegisterFunc(ws.ActionMCPAskUserQuestion, h.handleAskUserQuestion) @@ -240,7 +288,7 @@ func (h *Handlers) RegisterHandlers(d *ws.Dispatcher) { d.RegisterFunc(ws.ActionTaskWalkthroughGet, h.handleGetWalkthrough) d.RegisterFunc(ws.ActionTaskWalkthroughDelete, h.handleDeleteWalkthrough) d.RegisterFunc(ws.ActionMCPClarificationTimeout, h.handleClarificationTimeout) - count := 24 + count := 25 // Config-mode handlers (registered when config deps are set) if h.workflowSvc != nil { @@ -2016,12 +2064,18 @@ type taskMessageQueueRollback struct { type taskMessageSessionRollbackRepository interface { GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error) - UpdateTaskSession(ctx context.Context, session *models.TaskSession) error + UpdateTaskSessionIfCurrentState( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, + ) (bool, error) SetSessionPrimary(ctx context.Context, sessionID string) error DeleteTaskSession(ctx context.Context, id string) error UpdateSessionMetadata(ctx context.Context, sessionID string, metadata map[string]interface{}) error } +var errTaskMessageRollbackSuperseded = errors.New("task message rollback superseded by coordinator cancellation") + type taskMessageExecutorReader interface { GetExecutorRunningBySessionID(ctx context.Context, sessionID string) (*models.ExecutorRunning, error) } @@ -2399,24 +2453,63 @@ func (h *Handlers) restoreTaskReviewForTaskMessage(ctx context.Context, taskID s h.logger.Warn("failed to restore task session after task message dispatch failure", zap.String("task_id", taskID), zap.Error(err)) - } - if !rollback.restoreTask { + // A coordinator cancellation owns the terminal session/task state and + // intentionally preserves the current queue. Never continue into task + // or queue snapshot restoration after that ownership is observed. return } - taskState := rollback.taskState - if taskState == "" { - taskState = v1.TaskStateReview + if rollback.restoreTask { + ownerID, ownerState, ok := rollback.taskRestoreOwner() + if !ok { + h.logger.Warn("skipping task message rollback without an owning session", + zap.String("task_id", taskID)) + return + } + taskState := rollback.taskState + if taskState == "" { + taskState = v1.TaskStateReview + } + _, restored, err := h.taskSvc.RestoreTaskMessageRollback( + ctx, + taskID, + ownerID, + ownerState, + taskState, + rollback.workflowStepID, + ) + if err != nil { + h.logger.Warn("failed to restore task after task message dispatch failure", + zap.String("task_id", taskID), + zap.Error(err)) + return + } + if !restored { + h.logger.Info("skipping task and queue rollback because session ownership changed", + zap.String("task_id", taskID), + zap.String("session_id", ownerID)) + return + } } - if _, err := h.taskSvc.UpdateTask(ctx, taskID, &service.UpdateTaskRequest{ - State: &taskState, - WorkflowStepID: &rollback.workflowStepID, - }); err != nil { - h.logger.Warn("failed to restore task to REVIEW after task message dispatch failure", + if err := h.restoreTaskMessageQueues(ctx, rollback); err != nil { + h.logger.Warn("failed to restore task message queue after dispatch failure", zap.String("task_id", taskID), zap.Error(err)) } } +func (r taskMessageReviewRollback) taskRestoreOwner() (string, models.TaskSessionState, bool) { + primaryID := r.primarySessionID() + for _, snapshot := range r.sessions { + if snapshot.sessionID == primaryID { + return snapshot.sessionID, snapshot.state, true + } + } + if len(r.sessions) == 0 { + return "", "", false + } + return r.sessions[0].sessionID, r.sessions[0].state, true +} + func (h *Handlers) restoreTaskMessageSessions(ctx context.Context, rollback taskMessageReviewRollback) error { if len(rollback.sessions) == 0 { return nil @@ -2430,9 +2523,6 @@ func (h *Handlers) restoreTaskMessageSessions(ctx context.Context, rollback task return err } } - if err := h.restoreTaskMessageQueues(ctx, rollback); err != nil { - return err - } return h.restoreSelectedTaskMessageSession(ctx, repo, rollback) } @@ -2444,6 +2534,13 @@ func (h *Handlers) restoreSelectedTaskMessageSession(ctx context.Context, repo t if _, ok := rollback.sessionIDs[rollback.selectedID]; ok { return nil } + selected, err := repo.GetTaskSession(ctx, rollback.selectedID) + if err != nil { + return err + } + if selected != nil && selected.State == models.TaskSessionStateCancelled { + return errTaskMessageRollbackSuperseded + } if primaryID != "" && rollback.selectedID != primaryID { h.clearTaskMessageQueue(ctx, rollback.selectedID) } @@ -2501,6 +2598,13 @@ func restoreTaskMessageSessionSnapshot(ctx context.Context, repo taskMessageSess if err != nil { return err } + if session == nil { + return fmt.Errorf("task message rollback session %q is nil", rollback.sessionID) + } + if session.State == models.TaskSessionStateCancelled { + return errTaskMessageRollbackSuperseded + } + expectedState := session.State session.State = rollback.state session.ErrorMessage = rollback.error session.CompletedAt = rollback.completedAt @@ -2508,9 +2612,20 @@ func restoreTaskMessageSessionSnapshot(ctx context.Context, repo taskMessageSess session.AgentProfileID = rollback.agentProfileID session.ExecutorProfileID = rollback.executorProfileID session.AgentProfileSnapshot = cloneTaskMessageMetadataMap(rollback.agentProfileSnapshot) - if err := repo.UpdateTaskSession(ctx, session); err != nil { + changed, err := repo.UpdateTaskSessionIfCurrentState(ctx, session, expectedState) + if err != nil { return err } + if !changed { + latest, loadErr := repo.GetTaskSession(ctx, rollback.sessionID) + if loadErr != nil { + return loadErr + } + if latest != nil && latest.State == models.TaskSessionStateCancelled { + return errTaskMessageRollbackSuperseded + } + return fmt.Errorf("task message rollback lost session %q state ownership", rollback.sessionID) + } if rollback.isPrimary { if err := repo.SetSessionPrimary(ctx, rollback.sessionID); err != nil { return err @@ -2750,17 +2865,34 @@ func (h *Handlers) handleAskUserQuestion(ctx context.Context, msg *ws.Message) ( // setSessionRunning restores the session state to running after a clarification is answered. func (h *Handlers) setSessionRunning(ctx context.Context, taskID, sessionID string) { - if err := h.sessionRepo.UpdateTaskSessionState(ctx, sessionID, models.TaskSessionStateRunning, ""); err != nil { + changed, updatedAt, err := h.updateClarificationSessionState( + ctx, + sessionID, + models.TaskSessionStateWaitingForInput, + models.TaskSessionStateRunning, + ) + if err != nil { h.logger.Warn("failed to update session state to RUNNING", zap.String("session_id", sessionID), zap.Error(err)) return } + if !changed { + return + } if taskID != "" { - if err := h.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateInProgress); err != nil { + taskStateChanged, err := h.setTaskInProgressForClarification(ctx, taskID, sessionID) + if err != nil { h.logger.Warn("failed to update task state to IN_PROGRESS", zap.String("task_id", taskID), zap.Error(err)) + return + } + if !taskStateChanged { + h.logger.Debug("skipping stale clarification resume after session state changed", + zap.String("task_id", taskID), + zap.String("session_id", sessionID)) + return } } @@ -2771,8 +2903,10 @@ func (h *Handlers) setSessionRunning(ctx context.Context, taskID, sessionID stri "session_id": sessionID, "new_state": string(models.TaskSessionStateRunning), } - if updatedAt, ok := h.sessionUpdatedAtForStateEvent(ctx, sessionID); ok { - eventData["updated_at"] = updatedAt + if !updatedAt.IsZero() { + eventData["updated_at"] = updatedAt.UTC().Format(time.RFC3339Nano) + } else if persistedUpdatedAt, ok := h.sessionUpdatedAtForStateEvent(ctx, sessionID); ok { + eventData["updated_at"] = persistedUpdatedAt } else { h.logger.Warn("skipping session state_changed publish; could not load authoritative updated_at", zap.String("session_id", sessionID)) @@ -2786,15 +2920,38 @@ func (h *Handlers) setSessionRunning(ctx context.Context, taskID, sessionID stri } } +func (h *Handlers) setTaskInProgressForClarification( + ctx context.Context, + taskID, sessionID string, +) (bool, error) { + if updater, ok := h.taskRepo.(sessionOwnedTaskStateUpdater); ok { + _, updated, err := updater.UpdateTaskStateIfSessionState( + ctx, + taskID, + sessionID, + models.TaskSessionStateRunning, + v1.TaskStateInProgress, + ) + return updated, err + } + if err := h.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateInProgress); err != nil { + return false, err + } + return true, nil +} + // setSessionWaitingForInput updates the session and task states to waiting for input func (h *Handlers) setSessionWaitingForInput(ctx context.Context, taskID, sessionID string) { - // Update session state to WAITING_FOR_INPUT - if err := h.sessionRepo.UpdateTaskSessionState(ctx, sessionID, models.TaskSessionStateWaitingForInput, ""); err != nil { + changed, updatedAt, err := h.updateSessionWaitingForClarification(ctx, sessionID) + if err != nil { h.logger.Warn("failed to update session state to WAITING_FOR_INPUT", zap.String("session_id", sessionID), zap.Error(err)) return } + if !changed { + return + } // Update task state to REVIEW if taskID != "" { @@ -2812,8 +2969,10 @@ func (h *Handlers) setSessionWaitingForInput(ctx context.Context, taskID, sessio "session_id": sessionID, "new_state": string(models.TaskSessionStateWaitingForInput), } - if updatedAt, ok := h.sessionUpdatedAtForStateEvent(ctx, sessionID); ok { - eventData["updated_at"] = updatedAt + if !updatedAt.IsZero() { + eventData["updated_at"] = updatedAt.UTC().Format(time.RFC3339Nano) + } else if persistedUpdatedAt, ok := h.sessionUpdatedAtForStateEvent(ctx, sessionID); ok { + eventData["updated_at"] = persistedUpdatedAt } else { h.logger.Warn("skipping session state_changed publish; could not load authoritative updated_at", zap.String("session_id", sessionID)) @@ -2827,6 +2986,52 @@ func (h *Handlers) setSessionWaitingForInput(ctx context.Context, taskID, sessio } } +func (h *Handlers) updateSessionWaitingForClarification( + ctx context.Context, + sessionID string, +) (bool, time.Time, error) { + // A fast MCP clarification can arrive before agent startup promotes the + // session from STARTING to RUNNING. Re-read and retry once if that promotion + // races the CAS; terminal states still reject the write. + for attempt := 0; attempt < 2; attempt++ { + session, err := h.sessionRepo.GetTaskSession(ctx, sessionID) + if err != nil { + return false, time.Time{}, err + } + if session == nil { + return false, time.Time{}, fmt.Errorf("session %q not found", sessionID) + } + if session.State != models.TaskSessionStateStarting && + session.State != models.TaskSessionStateRunning { + return false, time.Time{}, nil + } + changed, updatedAt, err := h.updateClarificationSessionState( + ctx, + sessionID, + session.State, + models.TaskSessionStateWaitingForInput, + ) + if err != nil || changed { + return changed, updatedAt, err + } + } + return false, time.Time{}, nil +} + +func (h *Handlers) updateClarificationSessionState( + ctx context.Context, + sessionID string, + expected, state models.TaskSessionState, +) (bool, time.Time, error) { + if updater, ok := h.sessionRepo.(conditionalSessionStateUpdater); ok { + return updater.UpdateTaskSessionStateIfCurrent(ctx, sessionID, expected, state, "") + } + if err := h.sessionRepo.UpdateTaskSessionState(ctx, sessionID, state, ""); err != nil { + return false, time.Time{}, err + } + return true, time.Time{}, nil +} + func (h *Handlers) sessionUpdatedAtForStateEvent(ctx context.Context, sessionID string) (string, bool) { if session, err := h.sessionRepo.GetTaskSession(ctx, sessionID); err == nil && session != nil && !session.UpdatedAt.IsZero() { return session.UpdatedAt.UTC().Format(time.RFC3339Nano), true diff --git a/apps/backend/internal/mcp/handlers/handlers_test.go b/apps/backend/internal/mcp/handlers/handlers_test.go index 8d764b0fff..540c1056b7 100644 --- a/apps/backend/internal/mcp/handlers/handlers_test.go +++ b/apps/backend/internal/mcp/handlers/handlers_test.go @@ -233,6 +233,12 @@ func TestSessionStateEventsIncludeUpdatedAt(t *testing.T) { run: (*Handlers).setSessionWaitingForInput, wantState: models.TaskSessionStateWaitingForInput, }, + { + name: "fast clarification waits while starting", + initial: models.TaskSessionStateStarting, + run: (*Handlers).setSessionWaitingForInput, + wantState: models.TaskSessionStateWaitingForInput, + }, } for _, tt := range tests { diff --git a/apps/backend/internal/mcp/handlers/message_task_test.go b/apps/backend/internal/mcp/handlers/message_task_test.go index 97a26dd751..ad19772737 100644 --- a/apps/backend/internal/mcp/handlers/message_task_test.go +++ b/apps/backend/internal/mcp/handlers/message_task_test.go @@ -1528,6 +1528,85 @@ func TestHandleMessageTask_DispatchErrorRollsBackTurnStartOutsideReview(t *testi assert.Equal(t, "original queued", status.Entries[0].Content) } +func TestHandleMessageTask_DispatchRollbackDoesNotOverwriteCoordinatorStop(t *testing.T) { + ctx := context.Background() + svc, repo, _ := newTestTaskServiceWithEventBus(t) + sender, target, session := seedTaskWithSession( + t, + svc, + repo, + models.TaskSessionStateWaitingForInput, + ) + task, err := svc.GetTask(ctx, target.ID) + require.NoError(t, err) + task.State = v1.TaskStateReview + task.WorkflowStepID = "step-before-message" + require.NoError(t, repo.UpdateTask(ctx, task)) + + h, orch := newMessageTaskHandler(t, svc, repo) + _, err = orch.queue.QueueMessageWithMetadata( + ctx, + session.ID, + target.ID, + "queued before dispatch", + "", + "agent", + false, + nil, + nil, + ) + require.NoError(t, err) + orch.onTurnStart = func(ctx context.Context, taskID, sessionID string) error { + // The rollback snapshot is already captured when this callback runs. + // Model a complete coordinator stop before dispatch reports failure. + changed, _, cancelErr := repo.CancelActiveTaskSession( + ctx, + sessionID, + "stopped by parent task via MCP", + ) + require.NoError(t, cancelErr) + require.True(t, changed) + stoppedTask, loadErr := svc.GetTask(ctx, taskID) + require.NoError(t, loadErr) + stoppedTask.State = v1.TaskStateReview + stoppedTask.WorkflowStepID = "step-owned-by-stop" + require.NoError(t, repo.UpdateTask(ctx, stoppedTask)) + _, queueErr := orch.queue.QueueMessageWithMetadata( + ctx, + sessionID, + taskID, + "queued after stop", + "", + "agent", + false, + nil, + nil, + ) + return queueErr + } + + msg := makeWSMessage( + t, + ws.ActionMCPMessageTask, + senderPayload(target.ID, "dispatch races stop", sender.ID), + ) + resp, err := h.handleMessageTask(ctx, msg) + require.NoError(t, err) + assertWSError(t, resp, ws.ErrorCodeInternalError) + + stoppedSession, err := svc.GetTaskSession(ctx, session.ID) + require.NoError(t, err) + assert.Equal(t, models.TaskSessionStateCancelled, stoppedSession.State) + updatedTask, err := svc.GetTask(ctx, target.ID) + require.NoError(t, err) + assert.Equal(t, v1.TaskStateReview, updatedTask.State) + assert.Equal(t, "step-owned-by-stop", updatedTask.WorkflowStepID) + queueStatus := orch.queue.GetStatus(ctx, session.ID) + require.Equal(t, 2, queueStatus.Count) + assert.Equal(t, "queued before dispatch", queueStatus.Entries[0].Content) + assert.Equal(t, "queued after stop", queueStatus.Entries[1].Content) +} + func TestHandleMessageTask_OfficeReviewDoesNotTransitionTaskState(t *testing.T) { ctx := context.Background() svc, repo := newTestTaskService(t) diff --git a/apps/backend/internal/mcp/handlers/stop_task.go b/apps/backend/internal/mcp/handlers/stop_task.go new file mode 100644 index 0000000000..59f48dd292 --- /dev/null +++ b/apps/backend/internal/mcp/handlers/stop_task.go @@ -0,0 +1,116 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "github.com/kandev/kandev/internal/orchestrator" + "github.com/kandev/kandev/internal/task/models" + taskrepo "github.com/kandev/kandev/internal/task/repository" + ws "github.com/kandev/kandev/pkg/websocket" +) + +const stopTaskStatusKey = "status" + +type stopTaskRequest struct { + TaskID string `json:"task_id"` + SenderTaskID string `json:"sender_task_id"` +} + +type stopTaskFailure struct { + response *ws.Message + err error +} + +func (h *Handlers) handleStopTask(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + req, validationError := parseStopTaskRequest(msg) + if validationError != nil { + return validationError.response, validationError.err + } + if h.stopTaskGetter == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "task lookup is not configured", nil) + } + + sender, lookupError := h.lookupStopTask(ctx, msg, req.SenderTaskID, "sender") + if lookupError != nil { + return lookupError.response, lookupError.err + } + + target, lookupError := h.lookupStopTask(ctx, msg, req.TaskID, "target") + if lookupError != nil { + return lookupError.response, lookupError.err + } + + if !canStopTask(sender, target) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, + "only a task's direct parent in the same workspace can stop it", nil) + } + if h.taskStopper == nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "task stop is not configured", nil) + } + + result, err := h.taskStopper.StopTaskForCoordinator(ctx, target.ID) + if err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "failed to stop target task", nil) + } + switch result.Status { + case orchestrator.CoordinatorTaskStopStatusStopped, orchestrator.CoordinatorTaskStopStatusNotRunning: + default: + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "task stop returned an invalid status", nil) + } + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + keyTaskID: target.ID, + stopTaskStatusKey: result.Status, + }) +} + +func parseStopTaskRequest(msg *ws.Message) (stopTaskRequest, *stopTaskFailure) { + var req stopTaskRequest + if err := json.Unmarshal(msg.Payload, &req); err != nil { + return req, newStopTaskFailure(msg, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error()) + } + req.TaskID = strings.TrimSpace(req.TaskID) + req.SenderTaskID = strings.TrimSpace(req.SenderTaskID) + if req.TaskID == "" { + return req, newStopTaskFailure(msg, ws.ErrorCodeValidation, "task_id is required") + } + if req.SenderTaskID == "" { + return req, newStopTaskFailure(msg, ws.ErrorCodeValidation, + "sender_task_id is required (the calling agent's MCP server must supply this)") + } + return req, nil +} + +func (h *Handlers) lookupStopTask( + ctx context.Context, + msg *ws.Message, + taskID string, + role string, +) (*models.Task, *stopTaskFailure) { + task, err := h.stopTaskGetter(ctx, taskID) + if err != nil { + if errors.Is(err, taskrepo.ErrTaskNotFound) { + return nil, newStopTaskFailure(msg, ws.ErrorCodeNotFound, role+" task not found") + } + return nil, newStopTaskFailure(msg, ws.ErrorCodeInternalError, "failed to look up "+role+" task") + } + if task == nil { + return nil, newStopTaskFailure(msg, ws.ErrorCodeNotFound, role+" task not found") + } + return task, nil +} + +func newStopTaskFailure(msg *ws.Message, code, message string) *stopTaskFailure { + response, err := ws.NewError(msg.ID, msg.Action, code, message, nil) + return &stopTaskFailure{response: response, err: err} +} + +func canStopTask(sender, target *models.Task) bool { + return sender.WorkspaceID != "" && + target.WorkspaceID != "" && + target.ParentID != "" && + target.ParentID == sender.ID && + target.WorkspaceID == sender.WorkspaceID +} diff --git a/apps/backend/internal/mcp/handlers/stop_task_test.go b/apps/backend/internal/mcp/handlers/stop_task_test.go new file mode 100644 index 0000000000..88f1638f11 --- /dev/null +++ b/apps/backend/internal/mcp/handlers/stop_task_test.go @@ -0,0 +1,271 @@ +package handlers + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/orchestrator" + "github.com/kandev/kandev/internal/task/models" + taskrepo "github.com/kandev/kandev/internal/task/repository" + ws "github.com/kandev/kandev/pkg/websocket" +) + +type recordingTaskStopper struct { + result orchestrator.CoordinatorTaskStopResult + err error + calls []string +} + +func (s *recordingTaskStopper) StopTaskForCoordinator( + _ context.Context, + taskID string, +) (orchestrator.CoordinatorTaskStopResult, error) { + s.calls = append(s.calls, taskID) + return s.result, s.err +} + +func stopTaskTestHandler( + t *testing.T, + tasks map[string]*models.Task, + errorsByID map[string]error, + stopper TaskStopper, +) *Handlers { + return &Handlers{ + taskStopper: stopper, + stopTaskGetter: func(_ context.Context, taskID string) (*models.Task, error) { + if err := errorsByID[taskID]; err != nil { + return nil, err + } + task, ok := tasks[taskID] + if !ok { + return nil, taskrepo.ErrTaskNotFound + } + return task, nil + }, + logger: testLogger(t), + } +} + +func TestHandleStopTask_AuthorizesOnlyDirectParentInWorkspace(t *testing.T) { + tasks := map[string]*models.Task{ + "grandparent": {ID: "grandparent", WorkspaceID: "ws-1"}, + "parent": {ID: "parent", WorkspaceID: "ws-1", ParentID: "grandparent"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + "sibling": {ID: "sibling", WorkspaceID: "ws-1", ParentID: "grandparent"}, + "unrelated": {ID: "unrelated", WorkspaceID: "ws-1"}, + "cross-child": {ID: "cross-child", WorkspaceID: "ws-2", ParentID: "parent"}, + } + tests := []struct { + name string + senderID string + targetID string + wantStatus orchestrator.CoordinatorTaskStopStatus + wantCode string + }{ + {name: "direct parent", senderID: "parent", targetID: "child", wantStatus: orchestrator.CoordinatorTaskStopStatusStopped}, + {name: "self", senderID: "parent", targetID: "parent", wantCode: ws.ErrorCodeForbidden}, + {name: "sibling", senderID: "parent", targetID: "sibling", wantCode: ws.ErrorCodeForbidden}, + {name: "child", senderID: "child", targetID: "parent", wantCode: ws.ErrorCodeForbidden}, + {name: "grandparent", senderID: "grandparent", targetID: "child", wantCode: ws.ErrorCodeForbidden}, + {name: "unrelated", senderID: "unrelated", targetID: "child", wantCode: ws.ErrorCodeForbidden}, + {name: "cross workspace", senderID: "parent", targetID: "cross-child", wantCode: ws.ErrorCodeForbidden}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stopper := &recordingTaskStopper{result: orchestrator.CoordinatorTaskStopResult{Status: tt.wantStatus}} + h := stopTaskTestHandler(t, tasks, nil, stopper) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": tt.targetID, + "sender_task_id": tt.senderID, + "reason": "forged destructive cleanup reason", + "force": true, + }) + + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + if tt.wantCode != "" { + assertWSError(t, resp, tt.wantCode) + if len(stopper.calls) != 0 { + t.Fatalf("forbidden request invoked stopper: %v", stopper.calls) + } + return + } + var payload struct { + TaskID string `json:"task_id"` + Status orchestrator.CoordinatorTaskStopStatus `json:"status"` + } + if err := resp.ParsePayload(&payload); err != nil { + t.Fatalf("parse response: %v", err) + } + if payload.TaskID != tt.targetID || payload.Status != tt.wantStatus { + t.Fatalf("response = %#v", payload) + } + if len(stopper.calls) != 1 || stopper.calls[0] != tt.targetID { + t.Fatalf("stopper calls = %v", stopper.calls) + } + }) + } +} + +func TestHandleStopTask_ValidatesPayload(t *testing.T) { + tests := []struct { + name string + payload []byte + code string + }{ + {name: "malformed", payload: []byte(`{"task_id":`), code: ws.ErrorCodeBadRequest}, + {name: "missing task", payload: []byte(`{"sender_task_id":"parent"}`), code: ws.ErrorCodeValidation}, + {name: "missing sender", payload: []byte(`{"task_id":"child"}`), code: ws.ErrorCodeValidation}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := stopTaskTestHandler(t, nil, nil, &recordingTaskStopper{}) + msg := &ws.Message{ID: "request-1", Action: ws.ActionMCPStopTask, Payload: tt.payload} + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + assertWSError(t, resp, tt.code) + }) + } +} + +func TestHandleStopTask_MapsLookupFailures(t *testing.T) { + infraFailure := errors.New("database unavailable") + tests := []struct { + name string + errorsByID map[string]error + wantCode string + }{ + {name: "sender missing", errorsByID: map[string]error{"parent": taskrepo.ErrTaskNotFound}, wantCode: ws.ErrorCodeNotFound}, + {name: "sender lookup failure", errorsByID: map[string]error{"parent": infraFailure}, wantCode: ws.ErrorCodeInternalError}, + {name: "target missing", errorsByID: map[string]error{"child": taskrepo.ErrTaskNotFound}, wantCode: ws.ErrorCodeNotFound}, + {name: "target lookup failure", errorsByID: map[string]error{"child": infraFailure}, wantCode: ws.ErrorCodeInternalError}, + } + tasks := map[string]*models.Task{ + "parent": {ID: "parent", WorkspaceID: "ws-1"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stopper := &recordingTaskStopper{} + h := stopTaskTestHandler(t, tasks, tt.errorsByID, stopper) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": "child", "sender_task_id": "parent", + }) + + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + assertWSError(t, resp, tt.wantCode) + if len(stopper.calls) != 0 { + t.Fatalf("lookup failure invoked stopper: %v", stopper.calls) + } + }) + } +} + +func TestHandleStopTask_MapsStopperFailure(t *testing.T) { + tasks := map[string]*models.Task{ + "parent": {ID: "parent", WorkspaceID: "ws-1"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + } + stopper := &recordingTaskStopper{err: errors.New("stop failed")} + h := stopTaskTestHandler(t, tasks, nil, stopper) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": "child", "sender_task_id": "parent", + }) + + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + assertWSError(t, resp, ws.ErrorCodeInternalError) +} + +func TestHandleStopTask_ReturnsNotRunning(t *testing.T) { + tasks := map[string]*models.Task{ + "parent": {ID: "parent", WorkspaceID: "ws-1"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + } + stopper := &recordingTaskStopper{result: orchestrator.CoordinatorTaskStopResult{ + Status: orchestrator.CoordinatorTaskStopStatusNotRunning, + }} + h := stopTaskTestHandler(t, tasks, nil, stopper) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": "child", "sender_task_id": "parent", + }) + + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + var payload struct { + Status orchestrator.CoordinatorTaskStopStatus `json:"status"` + } + if err := resp.ParsePayload(&payload); err != nil { + t.Fatalf("parse response: %v", err) + } + if payload.Status != orchestrator.CoordinatorTaskStopStatusNotRunning { + t.Fatalf("status = %q, want not_running", payload.Status) + } +} + +func TestHandleStopTask_RejectsMissingStopperAndInvalidStatus(t *testing.T) { + tasks := map[string]*models.Task{ + "parent": {ID: "parent", WorkspaceID: "ws-1"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + } + tests := []struct { + name string + stopper TaskStopper + }{ + {name: "missing stopper"}, + {name: "invalid status", stopper: &recordingTaskStopper{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := stopTaskTestHandler(t, tasks, nil, tt.stopper) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": "child", "sender_task_id": "parent", + }) + resp, err := h.handleStopTask(context.Background(), msg) + if err != nil { + t.Fatalf("handleStopTask: %v", err) + } + assertWSError(t, resp, ws.ErrorCodeInternalError) + }) + } +} + +func TestRegisterHandlers_RegistersStopTask(t *testing.T) { + tasks := map[string]*models.Task{ + "parent": {ID: "parent", WorkspaceID: "ws-1"}, + "child": {ID: "child", WorkspaceID: "ws-1", ParentID: "parent"}, + } + stopper := &recordingTaskStopper{result: orchestrator.CoordinatorTaskStopResult{ + Status: orchestrator.CoordinatorTaskStopStatusStopped, + }} + h := stopTaskTestHandler(t, tasks, nil, stopper) + dispatcher := ws.NewDispatcher() + h.RegisterHandlers(dispatcher) + msg := makeWSMessage(t, ws.ActionMCPStopTask, map[string]interface{}{ + "task_id": "child", "sender_task_id": "parent", + }) + + resp, err := dispatcher.Dispatch(context.Background(), msg) + if err != nil { + t.Fatalf("dispatch stop task: %v", err) + } + if resp == nil || resp.Type != ws.MessageTypeResponse { + t.Fatalf("dispatch response = %#v", resp) + } + if len(stopper.calls) != 1 || stopper.calls[0] != "child" { + t.Fatalf("stopper calls = %v", stopper.calls) + } +} diff --git a/apps/backend/internal/mcp/server/handlers.go b/apps/backend/internal/mcp/server/handlers.go index 09e1bd5018..9b514104a2 100644 --- a/apps/backend/internal/mcp/server/handlers.go +++ b/apps/backend/internal/mcp/server/handlers.go @@ -254,6 +254,27 @@ func (s *Server) messageTaskHandler() server.ToolHandlerFunc { } } +func (s *Server) stopTaskHandler() server.ToolHandlerFunc { + return func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + taskID, err := req.RequireString(mcpKeyTaskID) + if err != nil { + return mcp.NewToolResultError("task_id is required"), nil + } + // Build a fresh payload so callers cannot override trusted sender + // attribution or supply runtime-level session, reason, or force controls. + payload := map[string]interface{}{ + mcpKeyTaskID: taskID, + "sender_task_id": s.taskID, + } + var result map[string]interface{} + if err := s.backend.RequestPayload(ctx, ws.ActionMCPStopTask, payload, &result); err != nil { + return mcp.NewToolResultError(err.Error()), nil + } + data, _ := json.MarshalIndent(result, "", " ") + return mcp.NewToolResultText(string(data)), nil + } +} + // spawnSessionHandler spawns an additional agent session on an existing task. // task_id defaults to the server's own task; sender identity is injected so // the spawned session can identify and reply to its spawner. diff --git a/apps/backend/internal/mcp/server/handlers_test.go b/apps/backend/internal/mcp/server/handlers_test.go index 5f9419550a..1ec38de614 100644 --- a/apps/backend/internal/mcp/server/handlers_test.go +++ b/apps/backend/internal/mcp/server/handlers_test.go @@ -2,6 +2,7 @@ package mcp import ( "encoding/json" + "errors" "testing" ws "github.com/kandev/kandev/pkg/websocket" @@ -468,8 +469,12 @@ func TestMessageTask_ForwardsToBackend(t *testing.T) { s := newTaskModeServer(t, backend, "task-current") result := callTool(t, s, "message_task_kandev", map[string]interface{}{ - "task_id": "task-target", - "prompt": "follow up", + "task_id": "task-target", + "session_id": "sess-target", + "prompt": "follow up", + "delivery_mode": "interrupt", + "sender_task_id": "spoofed-task", + "sender_session_id": "spoofed-session", }) assert.False(t, result.IsError) @@ -478,7 +483,26 @@ func TestMessageTask_ForwardsToBackend(t *testing.T) { payload, ok := backend.lastPayload.(map[string]interface{}) require.True(t, ok) assert.Equal(t, "task-target", payload["task_id"]) + assert.Equal(t, "sess-target", payload["session_id"]) assert.Equal(t, "follow up", payload["prompt"]) + assert.Equal(t, "interrupt", payload["delivery_mode"]) + assert.Equal(t, "task-current", payload["sender_task_id"]) + assert.Equal(t, "test-session", payload["sender_session_id"]) +} + +func TestMessageTask_DescriptionExplainsQueueInterruptAndStop(t *testing.T) { + backend := &testBackend{} + s := newTaskModeServer(t, backend, "task-current") + + tools := s.mcpServer.ListTools() + messageTool, ok := tools["message_task_kandev"] + require.True(t, ok) + description := messageTool.Tool.Description + + assert.Contains(t, description, `delivery_mode="queued"`) + assert.Contains(t, description, `delivery_mode="interrupt"`) + assert.Contains(t, description, "stop_task_kandev") + assert.Contains(t, description, `safely falls back to "queued"`) } func TestMessageTask_MissingTaskID_ReturnsError(t *testing.T) { @@ -503,6 +527,105 @@ func TestMessageTask_MissingPrompt_ReturnsError(t *testing.T) { assert.True(t, result.IsError) } +func TestStopTask_ToolSchemaIsMinimalAndDescriptionIsAccurate(t *testing.T) { + backend := &testBackend{} + s := newTaskModeServer(t, backend, "task-current") + + tools := s.mcpServer.ListTools() + stopTool, ok := tools["stop_task_kandev"] + require.True(t, ok, "stop_task_kandev must be registered in task mode") + + schema, err := json.Marshal(stopTool.Tool.InputSchema) + require.NoError(t, err) + var parsed map[string]interface{} + require.NoError(t, json.Unmarshal(schema, &parsed)) + + properties, ok := parsed["properties"].(map[string]interface{}) + require.True(t, ok, "stop schema must declare properties") + require.Len(t, properties, 1, "stop schema must not expose sender, session, reason, or force controls") + assert.Contains(t, properties, "task_id") + for _, forbidden := range []string{"sender_task_id", "sender_session_id", "session_id", "reason", "force"} { + assert.NotContains(t, properties, forbidden) + } + + required, ok := parsed["required"].([]interface{}) + require.True(t, ok, "stop schema must declare task_id as required") + assert.Equal(t, []interface{}{"task_id"}, required) + + description := stopTool.Tool.Description + for _, phrase := range []string{ + "direct child", + "all live sessions", + "halt-only", + "does not send a prompt or start a replacement turn", + "CANCELLED", + "REVIEW", + "asynchronously", + "not_running", + "message_task_kandev", + `delivery_mode="interrupt"`, + } { + assert.Contains(t, description, phrase) + } +} + +func TestStopTask_ForwardsTrustedSenderToBackend(t *testing.T) { + backend := &testBackend{ + response: map[string]interface{}{ + "task_id": "task-target", + "status": "stopped", + }, + } + s := newTaskModeServer(t, backend, "task-current") + + result := callTool(t, s, "stop_task_kandev", map[string]interface{}{ + "task_id": "task-target", + "sender_task_id": "spoofed-task", + "sender_session_id": "spoofed-session", + "session_id": "spoofed-target-session", + "reason": "unsafe caller-controlled reason", + "force": true, + }) + + assert.False(t, result.IsError) + assert.Equal(t, "mcp.stop_task", backend.lastAction) + payload, ok := backend.lastPayload.(map[string]interface{}) + require.True(t, ok) + require.Len(t, payload, 2, "forwarder must build a fresh trusted payload") + assert.Equal(t, "task-target", payload["task_id"]) + assert.Equal(t, "task-current", payload["sender_task_id"]) + assert.NotContains(t, payload, "sender_session_id") + assert.NotContains(t, payload, "session_id") + assert.NotContains(t, payload, "reason") + assert.NotContains(t, payload, "force") +} + +func TestStopTask_MissingTaskIDReturnsErrorWithoutForwarding(t *testing.T) { + backend := &testBackend{} + s := newTaskModeServer(t, backend, "task-current") + + result := callTool(t, s, "stop_task_kandev", map[string]interface{}{}) + + assert.True(t, result.IsError) + assert.Empty(t, backend.lastAction) + assert.Nil(t, backend.lastPayload) +} + +func TestStopTask_BackendErrorReturnsToolError(t *testing.T) { + backend := &testBackend{err: errors.New("stop refused")} + s := newTaskModeServer(t, backend, "task-current") + + result := callTool(t, s, "stop_task_kandev", map[string]interface{}{ + "task_id": "task-target", + }) + + assert.True(t, result.IsError) + require.NotEmpty(t, result.Content) + text, ok := result.Content[0].(mcp.TextContent) + require.True(t, ok) + assert.Contains(t, text.Text, "stop refused") +} + func TestGetTaskConversation_ForwardsToBackend(t *testing.T) { backend := &testBackend{ response: map[string]interface{}{ diff --git a/apps/backend/internal/mcp/server/server.go b/apps/backend/internal/mcp/server/server.go index 4800b12d38..1e472ca135 100644 --- a/apps/backend/internal/mcp/server/server.go +++ b/apps/backend/internal/mcp/server/server.go @@ -362,7 +362,7 @@ func (s *Server) registerTools() { // a sibling to message_task_kandev) but NOT the task-document // tools — those are office coordination plumbing. s.registerKanbanTools() - count += 14 + count += 15 if !s.disableAskQuestion { s.registerInteractionTools() count++ @@ -480,8 +480,13 @@ func (s *Server) registerKanbanTools() { Use this to communicate with a sibling task, a parent task, or any task you know the ID of — for example to ask a delegated subtask for clarification, hand it new context, or nudge a paused task forward. Pass session_id to target a specific session — including a sibling session on your OWN task (e.g. one you spawned with spawn_session_kandev). +Choose the control by intent: +- Information that can wait: use delivery_mode="queued" (the default). The current turn continues and the message waits FIFO. +- Urgent replacement work for a running/starting direct child: use delivery_mode="interrupt". This requests immediate cancel-and-redispatch. Only the target task's direct parent may request it; non-parent requests fail. If immediate cancellation and dispatch cannot be confirmed safely, the message safely falls back to "queued". +- Halt-only work with no replacement prompt: use stop_task_kandev instead. + Behaviour by session state: -- Running/starting: the message is queued and delivered when the current turn ends. Pass delivery_mode="interrupt" to instead interrupt the target's current turn immediately so the message is delivered without waiting — keeps a parent's steering/stop messages from piling up behind a long-running child's turn. Only honored when you are the target task's direct parent; a non-parent sender requesting "interrupt" gets a hard error instead of being silently downgraded to "queued". +- Running/starting: queued delivery waits for turn-end; interrupt delivery follows the direct-parent behavior and safe fallback above. - Idle (waiting for input or completed): the message is sent immediately as a new turn (delivery_mode has no effect). - Created (not yet started): the agent is started with this message as its first prompt (delivery_mode has no effect). - Failed/cancelled: an error is returned (use create_task_kandev to start fresh). @@ -498,6 +503,19 @@ Returns the dispatch status: "queued", "sent", or "started".`), ), s.wrapHandler("message_task_kandev", s.messageTaskHandler()), ) + s.mcpServer.AddTool( + mcp.NewTool("stop_task_kandev", + mcp.WithDescription(`Stop all live sessions Kandev observes for a direct child task. Only the target task's direct parent may use this tool; self, sibling, child-to-parent, grandparent, unrelated, and cross-workspace requests are rejected. The operation has no session-specific option. + +This is halt-only: it does not send a prompt or start a replacement turn. For urgent stop-and-steer work, use message_task_kandev with delivery_mode="interrupt". For ordinary information that can wait, use message_task_kandev with delivery_mode="queued" or omit delivery_mode. + +For every accepted live execution, Kandev first marks its session CANCELLED, then schedules graceful runtime teardown. An eligible active, unarchived, non-Office task is also moved to REVIEW through the normal guarded transition; other task states are preserved. Runtime teardown continues asynchronously, so status="stopped" confirms logical cancellation and scheduled teardown, not process exit. + +If the child has no live execution, the call succeeds idempotently with status="not_running" and changes no task or session state. Worktrees, environments, commits, task records, descendants, and queued messages are preserved.`), + mcp.WithString(mcpKeyTaskID, mcp.Required(), mcp.Description("The direct child task's full UUID (not a truncated prefix)")), + ), + s.wrapHandler("stop_task_kandev", s.stopTaskHandler()), + ) s.registerSpawnSessionTool() s.mcpServer.AddTool( mcp.NewTool("get_task_conversation_kandev", diff --git a/apps/backend/internal/mcp/server/server_test.go b/apps/backend/internal/mcp/server/server_test.go index 9620346e06..4ee4832ba1 100644 --- a/apps/backend/internal/mcp/server/server_test.go +++ b/apps/backend/internal/mcp/server/server_test.go @@ -44,6 +44,7 @@ func TestServerModeTask_RegistersCorrectTools(t *testing.T) { assert.Contains(t, tools, "update_task_kandev") assert.Contains(t, tools, "move_task_kandev") assert.Contains(t, tools, "message_task_kandev") + assert.Contains(t, tools, "stop_task_kandev") assert.Contains(t, tools, "get_task_conversation_kandev") // Task mode should have plan tools @@ -158,6 +159,7 @@ func TestServerModeConfig_RegistersCorrectTools(t *testing.T) { // Config mode should NOT have task-mode kanban create/update tools assert.NotContains(t, tools, "create_task_kandev") assert.NotContains(t, tools, "update_task_kandev") + assert.NotContains(t, tools, "stop_task_kandev") } func TestServerModeDefault_DefaultsToTask(t *testing.T) { @@ -172,6 +174,7 @@ func TestServerModeDefault_DefaultsToTask(t *testing.T) { tools := getRegisteredToolNames(s) assert.Contains(t, tools, "create_task_kandev") assert.Contains(t, tools, "create_task_plan_kandev") + assert.Contains(t, tools, "stop_task_kandev") assert.NotContains(t, tools, "create_workflow_step_kandev") } @@ -210,15 +213,15 @@ func TestServerModeTask_ToolCount(t *testing.T) { s := New(backend, "test-session", "test-task", 10005, log, "", false, ModeTask) tools := getRegisteredToolNames(s) - // 14 kanban (incl. delete + archive task + spawn_session) + + // 15 kanban (incl. delete + archive task + stop_task + spawn_session) + // 1 add_branch_to_task + 1 update_repository_base_branch + // 1 step_complete (ADR 0015) + 1 interaction + 4 plan + 3 walkthrough + - // 1 related-tasks = 26. + // 1 related-tasks = 27. // Task-document tools (list/get/write) are office-only. assert.Contains(t, tools, "step_complete_kandev", "ADR 0015 explicit-completion signal must be registered in task mode") assert.Contains(t, tools, "show_walkthrough_kandev", "walkthrough tool must be registered in task mode") assert.Contains(t, tools, "spawn_session_kandev", "spawn_session must be registered in task mode") - assert.Equal(t, 26, len(tools)) + assert.Equal(t, 27, len(tools)) } func TestServerModeConfig_ToolCount(t *testing.T) { @@ -286,6 +289,7 @@ func TestServerModeOffice_RegistersCorrectTools(t *testing.T) { assert.NotContains(t, tools, "list_workflow_steps_kandev") assert.NotContains(t, tools, "list_agents_kandev") assert.NotContains(t, tools, "list_executor_profiles_kandev") + assert.NotContains(t, tools, "stop_task_kandev") // Office mode should NOT have config tools assert.NotContains(t, tools, "create_workflow_kandev") @@ -365,6 +369,7 @@ func TestServerModeExternal_RegistersCorrectTools(t *testing.T) { // External mode does NOT include message_task_kandev (no live session context) assert.NotContains(t, tools, "message_task_kandev") + assert.NotContains(t, tools, "stop_task_kandev") } func TestServerModeExternal_ToolCount(t *testing.T) { diff --git a/apps/backend/internal/mcp/server/sysprompt_sync_test.go b/apps/backend/internal/mcp/server/sysprompt_sync_test.go index 2a55a67ac2..0ab6be4c92 100644 --- a/apps/backend/internal/mcp/server/sysprompt_sync_test.go +++ b/apps/backend/internal/mcp/server/sysprompt_sync_test.go @@ -105,6 +105,65 @@ func TestSyspromptToolNames_MatchMCPTaskMode(t *testing.T) { } } +// TestTaskControlDocs_MatchMessageSchemaAndStopChoice keeps the short injected +// context aligned with the task-control choices exposed by task-mode MCP. The +// full tool descriptions remain authoritative for lifecycle detail, but the +// first-turn context must make the queue/interrupt/stop decision discoverable. +func TestTaskControlDocs_MatchMessageSchemaAndStopChoice(t *testing.T) { + log := newTestLogger(t) + backend := NewChannelBackendClient(log) + defer backend.Close() + + s := New(backend, "test-session", "test-task", 10005, log, "", false, ModeTask) + require.NotNil(t, s) + tools := s.mcpServer.ListTools() + messageTool, ok := tools["message_task_kandev"] + require.True(t, ok) + assert.Contains(t, tools, "stop_task_kandev") + + schema, err := json.Marshal(messageTool.Tool.InputSchema) + require.NoError(t, err) + var parsed map[string]interface{} + require.NoError(t, json.Unmarshal(schema, &parsed)) + properties, ok := parsed["properties"].(map[string]interface{}) + require.True(t, ok) + deliveryMode, ok := properties["delivery_mode"].(map[string]interface{}) + require.True(t, ok, "message_task_kandev must expose delivery_mode") + enumValues, ok := deliveryMode["enum"].([]interface{}) + require.True(t, ok) + defaultValue, ok := deliveryMode["default"].(string) + require.True(t, ok) + + context := sysprompt.KandevContext() + assert.Contains(t, context, "delivery_mode") + for _, rawValue := range enumValues { + value, ok := rawValue.(string) + require.True(t, ok) + assert.Contains(t, context, fmt.Sprintf(`delivery_mode="%s"`, value)) + } + assert.Contains(t, context, fmt.Sprintf(`delivery_mode="%s" or omit it`, defaultValue)) + assert.Contains(t, context, "direct parent") + assert.Contains(t, context, "stop_task_kandev") + assert.Contains(t, context, "halt-only") + assert.Contains(t, context, "no replacement turn") +} + +func TestTaskControlDocs_OmittedFromRestrictedModes(t *testing.T) { + log := newTestLogger(t) + backend := NewChannelBackendClient(log) + defer backend.Close() + + context := sysprompt.FormatKandevContextWithOptions("task", "session", sysprompt.KandevContextOptions{}) + for _, mode := range []string{ModeOffice, ModeConfig} { + t.Run(mode, func(t *testing.T) { + s := New(backend, "session", "task", 10005, log, "", false, mode) + assert.NotContains(t, s.mcpServer.ListTools(), "stop_task_kandev") + assert.NotContains(t, context, "stop_task_kandev") + assert.NotContains(t, context, "delivery_mode") + }) + } +} + // TestSyspromptToolNames_MatchMCPConfigMode verifies that every `_kandev` // tool referenced in ConfigContext is registered by an MCP server in ModeConfig. func TestSyspromptToolNames_MatchMCPConfigMode(t *testing.T) { diff --git a/apps/backend/internal/orchestrator/coordinator_stop_edges_test.go b/apps/backend/internal/orchestrator/coordinator_stop_edges_test.go new file mode 100644 index 0000000000..0206ba5186 --- /dev/null +++ b/apps/backend/internal/orchestrator/coordinator_stop_edges_test.go @@ -0,0 +1,366 @@ +package orchestrator + +import ( + "context" + "errors" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + orchestratorexec "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +func TestStopTaskForCoordinator_ReviewGuardsPreserveTaskState(t *testing.T) { + tests := []struct { + name string + initialState v1.TaskState + office bool + archived bool + hasExecution bool + wantStatus CoordinatorTaskStopStatus + wantSession models.TaskSessionState + }{ + { + name: "Office task", initialState: v1.TaskStateInProgress, office: true, hasExecution: true, + wantStatus: CoordinatorTaskStopStatusStopped, wantSession: models.TaskSessionStateCancelled, + }, + { + name: "archived task", initialState: v1.TaskStateInProgress, archived: true, hasExecution: true, + wantStatus: CoordinatorTaskStopStatusStopped, wantSession: models.TaskSessionStateCancelled, + }, + { + name: "non-active task", initialState: v1.TaskStateCompleted, hasExecution: true, + wantStatus: CoordinatorTaskStopStatusStopped, wantSession: models.TaskSessionStateCancelled, + }, + { + name: "not running", initialState: v1.TaskStateInProgress, hasExecution: false, + wantStatus: CoordinatorTaskStopStatusNotRunning, wantSession: models.TaskSessionStateRunning, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-review-guard", "session-review-guard", models.TaskSessionStateRunning) + var serviceRepo repoStore = baseRepo + if tt.office { + serviceRepo = &coordinatorStopRepoHooks{ + repoStore: baseRepo, + getTaskFunc: func(readCtx context.Context, taskID string) (*models.Task, error) { + task, err := baseRepo.GetTask(readCtx, taskID) + if err != nil || task == nil { + return task, err + } + copyTask := *task + copyTask.AssigneeAgentProfileID = "office-agent" + return ©Task, nil + }, + } + } + if tt.archived { + require.NoError(t, baseRepo.ArchiveTask(ctx, "task-review-guard")) + } + + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + if !tt.hasExecution { + return "", lifecycle.ErrNoExecutionForSession + } + return "execution-review-guard", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-review-guard", tt.initialState) + svc := newCoordinatorStopTestService(serviceRepo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-review-guard") + + require.NoError(t, err) + require.Equal(t, tt.wantStatus, result.Status) + if tt.hasExecution { + coordinatorStopAwaitSignal(t, teardownCalled, "guarded task runtime teardown") + } + session, err := baseRepo.GetTaskSession(ctx, "session-review-guard") + require.NoError(t, err) + require.Equal(t, tt.wantSession, session.State) + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-review-guard") + require.Equal(t, tt.initialState, state) + require.Empty(t, history, "task REVIEW guard allowed an ineligible state write") + }) + } +} + +func TestStopTaskForCoordinator_PreRegistrationLaunchCanEscape(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-launch-race", "session-launch-race", models.TaskSessionStateRunning) + + launchStarted := make(chan struct{}) + allowRegistration := make(chan struct{}) + registrationDone := make(chan error, 1) + var allowRegistrationOnce sync.Once + releaseRegistration := func() { allowRegistrationOnce.Do(func() { close(allowRegistration) }) } + t.Cleanup(releaseRegistration) + go func() { + close(launchStarted) + <-allowRegistration + registrationDone <- repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "session-launch-race", SessionID: "session-launch-race", TaskID: "task-launch-race", + AgentExecutionID: "execution-late", Status: "ready", + }) + }() + coordinatorStopAwaitSignal(t, launchStarted, "pre-registration launch start") + + lookupObserved := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + lookupObserved <- struct{}{} + return "", lifecycle.ErrNoExecutionForSession + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-launch-race", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-launch-race") + + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusNotRunning, result.Status) + coordinatorStopAwaitSignal(t, lookupObserved, "absent lifecycle lookup") + releaseRegistration() + select { + case registrationErr := <-registrationDone: + require.NoError(t, registrationErr) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for late execution registration") + } + running, err := repo.GetExecutorRunningBySessionID(ctx, "session-launch-race") + require.NoError(t, err) + require.Equal(t, "execution-late", running.AgentExecutionID) + session, err := repo.GetTaskSession(ctx, "session-launch-race") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, session.State) + agentManager.mu.Lock() + stopCalls := append([]stopAgentCall(nil), agentManager.stopAgentWithReasonArgs...) + agentManager.mu.Unlock() + require.Empty(t, stopCalls, "v1 has no task-wide fence for a late registration") + _, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-launch-race") + require.Empty(t, history) +} + +func TestFallbackFreshLaunch_CoordinatorCancellationWinsBeforeResetWrite(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession( + t, + baseRepo, + "task-fallback-stop-race", + "session-fallback-stop-race", + models.TaskSessionStateWaitingForInput, + ) + + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + updateFullRowCAS: func( + writeCtx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, + ) (bool, error) { + changed, _, err := baseRepo.CancelActiveTaskSession( + writeCtx, + session.ID, + coordinatorMCPStopReason, + ) + require.NoError(t, err) + require.True(t, changed) + return baseRepo.UpdateTaskSessionIfCurrentState(writeCtx, session, expected) + }, + } + var launchCalled atomic.Bool + agentManager := &mockAgentManager{ + launchAgentFunc: func( + context.Context, + *orchestratorexec.LaunchAgentRequest, + ) (*orchestratorexec.LaunchAgentResponse, error) { + launchCalled.Store(true) + return nil, errors.New("unexpected launch") + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-fallback-stop-race", v1.TaskStateReview) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + err := svc.fallbackFreshLaunchOnMissingExecution( + ctx, + "task-fallback-stop-race", + "session-fallback-stop-race", + "replacement prompt", + false, + nil, + nil, + ) + + require.ErrorIs(t, err, orchestratorexec.ErrSessionStateSuperseded) + require.False(t, launchCalled.Load(), "cancelled session started replacement work") + session, getErr := baseRepo.GetTaskSession(ctx, "session-fallback-stop-race") + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, session.State) +} + +func TestFallbackFreshLaunch_DoesNotResetCancellationObservedBeforeGuard(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession( + t, + repo, + "task-fallback-already-stopped", + "session-fallback-already-stopped", + models.TaskSessionStateWaitingForInput, + ) + changed, _, err := repo.CancelActiveTaskSession( + ctx, + "session-fallback-already-stopped", + coordinatorMCPStopReason, + ) + require.NoError(t, err) + require.True(t, changed) + cancelled, err := repo.GetTaskSession(ctx, "session-fallback-already-stopped") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, cancelled.State) + + var launchCalled atomic.Bool + manager := &mockAgentManager{ + launchAgentFunc: func( + context.Context, + *orchestratorexec.LaunchAgentRequest, + ) (*orchestratorexec.LaunchAgentResponse, error) { + launchCalled.Store(true) + return nil, errors.New("unexpected launch") + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-fallback-already-stopped", v1.TaskStateReview) + svc := newCoordinatorStopTestService(repo, taskRepo, manager) + + err = svc.fallbackFreshLaunchOnMissingExecution( + ctx, + "task-fallback-already-stopped", + "session-fallback-already-stopped", + "replacement prompt", + false, + nil, + nil, + ) + + require.ErrorIs(t, err, orchestratorexec.ErrSessionStateSuperseded) + require.False(t, launchCalled.Load(), "cancelled session started replacement work") + session, getErr := repo.GetTaskSession(ctx, "session-fallback-already-stopped") + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, session.State) +} + +func coordinatorStopAddSession( + t *testing.T, + repo *sqliterepo.Repository, + taskID, sessionID string, + state models.TaskSessionState, +) { + t.Helper() + now := time.Now().UTC() + require.NoError(t, repo.CreateTaskSession(context.Background(), &models.TaskSession{ + ID: sessionID, TaskID: taskID, State: state, StartedAt: now, UpdatedAt: now, + })) +} + +func coordinatorStopAwaitSignal(t *testing.T, signal <-chan struct{}, description string) { + t.Helper() + select { + case <-signal: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for %s", description) + } +} + +func coordinatorStopAwaitCall( + t *testing.T, + done <-chan coordinatorStopCallOutcome, +) coordinatorStopCallOutcome { + t.Helper() + select { + case outcome := <-done: + return outcome + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for coordinator stop call") + return coordinatorStopCallOutcome{} + } +} + +func coordinatorStopWaitForGuardRefs(t *testing.T, svc *Service, sessionID string, want int) { + t.Helper() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for { + svc.cancelInFlightMu.Lock() + guard := svc.cancelInFlight[sessionID] + refs := 0 + if guard != nil { + refs = guard.refs + } + svc.cancelInFlightMu.Unlock() + if refs >= want { + return + } + select { + case <-deadline.C: + t.Fatalf("timed out waiting for %d cancel guard references; got %d", want, refs) + default: + runtime.Gosched() + } + } +} + +func coordinatorStopWaitForGuardReleased(t *testing.T, svc *Service, sessionID string) { + t.Helper() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + for { + svc.cancelInFlightMu.Lock() + _, exists := svc.cancelInFlight[sessionID] + svc.cancelInFlightMu.Unlock() + if !exists { + return + } + select { + case <-deadline.C: + t.Fatalf("timed out waiting for cancel guard %q release", sessionID) + default: + runtime.Gosched() + } + } +} + +func coordinatorStopTaskStateSnapshot( + repo *mockTaskRepo, + taskID string, +) (v1.TaskState, []v1.TaskState) { + repo.mu.Lock() + defer repo.mu.Unlock() + state := v1.TaskState("") + if task := repo.tasks[taskID]; task != nil { + state = task.State + } + return state, append([]v1.TaskState(nil), repo.stateHistory[taskID]...) +} diff --git a/apps/backend/internal/orchestrator/coordinator_stop_test.go b/apps/backend/internal/orchestrator/coordinator_stop_test.go new file mode 100644 index 0000000000..78abe9f57b --- /dev/null +++ b/apps/backend/internal/orchestrator/coordinator_stop_test.go @@ -0,0 +1,798 @@ +package orchestrator + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + orchestratorexec "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/orchestrator/messagequeue" + "github.com/kandev/kandev/internal/orchestrator/watcher" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +type coordinatorStopCallOutcome struct { + result CoordinatorTaskStopResult + err error +} + +type coordinatorStopRepoHooks struct { + repoStore + listActiveFunc func(context.Context, string) ([]*models.TaskSession, error) + getSessionFunc func(context.Context, string) (*models.TaskSession, error) + cancelActiveFunc func(context.Context, string, string) (bool, time.Time, error) + getTaskFunc func(context.Context, string) (*models.Task, error) + updateFullRowCAS func(context.Context, *models.TaskSession, models.TaskSessionState) (bool, error) +} + +func (r *coordinatorStopRepoHooks) UpdateTaskSessionIfCurrentState( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) (bool, error) { + if r.updateFullRowCAS != nil { + return r.updateFullRowCAS(ctx, session, expected) + } + return r.repoStore.UpdateTaskSessionIfCurrentState(ctx, session, expected) +} + +func (r *coordinatorStopRepoHooks) ListActiveTaskSessionsByTaskID( + ctx context.Context, + taskID string, +) ([]*models.TaskSession, error) { + if r.listActiveFunc != nil { + return r.listActiveFunc(ctx, taskID) + } + return r.repoStore.ListActiveTaskSessionsByTaskID(ctx, taskID) +} + +func (r *coordinatorStopRepoHooks) GetTaskSession( + ctx context.Context, + sessionID string, +) (*models.TaskSession, error) { + if r.getSessionFunc != nil { + return r.getSessionFunc(ctx, sessionID) + } + return r.repoStore.GetTaskSession(ctx, sessionID) +} + +func (r *coordinatorStopRepoHooks) CancelActiveTaskSession( + ctx context.Context, + sessionID, reason string, +) (bool, time.Time, error) { + if r.cancelActiveFunc != nil { + return r.cancelActiveFunc(ctx, sessionID, reason) + } + canceller, ok := r.repoStore.(activeTaskSessionCanceller) + if !ok { + return false, time.Time{}, errors.New("coordinator stop test repository cannot cancel an active session") + } + return canceller.CancelActiveTaskSession(ctx, sessionID, reason) +} + +func (r *coordinatorStopRepoHooks) GetTask(ctx context.Context, taskID string) (*models.Task, error) { + if r.getTaskFunc != nil { + return r.getTaskFunc(ctx, taskID) + } + return r.repoStore.GetTask(ctx, taskID) +} + +type coordinatorStopReadyReadKey struct{} + +func TestScheduleTaskForSession_CancelledSessionDoesNotOverwriteReview(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-cancelled", "session-cancelled", models.TaskSessionStateCancelled) + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-cancelled", v1.TaskStateReview) + svc := newCoordinatorStopTestService(repo, taskRepo, &mockAgentManager{}) + + err := svc.scheduleTaskForSession(ctx, "task-cancelled", "session-cancelled") + + require.ErrorIs(t, err, orchestratorexec.ErrSessionStateSuperseded) + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-cancelled") + require.Equal(t, v1.TaskStateReview, state) + require.Empty(t, history, "cancelled session must not write task SCHEDULING") +} + +func TestStopTaskForCoordinator_NotRunningDisarmsTransientRetry(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-no-execution", "session-no-execution", models.TaskSessionStateRunning) + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-no-execution", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "", lifecycle.ErrNoExecutionForSession + }, + }) + cancelled := make(chan struct{}, 1) + svc.transientRetries.Store("session-no-execution", &transientRetryEntry{ + attempt: 1, + cancel: func() { + cancelled <- struct{}{} + }, + }) + svc.rememberTurnPrompt("session-no-execution", "retry me", "", false, nil) + + result, err := svc.StopTaskForCoordinator(ctx, "task-no-execution") + + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusNotRunning, result.Status) + coordinatorStopAwaitSignal(t, cancelled, "transient retry cancellation") + _, retryArmed := svc.transientRetries.Load("session-no-execution") + require.False(t, retryArmed) + _, promptCached := svc.lastTurnPrompt.Load("session-no-execution") + require.False(t, promptCached) + session, err := repo.GetTaskSession(ctx, "session-no-execution") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, session.State) + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-no-execution") + require.Equal(t, v1.TaskStateInProgress, state) + require.Empty(t, history, "not_running stop must not change task state") +} + +func TestStopTaskForCoordinator_ReleasesSessionGuardBeforeTeardown(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-teardown-order", "session-teardown-order", models.TaskSessionStateRunning) + + guardHeldAtTeardown := make(chan bool, 1) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-teardown-order", nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-teardown-order", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, manager) + manager.stopAgentWithReasonFunc = func(context.Context, string, string, bool) error { + guardHeldAtTeardown <- svc.isCancelInFlight("session-teardown-order") + return nil + } + + result, err := svc.StopTaskForCoordinator(ctx, "task-teardown-order") + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + select { + case held := <-guardHeldAtTeardown: + require.False(t, held, "detached teardown entered before the session guard was released") + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for teardown ordering check") + } +} + +func TestStopTaskForCoordinator_FirstForceTeardownIntentWins(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-teardown-intent", "session-teardown-intent", models.TaskSessionStateRunning) + + stopEntered := make(chan stopAgentCall, 2) + allowForceStop := make(chan struct{}) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-teardown-intent", nil + }, + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + stopEntered <- stopAgentCall{ExecutionID: executionID, Reason: reason, Force: force} + if force { + <-allowForceStop + } + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-teardown-intent", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, manager) + + cleanupDone := make(chan struct{}) + go func() { + svc.cleanupAgentExecution( + "execution-teardown-intent", + "task-teardown-intent", + "session-teardown-intent", + ) + close(cleanupDone) + }() + first := <-stopEntered + require.True(t, first.Force) + + result, err := svc.StopTaskForCoordinator(ctx, "task-teardown-intent") + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + close(allowForceStop) + coordinatorStopAwaitSignal(t, cleanupDone, "force teardown completion") + select { + case duplicate := <-stopEntered: + t.Fatalf("duplicate teardown intent reached runtime: %#v", duplicate) + case <-time.After(100 * time.Millisecond): + } + session, err := repo.GetTaskSession(ctx, "session-teardown-intent") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) +} + +func TestHandleAgentFailed_UnavailableSessionStateStillCleansExecution(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-failure-unavailable", "session-failure-unavailable", models.TaskSessionStateRunning) + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + getSessionFunc: func(context.Context, string) (*models.TaskSession, error) { + return nil, errors.New("session store unavailable") + }, + } + stopCalled := make(chan stopAgentCall, 1) + manager := &mockAgentManager{ + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + stopCalled <- stopAgentCall{ExecutionID: executionID, Reason: reason, Force: force} + return nil + }, + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), manager) + + svc.handleAgentFailed(ctx, watcher.AgentEventData{ + TaskID: "task-failure-unavailable", + SessionID: "session-failure-unavailable", + AgentExecutionID: "execution-failure-unavailable", + ErrorMessage: "agent failed", + }) + + select { + case call := <-stopCalled: + require.Equal(t, "execution-failure-unavailable", call.ExecutionID) + require.True(t, call.Force) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for execution-safe cleanup") + } +} + +func TestStopTaskForCoordinator_SerializesStaleAgentFailure(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-failure-race", "session-failure-race", models.TaskSessionStateRunning) + + lookupEntered := make(chan struct{}) + allowLookup := make(chan struct{}) + var lookupOnce sync.Once + var allowLookupOnce sync.Once + releaseLookup := func() { allowLookupOnce.Do(func() { close(allowLookup) }) } + t.Cleanup(releaseLookup) + stopCalls := make(chan stopAgentCall, 2) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + lookupOnce.Do(func() { close(lookupEntered) }) + <-allowLookup + return "execution-failure-race", nil + }, + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + stopCalls <- stopAgentCall{ExecutionID: executionID, Reason: reason, Force: force} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-failure-race", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + stopDone := make(chan coordinatorStopCallOutcome, 1) + go func() { + result, err := svc.StopTaskForCoordinator(ctx, "task-failure-race") + stopDone <- coordinatorStopCallOutcome{result: result, err: err} + }() + coordinatorStopAwaitSignal(t, lookupEntered, "coordinator lifecycle lookup") + + failureDone := make(chan struct{}) + go func() { + defer close(failureDone) + svc.handleAgentFailed(ctx, watcher.AgentEventData{ + TaskID: "task-failure-race", + SessionID: "session-failure-race", + AgentExecutionID: "execution-failure-race", + ErrorMessage: "agent crashed", + }) + }() + coordinatorStopWaitForGuardRefs(t, svc, "session-failure-race", 2) + + releaseLookup() + outcome := coordinatorStopAwaitCall(t, stopDone) + require.NoError(t, outcome.err) + require.Equal(t, CoordinatorTaskStopStatusStopped, outcome.result.Status) + coordinatorStopAwaitSignal(t, failureDone, "stale agent.failed completion") + + select { + case call := <-stopCalls: + require.False(t, call.Force, "coordinator graceful teardown must remain non-force") + require.Equal(t, coordinatorMCPStopReason, call.Reason) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for coordinator graceful teardown") + } + select { + case call := <-stopCalls: + t.Fatalf("stale agent.failed launched extra cleanup: %#v", call) + case <-time.After(100 * time.Millisecond): + } + + session, err := repo.GetTaskSession(ctx, "session-failure-race") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + require.Empty(t, messages.sessionMessages, "stale failure must not emit recovery messages") + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-failure-race") + require.Equal(t, v1.TaskStateReview, state) + require.Equal(t, []v1.TaskState{v1.TaskStateReview}, history) +} + +func TestStopTaskForCoordinator_ProcessesCandidatesInStableIDOrder(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-order", "session-c", models.TaskSessionStateRunning) + coordinatorStopAddSession(t, baseRepo, "task-order", "session-a", models.TaskSessionStateRunning) + coordinatorStopAddSession(t, baseRepo, "task-order", "session-b", models.TaskSessionStateRunning) + + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + listActiveFunc: func(context.Context, string) ([]*models.TaskSession, error) { + return []*models.TaskSession{ + {ID: "session-c", TaskID: "task-order", State: models.TaskSessionStateRunning}, + {ID: "session-a", TaskID: "task-order", State: models.TaskSessionStateRunning}, + {ID: "session-b", TaskID: "task-order", State: models.TaskSessionStateRunning}, + }, nil + }, + } + var lookupMu sync.Mutex + var lookupOrder []string + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(_ context.Context, sessionID string) (string, error) { + lookupMu.Lock() + lookupOrder = append(lookupOrder, sessionID) + lookupMu.Unlock() + return "", lifecycle.ErrNoExecutionForSession + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-order", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-order") + + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusNotRunning, result.Status) + lookupMu.Lock() + gotOrder := append([]string(nil), lookupOrder...) + lookupMu.Unlock() + require.Equal(t, []string{"session-a", "session-b", "session-c"}, gotOrder) +} + +func TestStopTaskForCoordinator_AcceptedAndAbsentReturnsStopped(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-mixed", "session-a", models.TaskSessionStateRunning) + coordinatorStopAddSession(t, repo, "task-mixed", "session-b", models.TaskSessionStateRunning) + + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(_ context.Context, sessionID string) (string, error) { + if sessionID == "session-a" { + return "execution-a", nil + } + return "", lifecycle.ErrNoExecutionForSession + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-mixed", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-mixed") + + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + coordinatorStopAwaitSignal(t, teardownCalled, "accepted runtime teardown") + accepted, err := repo.GetTaskSession(ctx, "session-a") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, accepted.State) + absent, err := repo.GetTaskSession(ctx, "session-b") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, absent.State) +} + +func TestStopTaskForCoordinator_CommittedCancellationSurvivesPostWriteReadFailure(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-post-write-read", "session-post-write-read", models.TaskSessionStateRunning) + + postWriteReadFailure := errors.New("post-cancellation read failed") + var cancelCommitted atomic.Bool + var failedRead atomic.Bool + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + getSessionFunc: func(readCtx context.Context, sessionID string) (*models.TaskSession, error) { + if cancelCommitted.Load() && failedRead.CompareAndSwap(false, true) { + return nil, postWriteReadFailure + } + return baseRepo.GetTaskSession(readCtx, sessionID) + }, + cancelActiveFunc: func(writeCtx context.Context, sessionID, reason string) (bool, time.Time, error) { + changed, updatedAt, err := baseRepo.CancelActiveTaskSession(writeCtx, sessionID, reason) + if changed && err == nil { + cancelCommitted.Store(true) + } + return changed, updatedAt, err + }, + } + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-post-write-read", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-post-write-read", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-post-write-read") + + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + coordinatorStopAwaitSignal(t, teardownCalled, "teardown after committed cancellation") + session, getErr := baseRepo.GetTaskSession(ctx, "session-post-write-read") + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, session.State) +} + +func TestStopTaskForCoordinator_DelayedStaleStateWriterCannotResurrectCancelledSession(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-stale-writer", "session-stale-writer", models.TaskSessionStateRunning) + staleSession, err := repo.GetTaskSession(ctx, "session-stale-writer") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, staleSession.State) + + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-stale-writer", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-stale-writer", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-stale-writer") + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + coordinatorStopAwaitSignal(t, teardownCalled, "stale-writer runtime teardown") + + // Model a delayed event handler that loaded RUNNING before the stop, then + // attempts to commit an active state after cancellation was accepted. + svc.updateTaskSessionState( + ctx, + "task-stale-writer", + "session-stale-writer", + models.TaskSessionStateWaitingForInput, + "", + false, + staleSession, + ) + + finalSession, getErr := repo.GetTaskSession(ctx, "session-stale-writer") + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, finalSession.State) +} + +func TestCompleteAndStopSession_DoesNotRestoreStaleRunningStateAfterCancellation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-workflow-cleanup", "session-workflow-cleanup", models.TaskSessionStateRunning) + + staleSession, err := repo.GetTaskSession(ctx, "session-workflow-cleanup") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, staleSession.State) + changed, _, err := repo.CancelActiveTaskSession(ctx, staleSession.ID, coordinatorMCPStopReason) + require.NoError(t, err) + require.True(t, changed) + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-workflow-cleanup", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, &mockAgentManager{repoForExecutionLookup: repo}) + + // Model workflow cleanup resuming after coordinator stop committed. The + // cleanup owns a stale RUNNING row and must not write it back wholesale. + svc.completeAndStopSession(ctx, "task-workflow-cleanup", staleSession) + + stored, err := repo.GetTaskSession(ctx, staleSession.ID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, stored.State) + require.Equal(t, coordinatorMCPStopReason, stored.ErrorMessage) +} + +func TestSetSessionStarting_CoordinatorCancellationWinsAfterCurrentStateRead(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-start-race", "session-start-race", models.TaskSessionStateRunning) + stale, err := baseRepo.GetTaskSession(ctx, "session-start-race") + require.NoError(t, err) + stale.State = models.TaskSessionStateStarting + + repo := &coordinatorStopRepoHooks{repoStore: baseRepo} + repo.updateFullRowCAS = func( + writeCtx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, + ) (bool, error) { + changed, _, cancelErr := baseRepo.CancelActiveTaskSession( + writeCtx, session.ID, coordinatorMCPStopReason, + ) + require.NoError(t, cancelErr) + require.True(t, changed) + return baseRepo.UpdateTaskSessionIfCurrentState(writeCtx, session, expected) + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), &mockAgentManager{}) + + err = svc.setSessionStarting(ctx, stale.TaskID, stale, true) + require.Error(t, err) + stored, getErr := baseRepo.GetTaskSession(ctx, stale.ID) + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, stored.State) + require.Equal(t, coordinatorMCPStopReason, stored.ErrorMessage) +} + +func TestStopTaskForCoordinator_PartialFailureAttemptsEveryCandidateAndSkipsReview(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-partial", "session-a", models.TaskSessionStateRunning) + coordinatorStopAddSession(t, repo, "task-partial", "session-b", models.TaskSessionStateRunning) + coordinatorStopAddSession(t, repo, "task-partial", "session-c", models.TaskSessionStateRunning) + + lookupFailure := errors.New("lifecycle lookup failed") + var lookupMu sync.Mutex + var lookupOrder []string + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(_ context.Context, sessionID string) (string, error) { + lookupMu.Lock() + lookupOrder = append(lookupOrder, sessionID) + lookupMu.Unlock() + switch sessionID { + case "session-a": + return "execution-a", nil + case "session-b": + return "", lookupFailure + default: + return "", lifecycle.ErrNoExecutionForSession + } + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-partial", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task-partial") + + require.ErrorIs(t, err, lookupFailure) + require.Empty(t, result.Status) + coordinatorStopAwaitSignal(t, teardownCalled, "successful candidate teardown") + lookupMu.Lock() + gotOrder := append([]string(nil), lookupOrder...) + lookupMu.Unlock() + require.Equal(t, []string{"session-a", "session-b", "session-c"}, gotOrder) + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-partial") + require.Equal(t, v1.TaskStateInProgress, state) + require.Empty(t, history, "partial failure must not reconcile the task to REVIEW") + accepted, getErr := repo.GetTaskSession(ctx, "session-a") + require.NoError(t, getErr) + require.Equal(t, models.TaskSessionStateCancelled, accepted.State) +} + +func TestStopTaskForCoordinator_PartialFailureReconcilesAcceptedStops(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-partial-review", "session-accepted", models.TaskSessionStateRunning) + acceptedSession, err := baseRepo.GetTaskSession(ctx, "session-accepted") + require.NoError(t, err) + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + listActiveFunc: func(context.Context, string) ([]*models.TaskSession, error) { + return []*models.TaskSession{acceptedSession, nil}, nil + }, + } + teardownCalled := make(chan struct{}, 1) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-accepted", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-partial-review", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, manager) + + result, stopErr := svc.StopTaskForCoordinator(ctx, "task-partial-review") + + require.ErrorContains(t, stopErr, "candidate is nil or has an empty ID") + require.Empty(t, result.Status) + coordinatorStopAwaitSignal(t, teardownCalled, "accepted partial-stop teardown") + state, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-partial-review") + require.Equal(t, v1.TaskStateReview, state) + require.Equal(t, []v1.TaskState{v1.TaskStateReview}, history) +} + +func TestStopTaskForCoordinator_TerminalRereadWinsAfterCandidateSnapshot(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-terminal", "session-terminal", models.TaskSessionStateRunning) + + snapshotReturned := make(chan struct{}) + var snapshotOnce sync.Once + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + listActiveFunc: func(context.Context, string) ([]*models.TaskSession, error) { + snapshotOnce.Do(func() { close(snapshotReturned) }) + return []*models.TaskSession{{ + ID: "session-terminal", TaskID: "task-terminal", State: models.TaskSessionStateRunning, + }}, nil + }, + } + var lookupCalls atomic.Int32 + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + lookupCalls.Add(1) + return "execution-terminal", nil + }, + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-terminal", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + guard, releaseGuard := svc.acquireCancelInFlightGuard("session-terminal") + guard.Lock() + var unlockOnce sync.Once + unlockGuard := func() { + unlockOnce.Do(func() { + guard.Unlock() + releaseGuard() + }) + } + t.Cleanup(unlockGuard) + + stopDone := make(chan coordinatorStopCallOutcome, 1) + go func() { + result, err := svc.StopTaskForCoordinator(ctx, "task-terminal") + stopDone <- coordinatorStopCallOutcome{result: result, err: err} + }() + coordinatorStopAwaitSignal(t, snapshotReturned, "active-session snapshot") + require.NoError(t, baseRepo.UpdateTaskSessionState( + ctx, "session-terminal", models.TaskSessionStateCompleted, "completed naturally", + )) + unlockGuard() + + outcome := coordinatorStopAwaitCall(t, stopDone) + require.NoError(t, outcome.err) + require.Equal(t, CoordinatorTaskStopStatusNotRunning, outcome.result.Status) + require.Zero(t, lookupCalls.Load(), "terminal guarded reread must skip lifecycle lookup") + session, err := baseRepo.GetTaskSession(ctx, "session-terminal") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCompleted, session.State) + _, history := coordinatorStopTaskStateSnapshot(taskRepo, "task-terminal") + require.Empty(t, history) +} + +func TestStopTaskForCoordinator_SerializesReadyBeforeQueueDrain(t *testing.T) { + ctx := context.Background() + baseRepo := setupTestRepo(t) + seedTaskAndSession(t, baseRepo, "task-ready", "session-ready", models.TaskSessionStateRunning) + + cancelWriteEntered := make(chan struct{}) + allowCancelWrite := make(chan struct{}) + readyInitialRead := make(chan struct{}) + var cancelWriteOnce sync.Once + var readyReadOnce sync.Once + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + getSessionFunc: func(readCtx context.Context, sessionID string) (*models.TaskSession, error) { + session, err := baseRepo.GetTaskSession(readCtx, sessionID) + if readCtx.Value(coordinatorStopReadyReadKey{}) == true { + readyReadOnce.Do(func() { close(readyInitialRead) }) + } + return session, err + }, + cancelActiveFunc: func(writeCtx context.Context, sessionID, reason string) (bool, time.Time, error) { + cancelWriteOnce.Do(func() { close(cancelWriteEntered) }) + <-allowCancelWrite + return baseRepo.CancelActiveTaskSession(writeCtx, sessionID, reason) + }, + } + var releaseCancelOnce sync.Once + releaseCancelWrite := func() { releaseCancelOnce.Do(func() { close(allowCancelWrite) }) } + t.Cleanup(releaseCancelWrite) + + teardownCalled := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-ready", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + teardownCalled <- struct{}{} + return nil + }, + promptDone: make(chan struct{}), + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-ready", v1.TaskStateInProgress) + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + _, err := svc.messageQueue.QueueMessage( + ctx, + "session-ready", + "task-ready", + "must remain queued", + "", + messagequeue.QueuedByAgent, + false, + nil, + ) + require.NoError(t, err) + + stopDone := make(chan coordinatorStopCallOutcome, 1) + go func() { + result, stopErr := svc.StopTaskForCoordinator(ctx, "task-ready") + stopDone <- coordinatorStopCallOutcome{result: result, err: stopErr} + }() + coordinatorStopAwaitSignal(t, cancelWriteEntered, "blocked cancellation write") + + readyDone := make(chan struct{}) + readyCtx := context.WithValue(ctx, coordinatorStopReadyReadKey{}, true) + go func() { + svc.handleAgentReady(readyCtx, watcher.AgentEventData{ + TaskID: "task-ready", SessionID: "session-ready", AgentExecutionID: "execution-ready", + }) + close(readyDone) + }() + coordinatorStopAwaitSignal(t, readyInitialRead, "ready event initial session read") + coordinatorStopWaitForGuardRefs(t, svc, "session-ready", 2) + select { + case <-readyDone: + t.Fatal("ready event completed while coordinator cancellation owned the shared guard") + default: + } + + releaseCancelWrite() + stopOutcome := coordinatorStopAwaitCall(t, stopDone) + coordinatorStopAwaitSignal(t, readyDone, "ready event guarded reread") + coordinatorStopAwaitSignal(t, teardownCalled, "runtime teardown") + + require.NoError(t, stopOutcome.err) + require.Equal(t, CoordinatorTaskStopStatusStopped, stopOutcome.result.Status) + session, err := baseRepo.GetTaskSession(ctx, "session-ready") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + require.Equal(t, 1, svc.messageQueue.GetStatus(ctx, "session-ready").Count) + agentManager.mu.Lock() + promptCalls := append([]promptCall(nil), agentManager.capturedPromptCalls...) + agentManager.mu.Unlock() + require.Empty(t, promptCalls, "ready event must not dispatch replacement work after cancellation") +} diff --git a/apps/backend/internal/orchestrator/event_handlers_agent.go b/apps/backend/internal/orchestrator/event_handlers_agent.go index 68eb37fd45..e59467f1c0 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent.go @@ -39,6 +39,10 @@ func (s *Service) handleAgentRunning(ctx context.Context, data watcher.AgentEven if !s.agentManager.IsPassthroughSession(ctx, data.SessionID) { return } + lock, release := s.acquireCancelInFlightGuard(data.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() session, err := s.repo.GetTaskSession(ctx, data.SessionID) if err != nil { @@ -48,6 +52,13 @@ func (s *Service) handleAgentRunning(ctx context.Context, data watcher.AgentEven zap.Error(err)) return } + if isTerminalSessionState(session.State) { + s.logger.Debug("ignoring agent running event for terminal session", + zap.String("task_id", data.TaskID), + zap.String("session_id", data.SessionID), + zap.String("session_state", string(session.State))) + return + } s.processOnTurnStartViaEngine(ctx, data.TaskID, session) // Move session to running and task to in progress. @@ -584,17 +595,29 @@ func (s *Service) executeQueuedMessage(callerSessionID string, queuedMsg *messag // handleAgentCompleted handles agent completion events func (s *Service) handleAgentCompleted(ctx context.Context, data watcher.AgentEventData) { + if data.SessionID == "" { + s.handleAgentCompletedLocked(ctx, data) + return + } + + // Completion owns workflow advancement only while serialized with every + // cancel/interrupt decision for this session. If coordinator stop won while + // the event waited, the guarded state reload below observes CANCELLED and + // suppresses all workflow/on_enter side effects. + lock, release := s.acquireCancelInFlightGuard(data.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() + + s.handleAgentCompletedLocked(ctx, data) +} + +func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.AgentEventData) { s.logger.Info("handling agent completed", zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.String("agent_execution_id", data.AgentExecutionID)) - // A successful completion clears any transient retry budget for the session. - s.resetTransientRetry(data.SessionID) - - // Update scheduler and remove from queue - s.scheduler.HandleTaskCompleted(data.TaskID, true) - s.scheduler.RemoveTask(data.TaskID) s.markExecutionCompleted(data.SessionID, data.AgentExecutionID) // Check for workflow transition based on session's current step. @@ -604,6 +627,7 @@ func (s *Service) handleAgentCompleted(ctx context.Context, data watcher.AgentEv zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.Error(err)) + go s.cleanupAgentExecution(data.AgentExecutionID, data.TaskID, data.SessionID) return } @@ -639,6 +663,12 @@ func (s *Service) handleAgentCompleted(ctx context.Context, data watcher.AgentEv return } + // A successful, still-live completion clears retry state and scheduler + // ownership only after the guarded terminal/rotation checks above. + s.resetTransientRetry(data.SessionID) + s.scheduler.HandleTaskCompleted(data.TaskID, true) + s.scheduler.RemoveTask(data.TaskID) + s.completeTurnForSession(context.WithoutCancel(ctx), data.SessionID) if s.sessionHasPendingClarification(ctx, data.SessionID) { @@ -683,12 +713,34 @@ func (s *Service) handleAgentCompleted(ctx context.Context, data watcher.AgentEv // handleAgentFailed handles agent failure events func (s *Service) handleAgentFailed(ctx context.Context, data watcher.AgentEventData) { + if data.SessionID == "" { + s.handleAgentFailedLocked(ctx, data) + return + } + + // Linearize every session-backed failure decision with coordinator stop, + // interrupt, and queued-dispatch ownership. The state is re-read inside + // handleAgentFailedLocked after this lock is held: if stop won, failure + // recovery must not create messages, arm retries, or force-clean the + // execution that graceful teardown now owns. + lock, release := s.acquireCancelInFlightGuard(data.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() + + s.handleAgentFailedLocked(ctx, data) +} + +func (s *Service) handleAgentFailedLocked(ctx context.Context, data watcher.AgentEventData) { s.logger.Warn("handling agent failed", zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.String("agent_execution_id", data.AgentExecutionID), zap.String("error_message", data.ErrorMessage)) + if drop, _ := s.shouldDropSessionFailure(ctx, data, "agent.failed", true); drop { + return + } s.markExecutionFailed(data.SessionID, data.AgentExecutionID) // Transient provider errors (529 Overloaded) get a paced, visible @@ -738,6 +790,179 @@ func (s *Service) handleAgentFailed(ctx context.Context, data watcher.AgentEvent go s.cleanupAgentExecution(data.AgentExecutionID, data.TaskID, data.SessionID) } +func (s *Service) shouldDropSessionFailure( + ctx context.Context, + data watcher.AgentEventData, + source string, + dropWhenUnavailable bool, +) (bool, models.TaskSessionState) { + if data.SessionID == "" { + return false, "" + } + session, err := s.repo.GetTaskSession(ctx, data.SessionID) + if err != nil || session == nil { + s.logger.Warn("dropping session failure because current state is unavailable", + zap.String("task_id", data.TaskID), + zap.String("session_id", data.SessionID), + zap.String("failure_source", source), + zap.Error(err)) + if dropWhenUnavailable { + // The workflow state is unavailable, but the execution ID is still + // authoritative enough for bounded runtime cleanup. Leaving it alive + // here leaks agentctl/port state until restart reconciliation. + go s.cleanupAgentExecution(data.AgentExecutionID, data.TaskID, data.SessionID) + } + return dropWhenUnavailable, "" + } + if isTerminalSessionState(session.State) { + s.resetTransientRetry(data.SessionID) + s.logger.Debug("dropping session failure for terminal session", + zap.String("task_id", data.TaskID), + zap.String("session_id", data.SessionID), + zap.String("failure_source", source), + zap.String("session_state", string(session.State))) + go s.cleanupAgentExecution(data.AgentExecutionID, data.TaskID, data.SessionID) + return true, session.State + } + liveExecutionID, _ := s.agentManager.GetExecutionIDForSession(ctx, data.SessionID) + if data.AgentExecutionID != "" && liveExecutionID != "" && liveExecutionID != data.AgentExecutionID { + s.logger.Debug("dropping session failure for rotated execution", + zap.String("task_id", data.TaskID), + zap.String("session_id", data.SessionID), + zap.String("failure_source", source), + zap.String("event_execution_id", data.AgentExecutionID), + zap.String("live_execution_id", liveExecutionID)) + go s.cleanupAgentExecution(data.AgentExecutionID, data.TaskID, data.SessionID) + return true, "" + } + return false, "" +} + +type executionTeardownIntent uint8 + +const ( + executionTeardownIntentGraceful executionTeardownIntent = iota + 1 + executionTeardownIntentForce +) + +type executionTeardownClaim struct { + intent executionTeardownIntent + expiresAt time.Time +} + +// claimExecutionTeardown accepts the first teardown intent for one concrete +// execution. Callers with a session ID must invoke it while holding that +// session's cancelInFlight guard, making the decision atomic with the state +// read/write that justified the intent. The blocking teardown itself happens +// only after that guard is released. +func (s *Service) claimExecutionTeardown( + sessionID, executionID string, + intent executionTeardownIntent, +) bool { + if sessionID == "" || executionID == "" { + return false + } + key := terminalExecutionKey(sessionID, executionID) + for { + now := time.Now() + claim := executionTeardownClaim{ + intent: intent, + expiresAt: now.Add(completedExecutionRetention), + } + value, loaded := s.executionTeardownClaims.LoadOrStore(key, claim) + if !loaded { + time.AfterFunc(completedExecutionRetention, func() { + s.deleteExecutionTeardownClaimIfExpired(key, claim.expiresAt) + }) + return true + } + current, ok := value.(executionTeardownClaim) + if !ok { + s.executionTeardownClaims.Delete(key) + continue + } + if now.Before(current.expiresAt) { + return false + } + if s.executionTeardownClaims.CompareAndDelete(key, current) { + continue + } + } +} + +// claimForcedExecutionCleanup serializes exact-execution cleanup arbitration +// with coordinator cancellation. The caller performs blocking cleanup only +// after this method releases the per-session guard. +func (s *Service) claimForcedExecutionCleanup(sessionID, executionID string) bool { + if executionID == "" { + return false + } + if sessionID == "" { + return true + } + lock, release := s.acquireCancelInFlightGuard(sessionID) + lock.Lock() + claimed := s.claimExecutionTeardown( + sessionID, + executionID, + executionTeardownIntentForce, + ) + lock.Unlock() + release() + return claimed +} + +// RegisterExecutionStopOwner records explicit teardown ownership before a +// cancellation write. Registration never suppresses the requested stop; it +// only keeps orphan cleanup from racing that owner for the same execution. +func (s *Service) RegisterExecutionStopOwner(sessionID, executionID string, force bool) { + if sessionID == "" || executionID == "" { + return + } + lock, release := s.acquireCancelInFlightGuard(sessionID) + lock.Lock() + defer func() { + lock.Unlock() + release() + }() + + intent := executionTeardownIntentGraceful + if force { + intent = executionTeardownIntentForce + } + if s.claimExecutionTeardown(sessionID, executionID, intent) || !force { + return + } + + // An explicit force request escalates advisory metadata, but it always runs + // regardless of whether another stop already registered this execution. + key := terminalExecutionKey(sessionID, executionID) + value, ok := s.executionTeardownClaims.Load(key) + current, valid := value.(executionTeardownClaim) + if !ok || !valid || current.intent != executionTeardownIntentGraceful { + return + } + upgraded := executionTeardownClaim{ + intent: executionTeardownIntentForce, + expiresAt: time.Now().Add(completedExecutionRetention), + } + s.executionTeardownClaims.Store(key, upgraded) + time.AfterFunc(completedExecutionRetention, func() { + s.deleteExecutionTeardownClaimIfExpired(key, upgraded.expiresAt) + }) +} + +func (s *Service) deleteExecutionTeardownClaimIfExpired(key string, expiresAt time.Time) { + value, ok := s.executionTeardownClaims.Load(key) + if !ok { + return + } + claim, ok := value.(executionTeardownClaim) + if !ok || !claim.expiresAt.After(expiresAt) { + s.executionTeardownClaims.Delete(key) + } +} + // wasResumeAttempt checks whether the session's last execution used a resume token. // If the token is still present in the DB, the agent was started with --resume. func (s *Service) wasResumeAttempt(ctx context.Context, sessionID string) bool { @@ -958,6 +1183,28 @@ func (s *Service) isOfficeSession(ctx context.Context, sessionID string) bool { // error on focus / auto-resume. // Returns true if the failure was handled (caller should skip default FAILED logic). func (s *Service) handleAgentStartFailed(ctx context.Context, taskID, sessionID, agentExecutionID string, err error, fromResume bool) bool { + if sessionID != "" { + lock, release := s.acquireCancelInFlightGuard(sessionID) + defer release() + lock.Lock() + defer lock.Unlock() + + if drop, terminalState := s.shouldDropSessionFailure(ctx, watcher.AgentEventData{ + TaskID: taskID, + SessionID: sessionID, + AgentExecutionID: agentExecutionID, + ErrorMessage: err.Error(), + }, "agent process start", false); drop { + // A cancellation that landed after the executor's first terminal-state + // read still needs its exact-execution cleanup path. Returning false lets + // the executor observe the final CANCELLED state and arbitrate teardown. + if terminalState == models.TaskSessionStateCancelled { + return false + } + return true + } + } + if !isAuthError(err.Error()) { if fromResume { s.logger.Info("suppressing toast for resume bootstrap failure", @@ -1122,6 +1369,13 @@ func (s *Service) cleanupAgentExecution(executionID, taskID, sessionID string) { if executionID == "" { return } + if !s.claimForcedExecutionCleanup(sessionID, executionID) { + s.logger.Debug("skipping duplicate execution teardown", + zap.String("execution_id", executionID), + zap.String("task_id", taskID), + zap.String("session_id", sessionID)) + return + } ctx := context.Background() if err := s.executor.StopExecution(ctx, executionID, "agent completed", true); err != nil { s.logger.Debug("agent execution cleanup after terminal state", diff --git a/apps/backend/internal/orchestrator/event_handlers_clarification.go b/apps/backend/internal/orchestrator/event_handlers_clarification.go index cffd955ccd..ec59b92a0d 100644 --- a/apps/backend/internal/orchestrator/event_handlers_clarification.go +++ b/apps/backend/internal/orchestrator/event_handlers_clarification.go @@ -59,6 +59,11 @@ func (s *Service) handleClarificationStaleDismissed(ctx context.Context, event * } writeCtx := context.WithoutCancel(ctx) + lock, release := s.acquireCancelInFlightGuard(data.SessionID) + defer release() + lock.Lock() + defer lock.Unlock() + if s.sessionHasPendingClarification(writeCtx, data.SessionID) { return nil } @@ -282,15 +287,42 @@ func (s *Service) retryClarificationAfterCancel(ctx context.Context, data clarif lock.Lock() defer lock.Unlock() + // Coordinator stop may have won while this recovery waited for the shared + // guard. Re-read inside the critical section and never revive a terminal + // session or queue replacement work for it. + session, sessionErr := s.repo.GetTaskSession(ctx, data.SessionID) + if sessionErr != nil || session == nil { + s.logger.Warn("cannot confirm live session for clarification recovery", + zap.String("session_id", data.SessionID), + zap.Error(sessionErr)) + return false + } + if isTerminalSessionState(session.State) { + s.logger.Debug("skipping clarification recovery for terminal session", + zap.String("session_id", data.SessionID), + zap.String("session_state", string(session.State))) + return false + } + if err := s.cancelAgentSilent(ctx, data.TaskID, data.SessionID); err != nil { s.logger.Warn("cancel failed (agent likely dead), force-transitioning session state", zap.String("session_id", data.SessionID), zap.Error(err)) - // Force-revert session state so the retry prompt can proceed - if revertErr := s.repo.UpdateTaskSessionState(ctx, data.SessionID, models.TaskSessionStateWaitingForInput, ""); revertErr != nil { + // Revert through the terminal-safe state writer. Production uses a + // compare-and-set, and the shared guard keeps coordinator cancellation + // outside this narrow mutation. + reverted := s.updateTaskSessionState( + ctx, + data.TaskID, + data.SessionID, + models.TaskSessionStateWaitingForInput, + "", + true, + session, + ) + if reverted == nil || reverted.State != models.TaskSessionStateWaitingForInput { s.logger.Error("failed to force-revert session state for clarification recovery", - zap.String("session_id", data.SessionID), - zap.Error(revertErr)) + zap.String("session_id", data.SessionID)) return false } s.completeTurnForSession(ctx, data.SessionID) diff --git a/apps/backend/internal/orchestrator/event_handlers_clarification_test.go b/apps/backend/internal/orchestrator/event_handlers_clarification_test.go index f494271929..4d51229cd5 100644 --- a/apps/backend/internal/orchestrator/event_handlers_clarification_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_clarification_test.go @@ -284,6 +284,63 @@ func TestHandleClarificationStaleDismissed(t *testing.T) { t.Fatalf("expected task state %q, got %q (ok=%v)", v1.TaskStateReview, state, ok) } }) + + t.Run("coordinator cancellation wins while stale-dismiss event waits", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + requireNoError(t, repo.UpdateTaskSessionState( + ctx, + "s1", + models.TaskSessionStateWaitingForInput, + "", + )) + stepGetter := newMockStepGetter() + stepGetter.steps["step1"] = &wfmodels.WorkflowStep{ + ID: "step1", WorkflowID: "wf1", Name: "Plan", Position: 0, + Events: wfmodels.StepEvents{OnTurnComplete: []wfmodels.OnTurnCompleteAction{{ + Type: wfmodels.OnTurnCompleteMoveToNext, + }}}, + } + stepGetter.steps["step2"] = &wfmodels.WorkflowStep{ + ID: "step2", WorkflowID: "wf1", Name: "Implement", Position: 1, + } + svc := createEngineService(t, repo, stepGetter, &mockAgentManager{}) + event := bus.NewEvent("clarification.stale_dismissed", "test", map[string]any{ + "session_id": "s1", + "task_id": "t1", + "pending_id": "pending-1", + }) + + guard, release := svc.acquireCancelInFlightGuard("s1") + guard.Lock() + done := make(chan error, 1) + go func() { done <- svc.handleClarificationStaleDismissed(ctx, event) }() + coordinatorStopWaitForGuardRefs(t, svc, "s1", 2) + changed, _, err := repo.CancelActiveTaskSession(ctx, "s1", coordinatorMCPStopReason) + requireNoError(t, err) + if !changed { + t.Fatal("coordinator cancellation did not change the waiting session") + } + guard.Unlock() + release() + select { + case handlerErr := <-done: + requireNoError(t, handlerErr) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for stale-dismiss handler") + } + + session, err := repo.GetTaskSession(ctx, "s1") + requireNoError(t, err) + if session.State != models.TaskSessionStateCancelled { + t.Fatalf("expected cancelled session, got %q", session.State) + } + task, err := repo.GetTask(ctx, "t1") + requireNoError(t, err) + if task.WorkflowStepID != "step1" { + t.Fatalf("stale-dismiss advanced workflow after cancellation: %q", task.WorkflowStepID) + } + }) } func TestPauseForClarificationInput_SilentlyCancelsTurnWithoutWorkflowTransition(t *testing.T) { @@ -684,6 +741,60 @@ func TestRetryClarificationAfterCancel_DoesNotStarveUserCancel(t *testing.T) { } } +func TestRetryClarificationAfterCancel_CoordinatorCancellationWinsWhileRetryWaits(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-retry-stop", "session-retry-stop", models.TaskSessionStateRunning) + manager := &mockAgentManager{isAgentRunning: true, repoForExecutionLookup: repo} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), manager) + svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{}) + + guard, release := svc.acquireCancelInFlightGuard("session-retry-stop") + guard.Lock() + done := make(chan bool, 1) + go func() { + done <- svc.retryClarificationAfterCancel( + ctx, + clarificationAnsweredData{TaskID: "task-retry-stop", SessionID: "session-retry-stop"}, + "clarification answer", + fmt.Errorf("wrapped: %w", ErrAgentPromptInProgress), + ) + }() + coordinatorStopWaitForGuardRefs(t, svc, "session-retry-stop", 2) + changed, _, err := repo.CancelActiveTaskSession( + ctx, + "session-retry-stop", + coordinatorMCPStopReason, + ) + if err != nil || !changed { + t.Fatalf("cancel running session: changed=%v err=%v", changed, err) + } + guard.Unlock() + release() + + select { + case recovered := <-done: + if recovered { + t.Fatal("clarification retry reported recovery after coordinator cancellation") + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for guarded clarification retry") + } + if got := manager.cancelAgentCalls.Load(); got != 0 { + t.Fatalf("clarification retry cancelled agent after stop won: %d calls", got) + } + if got := svc.messageQueue.GetStatus(ctx, "session-retry-stop").Count; got != 0 { + t.Fatalf("clarification retry queued replacement after stop won: %d messages", got) + } + session, err := repo.GetTaskSession(ctx, "session-retry-stop") + if err != nil { + t.Fatalf("get session: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Fatalf("expected cancelled session, got %q", session.State) + } +} + // TestDispatchClarificationResumeLocked_ReturnPaths pins the two outcomes that // are deterministically reachable at this seam — a genuine error (nil queue) // and an immediate dispatch (nil) — so the caller can tell a real failure from diff --git a/apps/backend/internal/orchestrator/event_handlers_runtime_state_race_test.go b/apps/backend/internal/orchestrator/event_handlers_runtime_state_race_test.go new file mode 100644 index 0000000000..448b42b55d --- /dev/null +++ b/apps/backend/internal/orchestrator/event_handlers_runtime_state_race_test.go @@ -0,0 +1,66 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +func TestReconcileTaskStateForRuntime_RejectsClarificationRace(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "task-runtime-race", "session-runtime-race", "") + require.NoError(t, repo.UpdateTaskState(ctx, "task-runtime-race", v1.TaskStateReview)) + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "session-runtime-race", + models.TaskSessionStateStarting, + "", + )) + + taskRepo := newMockTaskRepo() + called := false + taskRepo.updateIfSessionState = func( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, + ) (bool, error) { + called = true + require.Equal(t, models.TaskSessionStateStarting, expectedSessionState) + + // Clarification wins after reconciliation's session read but before its + // task write. The repository CAS must observe WAITING and reject the + // stale STARTING -> IN_PROGRESS writer. + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + sessionID, + models.TaskSessionStateWaitingForInput, + "", + )) + require.NoError(t, repo.UpdateTaskState(ctx, taskID, v1.TaskStateReview)) + _, updated, err := repo.UpdateTaskStateIfSessionState( + ctx, taskID, sessionID, expectedSessionState, state, + ) + return updated, err + } + + svc := createTestService(repo, newMockStepGetter(), taskRepo) + require.NoError(t, svc.reconcileTaskStateForRuntime( + ctx, + "task-runtime-race", + "session-runtime-race", + v1.TaskStateInProgress, + )) + require.True(t, called) + + task, err := repo.GetTask(ctx, "task-runtime-race") + require.NoError(t, err) + require.Equal(t, v1.TaskStateReview, task.State) + session, err := repo.GetTaskSession(ctx, "session-runtime-race") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateWaitingForInput, session.State) +} diff --git a/apps/backend/internal/orchestrator/event_handlers_step_completion.go b/apps/backend/internal/orchestrator/event_handlers_step_completion.go index 096b13c524..d23285373d 100644 --- a/apps/backend/internal/orchestrator/event_handlers_step_completion.go +++ b/apps/backend/internal/orchestrator/event_handlers_step_completion.go @@ -79,6 +79,10 @@ func (s *Service) onStepCompletionSignaled(ctx context.Context, event *bus.Event } return } + lock, release := s.acquireCancelInFlightGuard(sessionID) + defer release() + lock.Lock() + defer lock.Unlock() session, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil { diff --git a/apps/backend/internal/orchestrator/event_handlers_step_completion_test.go b/apps/backend/internal/orchestrator/event_handlers_step_completion_test.go index c8a1a4a3de..aebfd6fd02 100644 --- a/apps/backend/internal/orchestrator/event_handlers_step_completion_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_step_completion_test.go @@ -470,4 +470,64 @@ func TestOnStepCompletionSignaled(t *testing.T) { t.Errorf("expected no transition for un-gated step, got %q", updated.WorkflowStepID) } }) + + t.Run("coordinator cancellation wins while signal subscriber waits", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + if err := repo.UpdateTaskSessionState(ctx, "s1", models.TaskSessionStateWaitingForInput, ""); err != nil { + t.Fatalf("flip session waiting: %v", err) + } + signal := models.PendingStepCompletionSignal{ + StepID: "step1", Source: models.StepCompletionSourceAgent, + Summary: "done", SignaledAt: time.Now().UTC(), + } + if err := repo.SetSessionMetadataKey(ctx, "s1", models.SessionMetaKeyPendingStepCompletion, signal); err != nil { + t.Fatalf("seed signal: %v", err) + } + stepGetter := newMockStepGetter() + stepGetter.steps["step1"] = &wfmodels.WorkflowStep{ + ID: "step1", WorkflowID: "wf1", Name: "Step 1", Position: 0, + AutoAdvanceRequiresSignal: true, + Events: wfmodels.StepEvents{OnTurnComplete: []wfmodels.OnTurnCompleteAction{{ + Type: wfmodels.OnTurnCompleteMoveToNext, + }}}, + } + stepGetter.steps["step2"] = &wfmodels.WorkflowStep{ + ID: "step2", WorkflowID: "wf1", Name: "Step 2", Position: 1, + } + svc := createTestService(repo, stepGetter, newMockTaskRepo()) + guard, release := svc.acquireCancelInFlightGuard("s1") + guard.Lock() + done := make(chan struct{}) + go func() { + svc.onStepCompletionSignaled(ctx, buildEvent("t1", "s1", "step1")) + close(done) + }() + coordinatorStopWaitForGuardRefs(t, svc, "s1", 2) + changed, _, err := repo.CancelActiveTaskSession(ctx, "s1", coordinatorMCPStopReason) + if err != nil || !changed { + t.Fatalf("cancel waiting session: changed=%v err=%v", changed, err) + } + guard.Unlock() + release() + coordinatorStopAwaitSignal(t, done, "guarded step-completion subscriber") + + updated, err := repo.GetTask(ctx, "t1") + if err != nil { + t.Fatalf("get task: %v", err) + } + if updated.WorkflowStepID != "step1" { + t.Fatalf("stale signal advanced workflow after cancellation: %q", updated.WorkflowStepID) + } + updatedSession, err := repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("get session: %v", err) + } + if updatedSession.State != models.TaskSessionStateCancelled { + t.Fatalf("expected cancelled session, got %q", updatedSession.State) + } + if _, hasSignal := models.LoadPendingStepSignal(updatedSession.Metadata); !hasSignal { + t.Fatal("stop-winning subscriber consumed the queued completion signal") + } + }) } diff --git a/apps/backend/internal/orchestrator/event_handlers_streaming.go b/apps/backend/internal/orchestrator/event_handlers_streaming.go index de9ed6f6dc..08eff6dc8c 100644 --- a/apps/backend/internal/orchestrator/event_handlers_streaming.go +++ b/apps/backend/internal/orchestrator/event_handlers_streaming.go @@ -11,6 +11,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/orchestrator/sessionstate" "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" @@ -471,8 +472,8 @@ func (s *Service) shouldDropCompletedExecutionStreamEvent(payload *lifecycle.Age // updateTaskSessionState transitions a session to nextState with guard checks. // When a preloadedSession is provided, its State is used for guard conditions (terminal-state // check, same-state check). This is an optimistic fast-path: between load and check another -// goroutine may have changed the state in the DB. The guards are best-effort to avoid -// unnecessary writes; the DB update via UpdateTaskSessionState is the atomic source of truth. +// goroutine may have changed the state in the DB. Production repositories use +// an expected-state compare-and-set so a delayed writer cannot revive a terminal session. // Returns the session row after a successful write (refreshed from DB when possible); callers // that need authoritative UpdatedAt should use the return value, not the preloaded input. func (s *Service) updateTaskSessionState(ctx context.Context, taskID, sessionID string, nextState models.TaskSessionState, errorMessage string, allowWakeFromWaiting bool, preloadedSession ...*models.TaskSession) *models.TaskSession { @@ -497,21 +498,12 @@ func (s *Service) updateTaskSessionState(ctx context.Context, taskID, sessionID if session.State == nextState { return session } - if err := s.repo.UpdateTaskSessionState(ctx, sessionID, nextState, errorMessage); err != nil { - s.logger.Error("failed to update task session state", - zap.String("session_id", sessionID), - zap.String("state", string(nextState)), - zap.Error(err)) + session, authoritativeUpdatedAt, changed := s.persistTaskSessionState( + ctx, sessionID, session, nextState, errorMessage, + ) + if !changed { return session } - var authoritativeUpdatedAt *time.Time - if refreshed, err := s.repo.GetTaskSession(ctx, sessionID); err == nil && refreshed != nil { - session = refreshed - if !refreshed.UpdatedAt.IsZero() { - t := refreshed.UpdatedAt.UTC() - authoritativeUpdatedAt = &t - } - } if authoritativeUpdatedAt == nil { s.logger.Warn("skipping session state_changed publish; could not read authoritative updated_at", zap.String("task_id", taskID), @@ -531,6 +523,236 @@ func (s *Service) updateTaskSessionState(ctx context.Context, taskID, sessionID return session } +func (s *Service) persistTaskSessionState( + ctx context.Context, + sessionID string, + session *models.TaskSession, + nextState models.TaskSessionState, + errorMessage string, +) (*models.TaskSession, *time.Time, bool) { + if updater, ok := s.repo.(conditionalTaskSessionStateUpdater); ok { + changed, updatedAt, err := updater.UpdateTaskSessionStateIfCurrent( + ctx, sessionID, session.State, nextState, errorMessage, + ) + if err != nil { + s.logTaskSessionStateWriteError(sessionID, nextState, err) + return session, nil, false + } + if !changed { + return s.refreshTaskSessionOr(ctx, sessionID, session), nil, false + } + persisted := taskSessionAfterStateWrite(session, nextState, errorMessage, updatedAt) + persisted = s.refreshTaskSessionOr(ctx, sessionID, persisted) + t := updatedAt.UTC() + return persisted, &t, true + } + + if err := s.repo.UpdateTaskSessionState(ctx, sessionID, nextState, errorMessage); err != nil { + s.logTaskSessionStateWriteError(sessionID, nextState, err) + return session, nil, false + } + refreshed := s.refreshTaskSessionOr(ctx, sessionID, session) + if refreshed.UpdatedAt.IsZero() { + return refreshed, nil, true + } + t := refreshed.UpdatedAt.UTC() + return refreshed, &t, true +} + +func (s *Service) refreshTaskSessionOr( + ctx context.Context, + sessionID string, + fallback *models.TaskSession, +) *models.TaskSession { + refreshed, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil || refreshed == nil { + return fallback + } + return refreshed +} + +func (s *Service) logTaskSessionStateWriteError( + sessionID string, + nextState models.TaskSessionState, + err error, +) { + s.logger.Error("failed to update task session state", + zap.String("session_id", sessionID), + zap.String("state", string(nextState)), + zap.Error(err)) +} + +// transitionTaskSessionState performs the strict state transition used by +// coordinator stop. It always reads current state, surfaces persistence/read +// failures, and publishes the accepted transition before returning. +func (s *Service) transitionTaskSessionState( + ctx context.Context, + taskID, sessionID string, + nextState models.TaskSessionState, + errorMessage string, +) (bool, models.TaskSessionState, error) { + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return false, "", fmt.Errorf("get session before state transition: %w", err) + } + if session == nil { + return false, "", fmt.Errorf("get session before state transition: session %q is nil", sessionID) + } + if isTerminalSessionState(session.State) || session.State == nextState { + return false, session.State, nil + } + + oldState := session.State + changed, refreshed, authoritativeUpdatedAt, err := s.persistStrictTaskSessionState( + ctx, sessionID, session, nextState, errorMessage, + ) + if err != nil { + return false, oldState, err + } + if !changed { + return false, refreshed.State, nil + } + s.publishTaskSessionStateChanged( + ctx, + taskID, + sessionID, + oldState, + nextState, + errorMessage, + authoritativeUpdatedAt, + refreshed, + ) + s.maybePromotePrimary(ctx, taskID, sessionID, nextState) + return true, nextState, nil +} + +func (s *Service) persistStrictTaskSessionState( + ctx context.Context, + sessionID string, + session *models.TaskSession, + nextState models.TaskSessionState, + errorMessage string, +) (bool, *models.TaskSession, *time.Time, error) { + if canceller, ok := s.repo.(activeTaskSessionCanceller); ok && nextState == models.TaskSessionStateCancelled { + return s.cancelActiveTaskSessionState(ctx, canceller, sessionID, session, errorMessage) + } + if updater, ok := s.repo.(conditionalTaskSessionStateUpdater); ok { + return s.persistConditionalTaskSessionState(ctx, updater, sessionID, session, nextState, errorMessage) + } + if err := s.repo.UpdateTaskSessionState(ctx, sessionID, nextState, errorMessage); err != nil { + return false, session, nil, err + } + refreshed, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return false, session, nil, fmt.Errorf("get session after state transition: %w", err) + } + if refreshed == nil { + return false, session, nil, fmt.Errorf("get session after state transition: session %q is nil", sessionID) + } + if refreshed.State != nextState { + return false, refreshed, nil, nil + } + var authoritativeUpdatedAt *time.Time + if !refreshed.UpdatedAt.IsZero() { + t := refreshed.UpdatedAt.UTC() + authoritativeUpdatedAt = &t + } + return true, refreshed, authoritativeUpdatedAt, nil +} + +func (s *Service) persistConditionalTaskSessionState( + ctx context.Context, + updater conditionalTaskSessionStateUpdater, + sessionID string, + session *models.TaskSession, + nextState models.TaskSessionState, + errorMessage string, +) (bool, *models.TaskSession, *time.Time, error) { + changed, updatedAt, err := updater.UpdateTaskSessionStateIfCurrent( + ctx, + sessionID, + session.State, + nextState, + errorMessage, + ) + if err != nil { + return false, session, nil, err + } + if !changed { + refreshed, readErr := s.repo.GetTaskSession(ctx, sessionID) + if readErr != nil { + return false, session, nil, fmt.Errorf("get session after rejected state transition: %w", readErr) + } + if refreshed == nil { + return false, session, nil, fmt.Errorf("get session after rejected state transition: session %q is nil", sessionID) + } + return false, refreshed, nil, nil + } + persisted := taskSessionAfterStateWrite(session, nextState, errorMessage, updatedAt) + persisted = s.refreshTaskSessionOr(ctx, sessionID, persisted) + t := updatedAt.UTC() + return true, persisted, &t, nil +} + +func (s *Service) cancelActiveTaskSessionState( + ctx context.Context, + canceller activeTaskSessionCanceller, + sessionID string, + session *models.TaskSession, + errorMessage string, +) (bool, *models.TaskSession, *time.Time, error) { + changed, updatedAt, err := canceller.CancelActiveTaskSession(ctx, sessionID, errorMessage) + if err != nil { + return false, session, nil, err + } + if !changed { + refreshed, readErr := s.repo.GetTaskSession(ctx, sessionID) + if readErr != nil { + return false, session, nil, fmt.Errorf("get session after rejected cancellation: %w", readErr) + } + if refreshed == nil { + return false, session, nil, fmt.Errorf("get session after rejected cancellation: session %q is nil", sessionID) + } + return false, refreshed, nil, nil + } + refreshed := taskSessionAfterStateWrite(session, models.TaskSessionStateCancelled, errorMessage, updatedAt) + refreshed = s.refreshTaskSessionOr(ctx, sessionID, refreshed) + t := updatedAt.UTC() + return true, refreshed, &t, nil +} + +type activeTaskSessionCanceller interface { + CancelActiveTaskSession(ctx context.Context, sessionID, reason string) (bool, time.Time, error) +} + +type conditionalTaskSessionStateUpdater interface { + UpdateTaskSessionStateIfCurrent( + ctx context.Context, + sessionID string, + expected, next models.TaskSessionState, + errorMessage string, + ) (bool, time.Time, error) +} + +func taskSessionAfterStateWrite( + session *models.TaskSession, + nextState models.TaskSessionState, + errorMessage string, + updatedAt time.Time, +) *models.TaskSession { + updated := *session + updated.State = nextState + updated.ErrorMessage = errorMessage + updated.UpdatedAt = updatedAt + if isTerminalSessionState(nextState) { + t := updatedAt + updated.CompletedAt = &t + } else { + updated.CompletedAt = nil + } + return &updated +} + func (s *Service) publishTaskSessionStateChanged( ctx context.Context, taskID, sessionID string, @@ -750,11 +972,11 @@ func (s *Service) setSessionStarting(ctx context.Context, taskID string, session session.State == models.TaskSessionStateStarting && current.State == models.TaskSessionStateFailed if isTerminalSessionState(current.State) && !allowedTerminalRecovery { - return fmt.Errorf("session %s is %s; cannot mark STARTING", session.ID, current.State) + return &executor.SessionStateSupersededError{SessionID: session.ID, State: current.State} } oldState = current.State - if err := s.repo.UpdateTaskSession(ctx, session); err != nil { + if err := s.persistFullTaskSessionIfCurrent(ctx, session, current.State); err != nil { return err } @@ -768,13 +990,13 @@ func (s *Service) setSessionStarting(ctx context.Context, taskID string, session } } - if promoteTask { - s.writeTaskInProgressForRuntime(ctx, taskID, session.ID) - } return nil }(); err != nil { return err } + if promoteTask { + s.writeTaskInProgressForRuntime(ctx, taskID, session.ID) + } if publishSession != nil { s.publishTaskSessionStateChanged(ctx, taskID, session.ID, oldState, session.State, session.ErrorMessage, stateUpdatedAt, publishSession) @@ -782,6 +1004,36 @@ func (s *Service) setSessionStarting(ctx context.Context, taskID string, session return nil } +func (s *Service) persistFullTaskSessionIfCurrent( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) error { + changed, err := s.repo.UpdateTaskSessionIfCurrentState(ctx, session, expected) + if err != nil { + return err + } + if changed { + return nil + } + latest, err := s.repo.GetTaskSession(ctx, session.ID) + if err != nil { + return err + } + if latest == nil { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, session.ID) + } + if isTerminalSessionState(latest.State) { + return &executor.SessionStateSupersededError{SessionID: session.ID, State: latest.State} + } + return fmt.Errorf( + "session %s state changed from %s to %s before full-row persistence", + session.ID, + expected, + latest.State, + ) +} + func (s *Service) setSessionWaitingForInput(ctx context.Context, taskID, sessionID string, preloadedSession ...*models.TaskSession) { // Resolve session up front so we can skip the redundant task-state write // when the session was already WAITING_FOR_INPUT. Without this guard, every @@ -1032,71 +1284,83 @@ func (s *Service) setSessionRunningForExecution(ctx context.Context, taskID, ses return } - s.writeTaskInProgressForRuntime(ctx, taskID, sessionID) + if err := s.reconcileTaskStateForRuntimeLocked( + ctx, + taskID, + sessionID, + v1.TaskStateInProgress, + ); err != nil { + s.logger.Error("failed to update task state to IN_PROGRESS", + zap.String("task_id", taskID), + zap.Error(err)) + } } func (s *Service) writeTaskInProgressForRuntime(ctx context.Context, taskID, sessionID string) { - // Task lookup errors fail closed so office/archived guards cannot be bypassed - // by a transient repository failure. - task, err := s.repo.GetTask(ctx, taskID) - if err != nil { - s.logger.Warn("skipping IN_PROGRESS transition because task lookup failed", - zap.String("task_id", taskID), - zap.Error(err)) - return - } - if taskArchived(task) { - s.logger.Debug("skipping IN_PROGRESS transition for archived task", - zap.String("task_id", taskID)) - return - } - // Office tasks do NOT transition to IN_PROGRESS when their agent's - // session enters RUNNING. The user-facing task status (todo / - // in_review / done / blocked) reflects workflow position, not the - // agent's runtime cycle — that's surfaced via the topbar Working - // spinner and the inline session timeline entry instead. Skipping - // the transition prevents the REVIEW → IN_PROGRESS → REVIEW flicker - // across follow-up comment turns. - if task != nil && task.AssigneeAgentProfileID != "" { - s.logger.Debug("skipping IN_PROGRESS transition for office task", - zap.String("task_id", taskID)) - return - } - if sessionID != "" { - session, sessionErr := s.repo.GetTaskSession(ctx, sessionID) - if sessionErr != nil { - s.logger.Warn("skipping IN_PROGRESS transition because session lookup failed", - zap.String("task_id", taskID), - zap.String("session_id", sessionID), - zap.Error(sessionErr)) - return - } - if session == nil || !sessionstate.IsWorking(session.State) { - state := "" - if session != nil { - state = string(session.State) - } - s.logger.Debug("skipping IN_PROGRESS transition because session is no longer active", - zap.String("task_id", taskID), - zap.String("session_id", sessionID), - zap.String("session_state", state)) - return - } - } - - // UpdateTaskStateIfNotArchived (not the unconditional UpdateTaskState) so - // the write is atomic against archived_at: the taskArchived guard above - // reads the row before this call, and ArchiveTask can commit in that - // window without changing task.State, so only an archive-aware - // conditional write closes the race (PR #1706 review). - updated, err := s.taskRepo.UpdateTaskStateIfNotArchived(ctx, taskID, v1.TaskStateInProgress) + err := s.reconcileTaskStateForRuntime(ctx, taskID, sessionID, v1.TaskStateInProgress) if err != nil { s.logger.Error("failed to update task state to IN_PROGRESS", zap.String("task_id", taskID), zap.Error(err)) - } else if updated { - s.logger.Info("task moved to IN_PROGRESS state", - zap.String("task_id", taskID)) + } +} + +func (s *Service) reconcileTaskStateForRuntime( + ctx context.Context, + taskID, sessionID string, + state v1.TaskState, +) error { + s.taskRuntimeStateMu.Lock() + defer s.taskRuntimeStateMu.Unlock() + return s.reconcileTaskStateForRuntimeLocked(ctx, taskID, sessionID, state) +} + +func (s *Service) reconcileTaskStateForRuntimeLocked( + ctx context.Context, + taskID, sessionID string, + state v1.TaskState, +) error { + task, err := s.repo.GetTask(ctx, taskID) + if err != nil { + return err + } + if taskArchived(task) { + return nil + } + if state == v1.TaskStateInProgress && task != nil && task.AssigneeAgentProfileID != "" { + return nil + } + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if !runtimeSessionOwnsTaskState(session, state) { + return nil + } + updated, err := s.taskRepo.UpdateTaskStateIfSessionState(ctx, taskID, sessionID, session.State, state) + if err != nil { + return err + } + if updated { + s.logger.Info("task state reconciled from active runtime", + zap.String("task_id", taskID), + zap.String("session_id", sessionID), + zap.String("state", string(state))) + } + return nil +} + +func runtimeSessionOwnsTaskState(session *models.TaskSession, state v1.TaskState) bool { + if session == nil { + return false + } + switch state { + case v1.TaskStateInProgress: + return sessionstate.IsWorking(session.State) + case v1.TaskStateFailed: + return session.State == models.TaskSessionStateFailed + default: + return false } } @@ -1935,13 +2199,25 @@ func (s *Service) persistSessionModelOnSession(ctx context.Context, sessionID st return } session.AgentProfileSnapshot["model"] = model - _ = s.repo.UpdateTaskSession(ctx, session) + if updater, ok := s.repo.(taskSessionAgentProfileSnapshotUpdater); ok { + _ = updater.UpdateTaskSessionAgentProfileSnapshot(ctx, sessionID, session.AgentProfileSnapshot) + } else { + _ = s.repo.UpdateTaskSession(ctx, session) + } // Invalidate the message creator's model cache so subsequent messages use the new model. if s.messageCreator != nil { s.messageCreator.InvalidateModelCache(sessionID) } } +type taskSessionAgentProfileSnapshotUpdater interface { + UpdateTaskSessionAgentProfileSnapshot( + ctx context.Context, + sessionID string, + snapshot map[string]interface{}, + ) error +} + func applySessionRuntimeConfigUpdate(cfg *models.SessionRuntimeConfig, model, mode string, options []streams.ConfigOption) { if model != "" { cfg.Model = model diff --git a/apps/backend/internal/orchestrator/event_handlers_streaming_test.go b/apps/backend/internal/orchestrator/event_handlers_streaming_test.go index d0e8999580..3ddb6135f9 100644 --- a/apps/backend/internal/orchestrator/event_handlers_streaming_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_streaming_test.go @@ -19,6 +19,7 @@ import ( "github.com/kandev/kandev/internal/task/models" sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" taskservice "github.com/kandev/kandev/internal/task/service" + wfmodels "github.com/kandev/kandev/internal/workflow/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) @@ -53,6 +54,20 @@ type failSetSessionMetadataRepo struct { repoStore } +type failSessionStateUpdateRepo struct { + repoStore + err error +} + +func (r failSessionStateUpdateRepo) UpdateTaskSessionState( + context.Context, + string, + models.TaskSessionState, + string, +) error { + return r.err +} + type failSetBaselineRepo struct { repoStore } @@ -237,6 +252,53 @@ func TestUpdateTaskSessionStatePublishesPersistedUpdatedAt(t *testing.T) { require.Equal(t, session.UpdatedAt.UTC().Format(time.RFC3339Nano), data["updated_at"]) } +func TestTransitionTaskSessionStateReportsAcceptedWrite(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + eb := &recordingEventBus{} + svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) + svc.eventBus = eb + + changed, finalState, err := svc.transitionTaskSessionState( + ctx, + "t1", + "s1", + models.TaskSessionStateCancelled, + "coordinator stop", + ) + + require.NoError(t, err) + require.True(t, changed) + require.Equal(t, models.TaskSessionStateCancelled, finalState) + require.Len(t, eb.events, 1) + require.Equal(t, events.TaskSessionStateChanged, eb.events[0].subject) +} + +func TestTransitionTaskSessionStateReportsPersistenceFailure(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + writeFailure := errors.New("database is read-only") + eb := &recordingEventBus{} + svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) + svc.repo = failSessionStateUpdateRepo{repoStore: repo, err: writeFailure} + svc.eventBus = eb + + changed, finalState, err := svc.transitionTaskSessionState( + ctx, + "t1", + "s1", + models.TaskSessionStateCancelled, + "coordinator stop", + ) + + require.ErrorIs(t, err, writeFailure) + require.False(t, changed) + require.Equal(t, models.TaskSessionStateRunning, finalState) + require.Empty(t, eb.events) +} + func TestHandleSessionModeEvent(t *testing.T) { t.Run("publishes plan mode", func(t *testing.T) { eb := &recordingEventBus{} @@ -1219,6 +1281,128 @@ func TestWriteTaskInProgressForRuntimeSkipsArchivedTask(t *testing.T) { "runtime reconciliation must not promote archived tasks to IN_PROGRESS") } +func TestWriteTaskInProgressForRuntimeSkipsCancelledSession(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "") + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "s1", + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + )) + task, err := repo.GetTask(ctx, "t1") + require.NoError(t, err) + task.State = v1.TaskStateReview + require.NoError(t, repo.UpdateTask(ctx, task)) + + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "t1", v1.TaskStateReview) + svc := createTestService(repo, newMockStepGetter(), taskRepo) + + svc.writeTaskInProgressForRuntime(ctx, "t1", "s1") + + require.Zero(t, taskRepo.stateWrites["t1"], + "cancelled session must not promote task from REVIEW to IN_PROGRESS") + require.Equal(t, v1.TaskStateReview, taskRepo.tasks["t1"].State) +} + +func TestHandleAgentCompleted_CancelledSessionDoesNotForceCleanup(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "") + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "s1", + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + )) + + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-completed", nil + }, + } + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), agentManager) + svc.RegisterExecutionStopOwner("s1", "execution-completed", false) + + svc.handleAgentCompleted(ctx, watcher.AgentEventData{ + TaskID: "t1", + SessionID: "s1", + AgentExecutionID: "execution-completed", + }) + coordinatorStopWaitForGuardReleased(t, svc, "s1") + + agentManager.mu.Lock() + calls := append([]stopAgentCall(nil), agentManager.stopAgentWithReasonArgs...) + agentManager.mu.Unlock() + require.Empty(t, calls, "cancelled completion must not replace graceful teardown with force cleanup") + session, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) +} + +func TestHandleAgentCompleted_CoordinatorCancellationWinsWhileEventWaits(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "task-completion-stop-race", "session-completion-stop-race", "step-1") + stepGetter := newMockStepGetter() + stepGetter.steps["step-1"] = &wfmodels.WorkflowStep{ + ID: "step-1", WorkflowID: "wf1", Name: "Work", Position: 0, + Events: wfmodels.StepEvents{OnTurnComplete: []wfmodels.OnTurnCompleteAction{{ + Type: wfmodels.OnTurnCompleteMoveToNext, + }}}, + } + stepGetter.steps["step-2"] = &wfmodels.WorkflowStep{ + ID: "step-2", WorkflowID: "wf1", Name: "Review", Position: 1, + } + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-completion-stop-race", nil + }, + } + svc := createTestServiceWithScheduler(repo, stepGetter, newMockTaskRepo(), manager) + + guard, release := svc.acquireCancelInFlightGuard("session-completion-stop-race") + guard.Lock() + eventDone := make(chan struct{}) + go func() { + svc.handleAgentCompleted(ctx, watcher.AgentEventData{ + TaskID: "task-completion-stop-race", + SessionID: "session-completion-stop-race", + AgentExecutionID: "execution-completion-stop-race", + }) + close(eventDone) + }() + coordinatorStopWaitForGuardRefs(t, svc, "session-completion-stop-race", 2) + changed, _, err := repo.CancelActiveTaskSession( + ctx, + "session-completion-stop-race", + coordinatorMCPStopReason, + ) + require.NoError(t, err) + require.True(t, changed) + require.True(t, svc.claimExecutionTeardown( + "session-completion-stop-race", + "execution-completion-stop-race", + executionTeardownIntentGraceful, + )) + guard.Unlock() + release() + coordinatorStopAwaitSignal(t, eventDone, "guarded completion event") + + session, err := repo.GetTaskSession(ctx, "session-completion-stop-race") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + task, err := repo.GetTask(ctx, "task-completion-stop-race") + require.NoError(t, err) + require.Equal(t, "step-1", task.WorkflowStepID, "stale completion advanced workflow after cancellation") + manager.mu.Lock() + stopCalls := append([]stopAgentCall(nil), manager.stopAgentWithReasonArgs...) + manager.mu.Unlock() + require.Empty(t, stopCalls, "stale completion replaced graceful teardown with force cleanup") +} + // TestWriteTaskInProgressForRuntimeUsesArchiveAwareCAS is the TOCTOU // companion to TestWriteTaskInProgressForRuntimeSkipsArchivedTask // (carlosflorencio review on PR #1706): the earlier taskArchived() guard is @@ -1955,6 +2139,26 @@ func TestPersistSessionModel(t *testing.T) { require.Equal(t, "gpt-5.4", preserved.AgentProfileSnapshot["model"]) } +func TestPersistSessionModelDoesNotRestoreStaleActiveStateAfterCancellation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + stale, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, stale.State) + changed, _, err := repo.CancelActiveTaskSession(ctx, "s1", "coordinator stop") + require.NoError(t, err) + require.True(t, changed) + svc := &Service{logger: testLogger(), repo: repo} + + svc.persistSessionModelOnSession(ctx, "s1", stale, "gpt-after-stop") + + stored, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, stored.State) + require.Equal(t, "gpt-after-stop", stored.AgentProfileSnapshot["model"]) +} + func TestPersistSessionRuntimeConfigFromSessionModels(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/orchestrator/event_handlers_test.go b/apps/backend/internal/orchestrator/event_handlers_test.go index c2473a1907..c32c5ced3b 100644 --- a/apps/backend/internal/orchestrator/event_handlers_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_test.go @@ -102,8 +102,15 @@ type mockTaskRepo struct { // callers must route guarded REVIEW writes through the CAS method so an // archive can't race a late write; tests assert this stays 0 for those // paths instead of just checking the resulting state. - unconditionalWrites map[string]int - getTaskErr error // if set, GetTask returns this error + unconditionalWrites map[string]int + getTaskErr error // if set, GetTask returns this error + updateIfSessionState func( + context.Context, + string, + string, + models.TaskSessionState, + v1.TaskState, + ) (bool, error) } func newMockTaskRepo() *mockTaskRepo { @@ -191,6 +198,18 @@ func (m *mockTaskRepo) UpdateTaskStateIfNotArchived( return true, nil } +func (m *mockTaskRepo) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + if m.updateIfSessionState != nil { + return m.updateIfSessionState(ctx, taskID, sessionID, expectedSessionState, state) + } + return m.UpdateTaskStateIfNotArchived(ctx, taskID, state) +} + // mockAgentManager is a minimal mock of executor.AgentManagerClient for testing. type mockAgentManager struct { isPassthrough bool @@ -275,6 +294,7 @@ type mockAgentManager struct { // returned to simulate "no running agent". setSessionModeCalls []sessionModeCall setSessionModeErr error + mcpModeCalls []sessionModeCall setSessionModelCalls []sessionModelCall setSessionModelSupported bool setSessionModelErr error @@ -448,7 +468,11 @@ func (m *mockAgentManager) SetSessionModeBySessionID(_ context.Context, sessionI m.setSessionModeCalls = append(m.setSessionModeCalls, sessionModeCall{SessionID: sessionID, ModeID: modeID}) return m.setSessionModeErr } -func (m *mockAgentManager) SetMcpMode(_ context.Context, _ string, _ string) error { + +func (m *mockAgentManager) SetMcpMode(_ context.Context, executionID, mode string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.mcpModeCalls = append(m.mcpModeCalls, sessionModeCall{SessionID: executionID, ModeID: mode}) return nil } func (m *mockAgentManager) WasSessionInitialized(_ string) bool { return false } @@ -1463,6 +1487,56 @@ func TestHandleAgentCompleted_CleansUpExecution(t *testing.T) { } } +func TestHandleAgentCompleted_LegacyAutoStartDoesNotReenterCancellationGuard(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "task-legacy-auto-start", "session-legacy-auto-start", "step-1") + seedExecutorRunning(t, repo, "session-legacy-auto-start", "task-legacy-auto-start", "execution-legacy-auto-start") + stepGetter := newMockStepGetter() + stepGetter.steps["step-1"] = &wfmodels.WorkflowStep{ + ID: "step-1", WorkflowID: "wf1", Name: "Work", Position: 0, + Events: wfmodels.StepEvents{OnTurnComplete: []wfmodels.OnTurnCompleteAction{{ + Type: wfmodels.OnTurnCompleteMoveToNext, + }}}, + } + stepGetter.steps["step-2"] = &wfmodels.WorkflowStep{ + ID: "step-2", WorkflowID: "wf1", Name: "Continue", Position: 1, + Events: wfmodels.StepEvents{OnEnter: []wfmodels.OnEnterAction{{ + Type: wfmodels.OnEnterAutoStartAgent, + }}}, + } + manager := &mockAgentManager{ + isAgentRunning: true, + repoForExecutionLookup: repo, + promptDone: make(chan struct{}), + } + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-legacy-auto-start", v1.TaskStateInProgress) + svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, manager) + // Explicitly pin the legacy path whose on_enter used to run inline and + // deadlock by re-entering the completion handler's session guard. + svc.workflowEngine = nil + + done := make(chan struct{}) + go func() { + svc.handleAgentCompleted(ctx, watcher.AgentEventData{ + TaskID: "task-legacy-auto-start", + SessionID: "session-legacy-auto-start", + AgentExecutionID: "execution-legacy-auto-start", + }) + close(done) + }() + coordinatorStopAwaitSignal(t, done, "legacy completion handler") + select { + case <-manager.promptDone: + case <-time.After(2 * time.Second): + t.Fatal("legacy on_enter auto-start did not acquire the released session guard") + } + task, err := repo.GetTask(ctx, "task-legacy-auto-start") + require.NoError(t, err) + require.Equal(t, "step-2", task.WorkflowStepID) +} + func TestHandleAgentCompleted_MarksRotatedExecutionCompleted(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) @@ -1826,6 +1900,49 @@ func TestHandleAgentRunning_PassthroughGuard(t *testing.T) { // Should not panic or error with empty session ID. svc.handleAgentRunning(ctx, watcher.AgentEventData{TaskID: "t1", SessionID: ""}) }) + + t.Run("coordinator cancellation wins while passthrough event waits", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + stepGetter := newMockStepGetter() + stepGetter.steps["step1"] = &wfmodels.WorkflowStep{ + ID: "step1", WorkflowID: "wf1", Name: "Step 1", Position: 0, + Events: wfmodels.StepEvents{OnTurnStart: []wfmodels.OnTurnStartAction{{ + Type: wfmodels.OnTurnStartMoveToNext, + }}}, + } + stepGetter.steps["step2"] = &wfmodels.WorkflowStep{ + ID: "step2", WorkflowID: "wf1", Name: "Step 2", Position: 1, + } + svc := createTestServiceWithAgent( + repo, + stepGetter, + newMockTaskRepo(), + &mockAgentManager{isPassthrough: true}, + ) + + guard, release := svc.acquireCancelInFlightGuard("s1") + guard.Lock() + done := make(chan struct{}) + go func() { + svc.handleAgentRunning(ctx, watcher.AgentEventData{TaskID: "t1", SessionID: "s1"}) + close(done) + }() + coordinatorStopWaitForGuardRefs(t, svc, "s1", 2) + changed, _, err := repo.CancelActiveTaskSession(ctx, "s1", coordinatorMCPStopReason) + require.NoError(t, err) + require.True(t, changed) + guard.Unlock() + release() + coordinatorStopAwaitSignal(t, done, "guarded passthrough running event") + + session, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, session.State) + task, err := repo.GetTask(ctx, "t1") + require.NoError(t, err) + require.Equal(t, "step1", task.WorkflowStepID) + }) } // TestHandleAgentRunning_DoesNotWakeWaitingAcpSession is the regression guard for @@ -2248,6 +2365,68 @@ func TestHandleAgentStartFailed(t *testing.T) { t.Error("expected handled=true for auth errors") } }) + + t.Run("session read failure leaves strict executor cleanup in charge", func(t *testing.T) { + baseRepo := setupTestRepo(t) + seedSession(t, baseRepo, "t1", "s1", "step1") + repo := &coordinatorStopRepoHooks{ + repoStore: baseRepo, + getSessionFunc: func(context.Context, string) (*models.TaskSession, error) { + return nil, errors.New("temporary session read failure") + }, + } + svc := createTestService(baseRepo, newMockStepGetter(), newMockTaskRepo()) + svc.repo = repo + + handled := svc.handleAgentStartFailed( + ctx, + "t1", + "s1", + "exec-1", + errors.New("ACP initialize handshake failed"), + false, + ) + + if handled { + t.Error("expected handled=false so executor transition and forced cleanup still run") + } + }) + + t.Run("cancelled session enqueues exact cleanup arbitration", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "s1", + models.TaskSessionStateCancelled, + "cancelled outside coordinator stop", + )) + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler( + repo, + newMockStepGetter(), + newMockTaskRepo(), + agentMgr, + ) + + handled := svc.handleAgentStartFailed( + ctx, + "t1", + "s1", + "exec-unclaimed", + errors.New("ACP initialize handshake failed"), + false, + ) + + require.False(t, handled) + waitForStopCall(t, agentMgr) + agentMgr.mu.Lock() + calls := append([]stopAgentCall(nil), agentMgr.stopAgentWithReasonArgs...) + agentMgr.mu.Unlock() + require.Len(t, calls, 1) + require.Equal(t, "exec-unclaimed", calls[0].ExecutionID) + require.True(t, calls[0].Force) + }) } func TestHandleResumeFailure(t *testing.T) { diff --git a/apps/backend/internal/orchestrator/event_handlers_transient.go b/apps/backend/internal/orchestrator/event_handlers_transient.go index 3a96a1a75d..6094428aac 100644 --- a/apps/backend/internal/orchestrator/event_handlers_transient.go +++ b/apps/backend/internal/orchestrator/event_handlers_transient.go @@ -19,6 +19,8 @@ import ( // manual recovery banner. const transientMaxAttempts = 3 +const transientRetryStopTimeout = 30 * time.Second + // recoverActionCancelRetry is the session.recover action that stops an // in-progress transient retry loop and surfaces manual recovery. const recoverActionCancelRetry = "cancel_retry" @@ -225,7 +227,14 @@ func (s *Service) retryTransientPrompt(ctx context.Context, taskID, sessionID, e cp, _ := v.(capturedPrompt) if execID != "" { - if err := s.executor.StopExecution(ctx, execID, "transient retry: relaunching agent", true); err != nil { + if !s.claimForcedExecutionCleanup(sessionID, execID) { + s.logger.Debug("skipping transient retry because execution teardown is already owned", + zap.String("session_id", sessionID), + zap.String("execution_id", execID)) + s.resetTransientRetry(sessionID) + return + } + if err := s.stopTransientRetryExecution(ctx, execID); err != nil { s.logger.Debug("failed to stop failed execution before transient retry", zap.String("session_id", sessionID), zap.String("execution_id", execID), @@ -254,6 +263,12 @@ func (s *Service) retryTransientPrompt(ctx context.Context, taskID, sessionID, e } } +func (s *Service) stopTransientRetryExecution(ctx context.Context, executionID string) error { + stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), transientRetryStopTimeout) + defer cancel() + return s.executor.StopExecution(stopCtx, executionID, "transient retry: relaunching agent", true) +} + // createTransientRetryStatusMessage emits the calm yellow "retrying" status // (variant=warning) with a Cancel action, driving the frontend's // AgentWarningStatus instead of the red AgentErrorStatus. diff --git a/apps/backend/internal/orchestrator/event_handlers_transient_test.go b/apps/backend/internal/orchestrator/event_handlers_transient_test.go index 868f7bbc45..af0cf05ee1 100644 --- a/apps/backend/internal/orchestrator/event_handlers_transient_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_transient_test.go @@ -4,12 +4,15 @@ import ( "context" "errors" "strings" + "sync/atomic" "testing" "time" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" "github.com/kandev/kandev/internal/orchestrator/watcher" "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" ) // overloaded529 is the exact transient 529 Overloaded envelope the @@ -106,6 +109,58 @@ func TestTransientFailedExecutionToolUpdateDoesNotCreateMessage(t *testing.T) { } } +func TestHandleAgentFailed_CancelledSessionDisarmsRetryWithoutRecovery(t *testing.T) { + ctx := context.Background() + svc, mc := newTransientTestService(t) + if err := svc.repo.UpdateTaskSessionState( + ctx, + "s1", + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + ); err != nil { + t.Fatalf("cancel session: %v", err) + } + cancelled := make(chan struct{}, 1) + svc.transientRetries.Store("s1", &transientRetryEntry{ + attempt: 1, + cancel: func() { + cancelled <- struct{}{} + }, + }) + svc.rememberTurnPrompt("s1", "retry me", "", false, nil) + + svc.handleAgentFailed(ctx, watcher.AgentEventData{ + TaskID: "t1", + SessionID: "s1", + AgentExecutionID: "stale-execution", + ErrorMessage: overloaded529, + }) + + select { + case <-cancelled: + default: + t.Fatal("cancelled session did not disarm transient retry") + } + _, retryArmed := svc.transientRetries.Load("s1") + if retryArmed { + t.Fatal("cancelled session retained transient retry") + } + _, promptCached := svc.lastTurnPrompt.Load("s1") + if promptCached { + t.Fatal("cancelled session retained cached prompt") + } + session, err := svc.repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("get cancelled session: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Fatalf("session state = %q, want %q", session.State, models.TaskSessionStateCancelled) + } + if len(mc.sessionMessages) != 0 { + t.Fatalf("stale failure emitted %d retry or recovery messages", len(mc.sessionMessages)) + } +} + func TestHandleTransientFailure_NonTransientReturnsFalse(t *testing.T) { svc, mc := newTransientTestService(t) t.Cleanup(svc.cancelAllTransientRetries) @@ -274,6 +329,90 @@ func TestRetryTransientPrompt_SynchronousPromptErrorSurfacesRecovery(t *testing. } } +func TestRetryTransientPrompt_DoesNotStopOrRelaunchCoordinatorOwnedExecution(t *testing.T) { + svc, _ := newTransientTestService(t) + svc.rememberTurnPrompt("s1", "retry me", "", false, nil) + svc.transientRetries.Store("s1", &transientRetryEntry{attempt: 1, cancel: func() {}}) + svc.RegisterExecutionStopOwner("s1", "exec-owned", true) + + svc.retryTransientPrompt(context.Background(), "t1", "s1", "exec-owned") + + agentManager := svc.agentManager.(*mockAgentManager) + agentManager.mu.Lock() + stopCalls := append([]stopAgentCall(nil), agentManager.stopAgentWithReasonArgs...) + promptCalls := append([]promptCall(nil), agentManager.capturedPromptCalls...) + agentManager.mu.Unlock() + if len(stopCalls) != 0 { + t.Fatalf("transient recovery duplicated coordinator stop: %#v", stopCalls) + } + if len(promptCalls) != 0 { + t.Fatalf("transient recovery relaunched after coordinator stop: %#v", promptCalls) + } + if _, ok := svc.transientRetries.Load("s1"); ok { + t.Fatal("coordinator-owned teardown left transient retry armed") + } +} + +func TestRetryTransientPrompt_OwningStopSurvivesCoordinatorCancellation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-retry-race", "session-retry-race", models.TaskSessionStateWaitingForInput) + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task-retry-race", v1.TaskStateInProgress) + + stopEntered := make(chan struct{}) + releaseStop := make(chan struct{}) + stopContextCancelled := make(chan error, 1) + var stopCalls atomic.Int32 + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-retry-race", nil + }, + stopAgentWithReasonFunc: func(stopCtx context.Context, _ string, _ string, _ bool) error { + stopCalls.Add(1) + close(stopEntered) + select { + case <-releaseStop: + return nil + case <-stopCtx.Done(): + stopContextCancelled <- stopCtx.Err() + return stopCtx.Err() + } + }, + } + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + retryCtx, cancelRetry := context.WithCancel(ctx) + svc.rememberTurnPrompt("session-retry-race", "retry me", "", false, nil) + svc.transientRetries.Store("session-retry-race", &transientRetryEntry{ + attempt: 1, + cancel: cancelRetry, + }) + retryDone := make(chan struct{}) + go func() { + svc.retryTransientPrompt( + retryCtx, + "task-retry-race", + "session-retry-race", + "execution-retry-race", + ) + close(retryDone) + }() + + coordinatorStopAwaitSignal(t, stopEntered, "transient retry teardown") + result, err := svc.StopTaskForCoordinator(ctx, "task-retry-race") + require.NoError(t, err) + require.Equal(t, CoordinatorTaskStopStatusStopped, result.Status) + select { + case err := <-stopContextCancelled: + t.Fatalf("owning force-stop inherited retry cancellation: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(releaseStop) + coordinatorStopAwaitSignal(t, retryDone, "transient retry completion") + require.Equal(t, int32(1), stopCalls.Load()) + require.Empty(t, agentManager.capturedPromptCalls, "cancelled retry must not relaunch") +} + func TestTransientRetryEntryClaimPreventsDoubleFire(t *testing.T) { entry := &transientRetryEntry{} if !entry.claim() { diff --git a/apps/backend/internal/orchestrator/event_handlers_workflow.go b/apps/backend/internal/orchestrator/event_handlers_workflow.go index 9d8e69b066..ce19e9dec6 100644 --- a/apps/backend/internal/orchestrator/event_handlers_workflow.go +++ b/apps/backend/internal/orchestrator/event_handlers_workflow.go @@ -208,10 +208,21 @@ func (s *Service) processOnTurnStart(ctx context.Context, task *models.Task, ses // ProcessOnTurnStart is the public API for triggering on_turn_start events. // Called by message handlers before sending a prompt to the agent. func (s *Service) ProcessOnTurnStart(ctx context.Context, taskID, sessionID string) error { + lock, release := s.acquireCancelInFlightGuard(sessionID) + defer release() + lock.Lock() + defer lock.Unlock() + session, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil { return fmt.Errorf("load session for on_turn_start: %w", err) } + if isTerminalSessionState(session.State) { + return &executor.SessionStateSupersededError{ + SessionID: session.ID, + State: session.State, + } + } // ADR 0015 — a fresh user message before the pending signal's // transition has fired cancels the signal (re-open semantics). The // user is continuing the conversation; this step is no longer "done". @@ -303,8 +314,18 @@ func (s *Service) executeStepTransition(ctx context.Context, taskID, sessionID s s.clearPendingStepSignalByID(ctx, sessionID) // Automated transitions always clear review: the agent just completed // a turn, so any pending review from a prior step is stale regardless - // of whether the new step has auto_start_agent. - s.finalizeStepEnter(ctx, taskID, sessionID, targetStep, task.Description, true) + // of whether the new step has auto_start_agent. Match the engine path's + // asynchronous on_enter dispatch: terminal-event handlers own the + // session cancel guard through this transition, and inline auto-start + // would re-enter that non-reentrant guard from PromptTask. + go s.finalizeStepEnter( + context.WithoutCancel(ctx), + taskID, + sessionID, + targetStep, + task.Description, + true, + ) } else { // on_turn_start transitions: user is about to send a message, no on_enter needed. // However, we still need to switch the agent profile if the target step requires @@ -772,9 +793,10 @@ func (s *Service) createNewSessionForStep(ctx context.Context, taskID string, cu // completeAndStopSession stops the agent for a session and marks it COMPLETED. // Used by both the reuse path and the create-new path to terminate the -// previous session in a uniform way. IsPrimary is cleared explicitly so the -// full-row UpdateTaskSession write doesn't overwrite the SetPrimarySession -// call the caller just made on the replacement session. +// previous session in a uniform way. The state transition is the only session +// row write here: SetPrimarySession already cleared the old primary flag, and +// writing the caller's stale full row could resurrect a concurrently stopped +// session. func (s *Service) completeAndStopSession(ctx context.Context, taskID string, session *models.TaskSession) { // Flip state to COMPLETED *before* stopping the agent. StopAgent fires an // agent.completed event, and handleAgentCompleted's terminal-state guard @@ -793,17 +815,6 @@ func (s *Service) completeAndStopSession(ctx context.Context, taskID string, ses zap.Error(stopErr)) } } - - now := time.Now().UTC() - session.State = models.TaskSessionStateCompleted - session.IsPrimary = false - session.CompletedAt = &now - session.UpdatedAt = now - if err := s.repo.UpdateTaskSession(ctx, session); err != nil { - s.logger.Warn("failed to complete old session after switch", - zap.String("session_id", session.ID), - zap.Error(err)) - } } // maybySwitchSessionForProfile checks whether the step requires a different agent profile @@ -1378,8 +1389,17 @@ func (s *Service) autoStartStepPrompt( // the agent CLI's TTY and the user sees it verbatim. recordedPrompt := prompt if session.State == models.TaskSessionStateCreated && !session.IsPassthrough && (prompt != "" || len(attachments) > 0) && !sysprompt.HasKandevContext(prompt) { + isOfficeTask, err := s.lookupOfficeTask(ctx, taskID) + if err != nil { + requeueTaken() + return fmt.Errorf("resolve MCP mode for workflow auto-start: %w", err) + } + configMode, _ := session.Metadata["config_mode"].(bool) requiresSignal := step != nil && step.AutoAdvanceRequiresSignal - recordedPrompt = sysprompt.InjectKandevContext(taskID, sessionID, prompt, requiresSignal) + recordedPrompt = sysprompt.InjectKandevContextWithOptions(taskID, sessionID, prompt, sysprompt.KandevContextOptions{ + RequiresCompletionSignal: requiresSignal, + IncludeCoordinatorTaskControls: !isOfficeTask && !configMode, + }) } userMsgRecorded := s.recordAutoStartMessage(ctx, taskID, sessionID, recordedPrompt, planMode, origin) @@ -1485,28 +1505,16 @@ func (s *Service) fallbackFreshLaunchOnMissingExecution( } } - fresh, err := s.repo.GetTaskSession(ctx, sessionID) + // Keep coordinator stop outside the reset-to-CREATED / fresh-runtime + // registration window. If fallback wins, stop observes the newly + // registered execution after this guard is released; if stop won earlier, + // the guarded state CAS below sees CANCELLED and aborts the replacement. + cancelLock, releaseCancelLock := s.acquireCancelInFlightGuard(sessionID) + cancelLock.Lock() + fresh, err := s.resetSessionForFreshFallback(ctx, sessionID) + cancelLock.Unlock() + releaseCancelLock() if err != nil { - s.logger.Error("auto-start fallback: failed to load session", - zap.String("session_id", sessionID), zap.Error(err)) - requeue() - return err - } - - // Drop the executors_running row for this session: the next StartCreatedSession - // will go through the full LaunchAgent path which creates a fresh row via - // lifecycle.persistExecutorRunning. Pre-refactor this also cleared - // fresh.AgentExecutionID; that field no longer drives runtime decisions — - // the in-memory store + executors_running are the source of truth. - if delErr := s.repo.DeleteExecutorRunningBySessionID(ctx, sessionID); delErr != nil && !errors.Is(delErr, models.ErrExecutorRunningNotFound) { - s.logger.Warn("auto-start fallback: failed to clear executors_running for fresh launch", - zap.String("session_id", sessionID), zap.Error(delErr)) - } - fresh.State = models.TaskSessionStateCreated - fresh.UpdatedAt = time.Now().UTC() - if err := s.repo.UpdateTaskSession(ctx, fresh); err != nil { - s.logger.Error("auto-start fallback: failed to reset session for fresh launch", - zap.String("session_id", sessionID), zap.Error(err)) requeue() return err } @@ -1520,6 +1528,49 @@ func (s *Service) fallbackFreshLaunchOnMissingExecution( return nil } +func (s *Service) resetSessionForFreshFallback( + ctx context.Context, + sessionID string, +) (*models.TaskSession, error) { + s.taskRuntimeStateMu.Lock() + defer s.taskRuntimeStateMu.Unlock() + + fresh, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + s.logger.Error("auto-start fallback: failed to load session", + zap.String("session_id", sessionID), zap.Error(err)) + return nil, err + } + if fresh == nil { + return nil, fmt.Errorf("auto-start fallback: session %q is nil", sessionID) + } + if isTerminalSessionState(fresh.State) { + return nil, &executor.SessionStateSupersededError{ + SessionID: fresh.ID, + State: fresh.State, + } + } + + reset := *fresh + reset.State = models.TaskSessionStateCreated + reset.UpdatedAt = time.Now().UTC() + if err := s.persistFullTaskSessionIfCurrent(ctx, &reset, fresh.State); err != nil { + s.logger.Error("auto-start fallback: failed to reset session for fresh launch", + zap.String("session_id", sessionID), zap.Error(err)) + return nil, err + } + + // Drop the executors_running row only after the guarded state reset wins. + // The next StartCreatedSession takes the full LaunchAgent path and creates a + // fresh row via lifecycle.persistExecutorRunning. + if delErr := s.repo.DeleteExecutorRunningBySessionID(ctx, sessionID); delErr != nil && + !errors.Is(delErr, models.ErrExecutorRunningNotFound) { + s.logger.Warn("auto-start fallback: failed to clear executors_running for fresh launch", + zap.String("session_id", sessionID), zap.Error(delErr)) + } + return &reset, nil +} + // takeAndMergeHandoffMessage drains any queued hand-off message for the session // (set by handleMoveTask via move_task_kandev or by drainQueuedMessageForPromptableSession) // and merges its content + attachments into the auto-start prompt. Returns the diff --git a/apps/backend/internal/orchestrator/execution_teardown_ownership_test.go b/apps/backend/internal/orchestrator/execution_teardown_ownership_test.go new file mode 100644 index 0000000000..621bcdd5d6 --- /dev/null +++ b/apps/backend/internal/orchestrator/execution_teardown_ownership_test.go @@ -0,0 +1,199 @@ +package orchestrator + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + orchestratorexec "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + "github.com/stretchr/testify/require" +) + +func TestClaimForcedExecutionCleanup_UsesExactExecutionClaim(t *testing.T) { + svc := newCoordinatorStopTestService( + setupTestRepo(t), + newMockTaskRepo(), + &mockAgentManager{}, + ) + guard, release := svc.acquireCancelInFlightGuard("session-claim") + guard.Lock() + require.True(t, svc.claimExecutionTeardown( + "session-claim", + "execution-old", + executionTeardownIntentGraceful, + )) + guard.Unlock() + release() + + require.True(t, svc.claimForcedExecutionCleanup("session-claim", "execution-new")) + require.False(t, svc.claimForcedExecutionCleanup("session-claim", "execution-new")) + require.False(t, svc.claimForcedExecutionCleanup("session-claim", "execution-old")) +} + +func TestRegisterExecutionStopOwner_SuppressesOrphanCleanupAndRecordsForceEscalation(t *testing.T) { + svc := newCoordinatorStopTestService( + setupTestRepo(t), + newMockTaskRepo(), + &mockAgentManager{}, + ) + + svc.RegisterExecutionStopOwner("session-owner", "execution-owner", false) + require.False(t, svc.claimForcedExecutionCleanup("session-owner", "execution-owner")) + + svc.RegisterExecutionStopOwner("session-owner", "execution-owner", true) + value, ok := svc.executionTeardownClaims.Load( + terminalExecutionKey("session-owner", "execution-owner"), + ) + require.True(t, ok) + claim, ok := value.(executionTeardownClaim) + require.True(t, ok) + require.Equal(t, executionTeardownIntentForce, claim.intent) +} + +func TestCleanupAgentExecution_CancelledSessionUsesExactExecutionOwnership(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-cleanup-claim", "session-cleanup-claim", models.TaskSessionStateRunning) + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "session-cleanup-claim", + models.TaskSessionStateCancelled, + "task archived", + )) + + stopCalls := make(chan stopAgentCall, 2) + manager := &mockAgentManager{ + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + stopCalls <- stopAgentCall{ExecutionID: executionID, Reason: reason, Force: force} + return nil + }, + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), manager) + svc.RegisterExecutionStopOwner("session-cleanup-claim", "execution-snapshot", true) + + svc.cleanupAgentExecution("execution-snapshot", "task-cleanup-claim", "session-cleanup-claim") + svc.cleanupAgentExecution("execution-late", "task-cleanup-claim", "session-cleanup-claim") + + select { + case call := <-stopCalls: + require.Equal(t, "execution-late", call.ExecutionID) + require.True(t, call.Force) + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for unclaimed execution cleanup") + } + select { + case duplicate := <-stopCalls: + t.Fatalf("claimed snapshot execution was stopped twice: %#v", duplicate) + case <-time.After(100 * time.Millisecond): + } +} + +func TestReapPromptUnreadyExecution_StopsOnlyWhenRecoveryOwnsExecution(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-recovery-owner", "session-recovery-owner", models.TaskSessionStateWaitingForInput) + + var stopCalls atomic.Int32 + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-recovery-owner", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + stopCalls.Add(1) + return nil + }, + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), manager) + svc.RegisterExecutionStopOwner("session-recovery-owner", "execution-recovery-owner", true) + + err := svc.reapPromptUnreadyExecution( + ctx, + "session-recovery-owner", + errors.New("agent never became prompt-ready"), + ) + + require.Error(t, err) + require.Zero(t, stopCalls.Load(), "recovery must not duplicate coordinator-owned teardown") +} + +func TestReapPromptUnreadyExecution_DoesNotResumeAfterConcurrentCancellation(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-recovery-race", "session-recovery-race", models.TaskSessionStateWaitingForInput) + + stopEntered := make(chan struct{}) + releaseStop := make(chan struct{}) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-recovery-race", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + close(stopEntered) + <-releaseStop + return nil + }, + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), manager) + reapDone := make(chan error, 1) + go func() { + reapDone <- svc.reapPromptUnreadyExecution( + ctx, + "session-recovery-race", + errors.New("agent never became prompt-ready"), + ) + }() + + coordinatorStopAwaitSignal(t, stopEntered, "prompt recovery stop") + svc.RegisterExecutionStopOwner("session-recovery-race", "execution-recovery-race", true) + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + "session-recovery-race", + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + )) + close(releaseStop) + + err := <-reapDone + require.ErrorIs(t, err, orchestratorexec.ErrSessionStateSuperseded) +} + +func TestStopSession_GracefulTeardownClaimSuppressesLateForceCleanup(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task-legacy-stop", "session-legacy-stop", models.TaskSessionStateRunning) + + stopCalls := make(chan stopAgentCall, 2) + allowGracefulStop := make(chan struct{}) + gracefulStopDone := make(chan struct{}) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-legacy-stop", nil + }, + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + stopCalls <- stopAgentCall{ExecutionID: executionID, Reason: reason, Force: force} + if !force { + <-allowGracefulStop + close(gracefulStopDone) + } + return nil + }, + } + svc := newCoordinatorStopTestService(repo, newMockTaskRepo(), manager) + + require.NoError(t, svc.StopSession(ctx, "session-legacy-stop", "legacy graceful stop", false)) + first := <-stopCalls + require.Equal(t, "execution-legacy-stop", first.ExecutionID) + require.False(t, first.Force) + + svc.cleanupAgentExecution("execution-legacy-stop", "task-legacy-stop", "session-legacy-stop") + close(allowGracefulStop) + coordinatorStopAwaitSignal(t, gracefulStopDone, "legacy graceful teardown") + select { + case duplicate := <-stopCalls: + t.Fatalf("late terminal cleanup duplicated legacy teardown: %#v", duplicate) + case <-time.After(100 * time.Millisecond): + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor.go b/apps/backend/internal/orchestrator/executor/executor.go index 1b2ed98243..668bc079ca 100644 --- a/apps/backend/internal/orchestrator/executor/executor.go +++ b/apps/backend/internal/orchestrator/executor/executor.go @@ -4,6 +4,7 @@ package executor import ( "context" "errors" + "fmt" "sync" "time" @@ -46,6 +47,7 @@ type executorStore interface { GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) SetSessionMetadataKey(ctx context.Context, sessionID, key string, value interface{}) error UpdateTaskSession(ctx context.Context, session *models.TaskSession) error + UpdateTaskSessionIfCurrentState(ctx context.Context, session *models.TaskSession, expected models.TaskSessionState) (bool, error) UpdateTaskSessionState(ctx context.Context, id string, state models.TaskSessionState, errorMessage string) error UpdateTaskSessionBaseCommit(ctx context.Context, id string, baseCommitSHA string) error GetTaskSessionByTaskAndAgent(ctx context.Context, taskID, agentInstanceID string) (*models.TaskSession, error) @@ -91,8 +93,26 @@ var ( ErrTaskArchived = errors.New("task is archived") ErrStaleExecution = errors.New("stale execution: no live execution in memory") ErrAgentCommandMissing = errors.New("existing execution has no agent command configured") + // ErrSessionStateSuperseded means a runtime registered successfully, but a + // concurrent terminal session transition won the persistence race. Callers + // must not start the process and must arbitrate exact-execution teardown + // ownership before deciding whether to force-stop the registered runtime. + ErrSessionStateSuperseded = errors.New("session state superseded by terminal transition") ) +// SessionStateSupersededError records the terminal state that rejected a +// stale runtime persistence write. +type SessionStateSupersededError struct { + SessionID string + State models.TaskSessionState +} + +func (e *SessionStateSupersededError) Error() string { + return fmt.Sprintf("%s: session %s is %s", ErrSessionStateSuperseded, e.SessionID, e.State) +} + +func (e *SessionStateSupersededError) Unwrap() error { return ErrSessionStateSuperseded } + // PromptResult contains the result of a prompt operation type PromptResult struct { StopReason string // The reason the agent stopped (e.g., "end_turn") @@ -520,17 +540,37 @@ func agentSessionStateToV1(state models.TaskSessionState) v1.TaskSessionState { // publish events (e.g. WebSocket notifications) alongside the DB update. type TaskStateChangeFunc func(ctx context.Context, taskID string, state v1.TaskState) error +// TaskRuntimeStateReconcileFunc updates task state only while the originating +// session still owns an eligible runtime state. +type TaskRuntimeStateReconcileFunc func(ctx context.Context, taskID, sessionID string, state v1.TaskState) error + // SessionStateChangeFunc is called when the executor needs to update a session's state. // When set, it replaces direct repo.UpdateTaskSessionState calls so the caller can // publish events (e.g. WebSocket notifications) alongside the DB update. type SessionStateChangeFunc func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) error +// SessionStateTransitionFunc performs a guarded session-state transition and +// reports whether the requested state was accepted plus the final observed +// state. Coordinator stop uses this stricter callback so terminal races and +// persistence failures cannot be mistaken for accepted cancellation. +type SessionStateTransitionFunc func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) (changed bool, finalState models.TaskSessionState, err error) + // SessionStartingFunc is called when the executor has prepared/resumed an // execution and needs to mark the session STARTING while preserving other // session-row updates such as metadata. promoteTask controls whether the // callback should also move the parent task to IN_PROGRESS immediately. type SessionStartingFunc func(ctx context.Context, taskID string, session *models.TaskSession, promoteTask bool) error +// ExecutionCleanupClaimFunc atomically claims forced cleanup for one exact +// session execution. It returns true when the executor owns cleanup and false +// when another teardown path already owns that execution. +type ExecutionCleanupClaimFunc func(sessionID, agentExecutionID string) bool + +// ExecutionStopOwnerRegistrationFunc records that an explicit teardown path +// owns one exact session execution. Registration is advisory: the explicit +// stop still runs, while orphan cleanup uses the record to avoid duplicating it. +type ExecutionStopOwnerRegistrationFunc func(sessionID, agentExecutionID string, force bool) + // TaskReviewStateReconcileFunc is called when runtime work stopped and the // parent task should move to REVIEW only if no session is still STARTING/RUNNING. type TaskReviewStateReconcileFunc func(ctx context.Context, taskID, completedSessionID string) @@ -577,16 +617,32 @@ type Executor struct { // Set by the orchestrator to route through the task service layer. onTaskStateChange TaskStateChangeFunc + // Session-aware task state callback used after agent-process start settles. + onTaskRuntimeStateReconcile TaskRuntimeStateReconcileFunc + // Callback for session state changes that need event publishing. // Set by the orchestrator to route through updateTaskSessionState which // updates the DB and publishes WebSocket events. onSessionStateChange SessionStateChangeFunc + // Strict session-state callback used by operations that need to distinguish + // accepted writes from terminal/no-op races. + onSessionStateTransition SessionStateTransitionFunc + // Callback for STARTING writes that carry full session-row changes. Set by // the orchestrator so launch/resume/model-switch transitions serialize with // runtime task-state reconciliation. onSessionStarting SessionStartingFunc + // Callback for exact-execution forced cleanup arbitration. Set by the + // orchestrator so coordinator graceful stop and launch cleanup cannot both + // tear down the same execution. + onExecutionCleanupClaim ExecutionCleanupClaimFunc + + // Callback for registering explicit exact-execution teardown ownership before + // a legacy stop persists CANCELLED. The requested stop always runs. + onExecutionStopOwnerRegistration ExecutionStopOwnerRegistrationFunc + // Callback for REVIEW reconciliation after runtime start failures. Set by the // orchestrator so failed-start writes share the same serialized guard as // normal turn completion. @@ -677,6 +733,11 @@ func (e *Executor) SetOnTaskStateChange(fn TaskStateChangeFunc) { e.onTaskStateChange = fn } +// SetOnTaskRuntimeStateReconcile sets the session-aware runtime task-state callback. +func (e *Executor) SetOnTaskRuntimeStateReconcile(fn TaskRuntimeStateReconcileFunc) { + e.onTaskRuntimeStateReconcile = fn +} + // SetOnSessionStateChange sets a callback for session state changes. // This allows the orchestrator to route state changes through updateTaskSessionState // which updates the DB and publishes WebSocket events to the frontend. @@ -684,11 +745,27 @@ func (e *Executor) SetOnSessionStateChange(fn SessionStateChangeFunc) { e.onSessionStateChange = fn } +// SetOnSessionStateTransition sets the guarded session-state callback used by +// detailed lifecycle operations. +func (e *Executor) SetOnSessionStateTransition(fn SessionStateTransitionFunc) { + e.onSessionStateTransition = fn +} + // SetOnSessionStarting sets a callback for full session-row STARTING updates. func (e *Executor) SetOnSessionStarting(fn SessionStartingFunc) { e.onSessionStarting = fn } +// SetOnExecutionCleanupClaim sets the exact-execution forced cleanup arbiter. +func (e *Executor) SetOnExecutionCleanupClaim(fn ExecutionCleanupClaimFunc) { + e.onExecutionCleanupClaim = fn +} + +// SetOnExecutionStopOwnerRegistration sets the explicit-stop ownership registrar. +func (e *Executor) SetOnExecutionStopOwnerRegistration(fn ExecutionStopOwnerRegistrationFunc) { + e.onExecutionStopOwnerRegistration = fn +} + // SetOnTaskReviewStateReconcile sets the guarded task REVIEW reconciliation // callback used after resume/start failures. func (e *Executor) SetOnTaskReviewStateReconcile(fn TaskReviewStateReconcileFunc) { diff --git a/apps/backend/internal/orchestrator/executor/executor_execute.go b/apps/backend/internal/orchestrator/executor/executor_execute.go index 56fbfd0fff..0f49d55aa6 100644 --- a/apps/backend/internal/orchestrator/executor/executor_execute.go +++ b/apps/backend/internal/orchestrator/executor/executor_execute.go @@ -32,6 +32,23 @@ func isConfigModeSession(session *models.TaskSession) bool { return ok && cm } +// resolveTaskSessionMCPMode derives restricted MCP access from canonical task +// ownership and session purpose. Config mode wins because those sessions need +// config tools even if their backing task is Office-owned. +func (e *Executor) resolveTaskSessionMCPMode(ctx context.Context, taskID string, session *models.TaskSession) (string, error) { + if isConfigModeSession(session) { + return McpModeConfig, nil + } + task, err := e.repo.GetTask(ctx, taskID) + if err != nil { + return "", fmt.Errorf("load task for MCP mode: %w", err) + } + if task != nil && task.AssigneeAgentProfileID != "" { + return McpModeOffice, nil + } + return "", nil +} + // isContainerizedExecutor returns true for executor types that run agents in // containers or remote sandboxes (Docker variants + Sprites). These executors // need GitHub token injection for git operations since they don't have access @@ -71,37 +88,18 @@ func (e *Executor) runAgentProcessAsync(ctx context.Context, taskID, sessionID, updateCtx := context.WithoutCancel(ctx) if err := e.agentManager.StartAgentProcess(startCtx, agentExecutionID); err != nil { - e.logger.Error("failed to start agent process", - zap.String("task_id", taskID), - zap.String("session_id", sessionID), - zap.String("agent_execution_id", agentExecutionID), - zap.Error(err)) - // Let the orchestrator handle auth errors as recoverable failures - // and (for resume) suppress the toast before the session is marked FAILED. - if e.onAgentStartFailed != nil && e.onAgentStartFailed(updateCtx, taskID, sessionID, agentExecutionID, err, fromResume) { - return - } - if updateErr := e.updateSessionState(updateCtx, taskID, sessionID, models.TaskSessionStateFailed, err.Error()); updateErr != nil { - e.logger.Warn("failed to mark session as failed after start error", - zap.String("session_id", sessionID), - zap.Error(updateErr)) - } - if escalateTaskOnFailure { - if updateErr := e.updateTaskState(updateCtx, taskID, v1.TaskStateFailed); updateErr != nil { - e.logger.Warn("failed to mark task as failed after start error", - zap.String("task_id", taskID), - zap.Error(updateErr)) - } - } else { - e.writeTaskReviewStateIfNoWorkingSessions(updateCtx, taskID, sessionID) - } - // Clean up the execution environment (e.g., destroy remote Sprites instance). - // Use force=true since the agent process never fully started. - if stopErr := e.agentManager.StopAgent(updateCtx, agentExecutionID, true); stopErr != nil { - e.logger.Warn("failed to clean up agent after start failure", - zap.String("agent_execution_id", agentExecutionID), - zap.Error(stopErr)) - } + e.handleAgentProcessStartFailure( + updateCtx, taskID, sessionID, agentExecutionID, err, + escalateTaskOnFailure, fromResume, + ) + return + } + if _, terminal := e.stopStartedExecutionIfSessionTerminal( + updateCtx, + sessionID, + agentExecutionID, + "terminal post-start race", + ); terminal { return } @@ -109,12 +107,110 @@ func (e *Executor) runAgentProcessAsync(ctx context.Context, taskID, sessionID, }() } +func (e *Executor) handleAgentProcessStartFailure( + ctx context.Context, + taskID, sessionID, agentExecutionID string, + startErr error, + escalateTaskOnFailure, fromResume bool, +) { + e.logger.Error("failed to start agent process", + zap.String("task_id", taskID), + zap.String("session_id", sessionID), + zap.String("agent_execution_id", agentExecutionID), + zap.Error(startErr)) + + // A terminal transition may have landed while StartAgentProcess was + // blocked. Drop all failure/recovery side effects in that case. CANCELLED + // owns teardown only when another path has claimed this exact execution. + if terminalState, terminal := e.currentTerminalSessionState(ctx, sessionID); terminal { + if terminalState != models.TaskSessionStateCancelled || + e.claimForcedExecutionCleanup(sessionID, agentExecutionID) { + e.stopFailedStartExecution(ctx, agentExecutionID, "terminal start race") + } + return + } + + // Let the orchestrator handle auth errors as recoverable failures and + // (for resume) suppress the toast before the session is marked FAILED. + if e.onAgentStartFailed != nil && e.onAgentStartFailed( + ctx, taskID, sessionID, agentExecutionID, startErr, fromResume, + ) { + return + } + + changed, finalState, updateErr := e.transitionSessionState( + ctx, taskID, sessionID, models.TaskSessionStateFailed, startErr.Error(), + ) + if updateErr != nil { + e.logger.Warn("failed to mark session as failed after start error", + zap.String("session_id", sessionID), + zap.Error(updateErr)) + } + if changed && finalState == models.TaskSessionStateFailed && escalateTaskOnFailure { + if updateErr := e.writeTaskFailedForRuntime(ctx, taskID, sessionID); updateErr != nil { + e.logger.Warn("failed to mark task as failed after start error", + zap.String("task_id", taskID), + zap.Error(updateErr)) + } + } else if changed && finalState == models.TaskSessionStateFailed { + e.writeTaskReviewStateIfNoWorkingSessions(ctx, taskID, sessionID) + } + + // The agent process never fully started. Skip forced cleanup only when a + // concurrent path owns teardown for this exact cancelled execution. + if finalState != models.TaskSessionStateCancelled || + e.claimForcedExecutionCleanup(sessionID, agentExecutionID) { + e.stopFailedStartExecution(ctx, agentExecutionID, "start failure") + } +} + +func (e *Executor) claimForcedExecutionCleanup(sessionID, agentExecutionID string) bool { + if e.onExecutionCleanupClaim == nil { + return true + } + return e.onExecutionCleanupClaim(sessionID, agentExecutionID) +} + +func (e *Executor) stopFailedStartExecution(ctx context.Context, agentExecutionID, phase string) { + if stopErr := e.agentManager.StopAgent(ctx, agentExecutionID, true); stopErr != nil { + e.logger.Warn("failed to clean up agent after "+phase, + zap.String("agent_execution_id", agentExecutionID), + zap.Error(stopErr)) + } +} + +func (e *Executor) currentTerminalSessionState( + ctx context.Context, + sessionID string, +) (models.TaskSessionState, bool) { + session, err := e.repo.GetTaskSession(ctx, sessionID) + if err != nil || session == nil || !isStopTerminalSessionState(session.State) { + return "", false + } + return session.State, true +} + +func (e *Executor) stopStartedExecutionIfSessionTerminal( + ctx context.Context, + sessionID, agentExecutionID, phase string, +) (models.TaskSessionState, bool) { + terminalState, terminal := e.currentTerminalSessionState(ctx, sessionID) + if !terminal { + return "", false + } + if e.claimForcedExecutionCleanup(sessionID, agentExecutionID) { + e.stopFailedStartExecution(ctx, agentExecutionID, phase) + } + return terminalState, true +} + // startAgentProcessAsync starts the agent subprocess and transitions the task to IN_PROGRESS on success. func (e *Executor) startAgentProcessAsync(ctx context.Context, taskID, sessionID, agentExecutionID string) { e.runAgentProcessAsync(ctx, taskID, sessionID, agentExecutionID, func(updCtx context.Context) { - if updateErr := e.updateTaskState(updCtx, taskID, v1.TaskStateInProgress); updateErr != nil { + if updateErr := e.writeTaskInProgressForRuntime(updCtx, taskID, sessionID); updateErr != nil { e.logger.Warn("failed to update task state to IN_PROGRESS after agent start", zap.String("task_id", taskID), + zap.String("session_id", sessionID), zap.Error(updateErr)) } }, true, false) @@ -134,6 +230,20 @@ func (e *Executor) stopUnstartedExecution(ctx context.Context, sessionID, agentE } } +func (e *Executor) cleanupUnstartedExecutionAfterPersistError( + ctx context.Context, + sessionID, agentExecutionID string, + persistErr error, +) { + var superseded *SessionStateSupersededError + if errors.As(persistErr, &superseded) && + superseded.State == models.TaskSessionStateCancelled && + !e.claimForcedExecutionCleanup(sessionID, agentExecutionID) { + return + } + e.stopUnstartedExecution(ctx, sessionID, agentExecutionID) +} + func (e *Executor) writeTaskReviewStateIfNoWorkingSessions(ctx context.Context, taskID, failedSessionID string) { if e.onTaskReviewStateReconcile != nil { e.onTaskReviewStateReconcile(ctx, taskID, failedSessionID) @@ -270,6 +380,52 @@ func (e *Executor) updateSessionState(ctx context.Context, taskID, sessionID str return e.repo.UpdateTaskSessionState(ctx, sessionID, state, errorMessage) } +// transitionSessionState updates a session only when its freshly observed +// state is non-terminal and different from the requested state. It returns the +// authoritative final state so callers can tell an accepted write from an +// idempotent or naturally-terminal race. +func (e *Executor) transitionSessionState( + ctx context.Context, + taskID, sessionID string, + state models.TaskSessionState, + errorMessage string, +) (bool, models.TaskSessionState, error) { + if e.onSessionStateTransition != nil { + return e.onSessionStateTransition(ctx, taskID, sessionID, state, errorMessage) + } + + current, err := e.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return false, "", fmt.Errorf("get session before state transition: %w", err) + } + if current == nil { + return false, "", fmt.Errorf("get session before state transition: session %q is nil", sessionID) + } + if isStopTerminalSessionState(current.State) || current.State == state { + return false, current.State, nil + } + if err := e.updateSessionState(ctx, taskID, sessionID, state, errorMessage); err != nil { + return false, current.State, err + } + refreshed, err := e.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return false, "", fmt.Errorf("get session after state transition: %w", err) + } + if refreshed == nil { + return false, "", fmt.Errorf("get session after state transition: session %q is nil", sessionID) + } + if refreshed.State != state { + return false, refreshed.State, nil + } + return true, refreshed.State, nil +} + +func isStopTerminalSessionState(state models.TaskSessionState) bool { + return state == models.TaskSessionStateCompleted || + state == models.TaskSessionStateFailed || + state == models.TaskSessionStateCancelled +} + // updateSessionStarting persists a full session-row STARTING transition, using // the orchestrator callback when present so task/session runtime state stays // serialized with guarded REVIEW reconciliation. @@ -277,7 +433,50 @@ func (e *Executor) updateSessionStarting(ctx context.Context, taskID string, ses if e.onSessionStarting != nil { return e.onSessionStarting(ctx, taskID, session, promoteTask) } - return e.repo.UpdateTaskSession(ctx, session) + current, err := e.repo.GetTaskSession(ctx, session.ID) + if err != nil { + return err + } + if current == nil { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, session.ID) + } + allowedTerminalRecovery := !promoteTask && + session.State == models.TaskSessionStateStarting && + current.State == models.TaskSessionStateFailed + if isStopTerminalSessionState(current.State) && !allowedTerminalRecovery { + return &SessionStateSupersededError{SessionID: session.ID, State: current.State} + } + return e.persistSessionFullRowIfCurrentState(ctx, session, current.State) +} + +func (e *Executor) persistSessionFullRowIfCurrentState( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) error { + changed, err := e.repo.UpdateTaskSessionIfCurrentState(ctx, session, expected) + if err != nil { + return err + } + if changed { + return nil + } + current, err := e.repo.GetTaskSession(ctx, session.ID) + if err != nil { + return err + } + if current == nil { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, session.ID) + } + if isStopTerminalSessionState(current.State) { + return &SessionStateSupersededError{SessionID: session.ID, State: current.State} + } + return fmt.Errorf( + "session %s state changed from %s to %s before runtime persistence", + session.ID, + expected, + current.State, + ) } // shouldUseWorktree returns true if the given executor type should use Git worktrees. @@ -550,6 +749,12 @@ func (e *Executor) LaunchPreparedSession(ctx context.Context, task *v1.Task, ses if session.TaskID != task.ID { return nil, fmt.Errorf("session does not belong to task") } + if opts.McpMode == "" { + opts.McpMode, err = e.resolveTaskSessionMCPMode(ctx, task.ID, session) + if err != nil { + return nil, err + } + } running, _ := e.repo.GetExecutorRunningBySessionID(ctx, sessionID) if running != nil && running.ExecutionProfileID != "" && @@ -739,7 +944,7 @@ func (e *Executor) handleLaunchFailure(ctx context.Context, taskID, sessionID st func (e *Executor) finalizeLaunch(ctx context.Context, task *v1.Task, session *models.TaskSession, agentProfileID, sessionID string, repoInfo *repoInfo, resp *LaunchAgentResponse, startAgent bool, execCfg executorConfig) (*TaskExecution, error) { now := time.Now().UTC() if err := e.persistLaunchState(ctx, task.ID, sessionID, session, resp, startAgent, now); err != nil { - e.stopUnstartedExecution(ctx, sessionID, resp.AgentExecutionID) + e.cleanupUnstartedExecutionAfterPersistError(ctx, sessionID, resp.AgentExecutionID, err) return nil, err } e.persistWorktreeAssociation(ctx, task.ID, session, repoInfo.RepositoryID, resp) diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction.go b/apps/backend/internal/orchestrator/executor/executor_interaction.go index 7900194dbc..817e42d89c 100644 --- a/apps/backend/internal/orchestrator/executor/executor_interaction.go +++ b/apps/backend/internal/orchestrator/executor/executor_interaction.go @@ -9,6 +9,7 @@ import ( "time" "github.com/kandev/kandev/internal/agent/agents" + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" agentctlshared "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" "github.com/kandev/kandev/internal/task/models" @@ -25,46 +26,148 @@ func (e *Executor) Stop(ctx context.Context, sessionID string, reason string, fo return e.stopWithSession(ctx, session, reason, force) } -// stopWithSession stops an active execution using an already-loaded session. -// It marks the session CANCELLED in the database immediately so the WebSocket -// caller can return quickly, then terminates the agent process asynchronously -// to avoid blocking the caller on agentctl's blocking stop HTTP call. +// SessionStopResult describes the synchronous, logical portion of a stop. A +// true Changed value means CANCELLED was accepted and runtime teardown is ready +// to be scheduled. FinalState is empty when no live execution exists. +type SessionStopResult struct { + Changed bool + FinalState models.TaskSessionState + ExecutionID string + teardown func() +} + +// ScheduleTeardown starts the detached runtime teardown prepared by +// StopSessionDetailed. It is intentionally explicit so a coordinator can +// release its session-control locks before the teardown goroutine begins. +// Repeated calls are harmless. +func (r *SessionStopResult) ScheduleTeardown() bool { + if r == nil || r.teardown == nil { + return false + } + teardown := r.teardown + r.teardown = nil + teardown() + return true +} + +// StopSessionDetailed stops a live session while preserving lookup and +// persistence failures for callers that need a truthful structured result. +// A naturally absent or terminal session returns Changed=false with no error. +// Callers accepting Changed=true must invoke ScheduleTeardown after releasing +// any lifecycle-arbitration locks. +func (e *Executor) StopSessionDetailed( + ctx context.Context, + session *models.TaskSession, + reason string, + force bool, +) (SessionStopResult, error) { + if session == nil { + return SessionStopResult{}, errors.New("stop session: session is nil") + } + if session.ID == "" { + return SessionStopResult{}, errors.New("stop session: session ID is empty") + } + return e.stopSession(ctx, session, reason, force) +} + +// stopWithSession preserves the legacy error-only contract. It intentionally +// keeps session persistence best-effort and schedules teardown whenever a live +// execution was found; existing UI and cleanup callers rely on that behavior. func (e *Executor) stopWithSession(ctx context.Context, session *models.TaskSession, reason string, force bool) error { - // Look up the live execution for this session via the lifecycle manager — - // the in-memory store is the single source of truth post-refactor. executionID, err := e.agentManager.GetExecutionIDForSession(ctx, session.ID) if err != nil || executionID == "" { + if err != nil { + if errors.Is(err, lifecycle.ErrNoExecutionForSession) { + return fmt.Errorf("%w: %w: %w", ErrExecutionNotFound, runtimeapi.ErrNotFound, err) + } + return fmt.Errorf("%w: lookup execution for session %q: %w", ErrExecutionNotFound, session.ID, err) + } return ErrExecutionNotFound } + e.logStop(session, executionID, reason, force) + if e.onExecutionStopOwnerRegistration != nil { + e.onExecutionStopOwnerRegistration(session.ID, executionID, force) + } + if dbErr := e.updateSessionState(ctx, session.TaskID, session.ID, models.TaskSessionStateCancelled, reason); dbErr != nil { + e.logger.Error("failed to update agent session status", + zap.String("session_id", session.ID), + zap.Error(dbErr)) + } + e.scheduleStop(ctx, session.ID, executionID, reason, force) + return nil +} + +func (e *Executor) stopSession( + ctx context.Context, + session *models.TaskSession, + reason string, + force bool, +) (SessionStopResult, error) { + // Look up the live execution for this session via the lifecycle manager — + // the in-memory store is the single source of truth post-refactor. + executionID, err := e.agentManager.GetExecutionIDForSession(ctx, session.ID) + if err != nil { + if errors.Is(err, lifecycle.ErrNoExecutionForSession) { + return SessionStopResult{}, nil + } + return SessionStopResult{}, fmt.Errorf("lookup execution for session %q: %w", session.ID, err) + } + if executionID == "" { + return SessionStopResult{}, fmt.Errorf("%w for session %q", errEmptyExecutionID, session.ID) + } + + e.logStop(session, executionID, reason, force) + + changed, finalState, stateErr := e.transitionSessionState( + ctx, + session.TaskID, + session.ID, + models.TaskSessionStateCancelled, + reason, + ) + if stateErr != nil { + return SessionStopResult{FinalState: finalState}, fmt.Errorf("cancel session %q: %w", session.ID, stateErr) + } + result := SessionStopResult{ + Changed: changed, + FinalState: finalState, + ExecutionID: executionID, + } + if !changed { + return result, nil + } + + // Preparing rather than starting teardown here lets the coordinator release + // its per-session cancellation guard first. agentctl's stop endpoint blocks + // until the process exits, so the actual call still runs detached. + result.teardown = func() { + e.scheduleStop(ctx, session.ID, executionID, reason, force) + } + return result, nil +} + +var errEmptyExecutionID = errors.New("empty execution ID") + +func (e *Executor) logStop(session *models.TaskSession, executionID, reason string, force bool) { e.logger.Info("stopping execution", zap.String("task_id", session.TaskID), zap.String("session_id", session.ID), zap.String("agent_execution_id", executionID), zap.String("reason", reason), zap.Bool("force", force)) +} - // Mark the session CANCELLED and publish a WS event immediately so the - // frontend updates without waiting for the agent process to exit. - if dbErr := e.updateSessionState(ctx, session.TaskID, session.ID, models.TaskSessionStateCancelled, reason); dbErr != nil { - e.logger.Error("failed to update agent session status", - zap.String("session_id", session.ID), - zap.Error(dbErr)) - } - - // Terminate the agent process in the background — agentctl's stop endpoint - // blocks until the process exits, which can exceed the WS request timeout. +func (e *Executor) scheduleStop(ctx context.Context, sessionID, executionID, reason string, force bool) { stopCtx := context.WithoutCancel(ctx) go func() { if err := e.agentManager.StopAgentWithReason(stopCtx, executionID, reason, force); err != nil { // Log the error; the agent instance may already be gone e.logger.Warn("failed to stop agent (may already be stopped)", - zap.String("session_id", session.ID), + zap.String("session_id", sessionID), zap.Error(err)) } }() - - return nil } // StopExecution stops a running execution by execution ID. @@ -80,7 +183,10 @@ func (e *Executor) StopExecution(ctx context.Context, executionID string, reason e.logger.Warn("failed to stop agent by execution id", zap.String("agent_execution_id", executionID), zap.Error(err)) - return ErrExecutionNotFound + if errors.Is(err, lifecycle.ErrExecutionNotFound) { + return fmt.Errorf("%w: %w: %w", ErrExecutionNotFound, runtimeapi.ErrNotFound, err) + } + return fmt.Errorf("%w: stop execution %q: %w", ErrExecutionNotFound, executionID, err) } return nil } @@ -410,7 +516,7 @@ func (e *Executor) launchModelSwitchAgent(ctx context.Context, taskID, sessionID } if err := e.persistModelSwitchState(ctx, taskID, sessionID, session, newModel); err != nil { - e.stopUnstartedExecution(ctx, sessionID, resp.AgentExecutionID) + e.cleanupUnstartedExecutionAfterPersistError(ctx, sessionID, resp.AgentExecutionID, err) return err } @@ -421,6 +527,14 @@ func (e *Executor) launchModelSwitchAgent(ctx context.Context, taskID, sessionID zap.Error(err)) return fmt.Errorf("failed to start agent after model switch: %w", err) } + if terminalState, terminal := e.stopStartedExecutionIfSessionTerminal( + ctx, + sessionID, + resp.AgentExecutionID, + "terminal model-switch start race", + ); terminal { + return &SessionStateSupersededError{SessionID: sessionID, State: terminalState} + } e.logger.Info("model switch complete, agent started", zap.String("task_id", taskID), @@ -449,10 +563,11 @@ func (e *Executor) buildSwitchModelRequest(ctx context.Context, task *models.Tas TaskEnvironmentID: session.TaskEnvironmentID, } - // Activate config-mode MCP tools when config_mode is set in session metadata. - if isConfigModeSession(session) { - req.McpMode = McpModeConfig + mcpMode, err := e.resolveTaskSessionMCPMode(ctx, task.ID, session) + if err != nil { + return nil, err } + req.McpMode = mcpMode repositoryPath, err := e.applyRepositoryToSwitchRequest(ctx, req, session, execConfig) if err != nil { @@ -553,15 +668,29 @@ func (e *Executor) persistInPlaceModelSwitch(ctx context.Context, sessionID, new session.AgentProfileSnapshot = make(map[string]interface{}) } session.AgentProfileSnapshot["model"] = newModel - if err := e.repo.UpdateTaskSession(ctx, session); err != nil { + var persistErr error + if updater, ok := e.repo.(taskSessionAgentProfileSnapshotUpdater); ok { + persistErr = updater.UpdateTaskSessionAgentProfileSnapshot(ctx, sessionID, session.AgentProfileSnapshot) + } else { + persistErr = e.repo.UpdateTaskSession(ctx, session) + } + if persistErr != nil { e.logger.Warn("failed to persist in-place model switch", zap.String("session_id", sessionID), - zap.Error(err)) + zap.Error(persistErr)) return } e.persistRuntimeModelMetadata(ctx, sessionID, session, newModel) } +type taskSessionAgentProfileSnapshotUpdater interface { + UpdateTaskSessionAgentProfileSnapshot( + ctx context.Context, + sessionID string, + snapshot map[string]interface{}, + ) error +} + func (e *Executor) persistRuntimeModelMetadata(ctx context.Context, sessionID string, session *models.TaskSession, modelID string) { cfg, _ := models.LoadSessionRuntimeConfig(session.Metadata) cfg.Model = modelID diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction_test.go b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go new file mode 100644 index 0000000000..f6b7830f14 --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go @@ -0,0 +1,540 @@ +package executor + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/task/models" +) + +func TestStopSessionDetailed_DoesNotOverwriteTerminalRace(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-1"] = &models.TaskSession{ + ID: "session-1", + TaskID: "task-1", + State: models.TaskSessionStateCompleted, + } + stopCalls := make(chan struct{}, 1) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-1", nil + }, + stopAgentFunc: func(context.Context, string, bool) error { + stopCalls <- struct{}{} + return nil + }, + } + exec := newTestExecutor(t, agentManager, repo) + + staleRunningSession := &models.TaskSession{ + ID: "session-1", + TaskID: "task-1", + State: models.TaskSessionStateRunning, + } + result, err := exec.StopSessionDetailed(context.Background(), staleRunningSession, "test stop", false) + + if err != nil { + t.Errorf("stop error = %v", err) + } + if result.Changed || result.FinalState != models.TaskSessionStateCompleted { + t.Errorf("result = %#v, want unchanged COMPLETED", result) + } + if got := repo.sessions["session-1"].State; got != models.TaskSessionStateCompleted { + t.Errorf("session state = %q, want %q", got, models.TaskSessionStateCompleted) + } + select { + case <-stopCalls: + t.Error("terminal session runtime was stopped") + default: + } +} + +func TestStopSessionDetailed_ClassifiesExecutionLookup(t *testing.T) { + lookupFailure := errors.New("lifecycle store unavailable") + tests := []struct { + name string + execution string + lookupErr error + wantErr error + }{ + { + name: "exact absence sentinel is idempotent", + lookupErr: fmt.Errorf("wrapped: %w", lifecycle.ErrNoExecutionForSession), + }, + { + name: "other lookup failure is preserved", + lookupErr: lookupFailure, + wantErr: lookupFailure, + }, + { + name: "empty execution ID is an invariant failure", + wantErr: errEmptyExecutionID, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-1"] = &models.TaskSession{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + } + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return tt.execution, tt.lookupErr + }, + } + exec := newTestExecutor(t, manager, repo) + + result, err := exec.StopSessionDetailed( + context.Background(), repo.sessions["session-1"], "test stop", false, + ) + + if tt.wantErr == errEmptyExecutionID { + if err == nil || !errors.Is(err, errEmptyExecutionID) { + t.Fatalf("error = %v, want empty execution ID error", err) + } + } else if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if result.Changed { + t.Error("lookup-only outcome reported a changed session") + } + }) + } +} + +func TestStopSessionDetailed_RejectsInvalidSession(t *testing.T) { + exec := newTestExecutor(t, &mockAgentManager{}, newMockRepository()) + + for _, session := range []*models.TaskSession{nil, &models.TaskSession{}} { + result, err := exec.StopSessionDetailed(context.Background(), session, "test stop", false) + if err == nil { + t.Errorf("session %#v: expected validation error", session) + } + if result.Changed { + t.Errorf("session %#v: invalid input reported a change", session) + } + } +} + +func TestStopSessionDetailed_PersistenceFailurePreventsTeardown(t *testing.T) { + writeFailure := errors.New("database is read-only") + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + } + repo.sessions[session.ID] = session + repo.updateTaskSessionStateFunc = func(context.Context, string, models.TaskSessionState, string) error { + return writeFailure + } + stopCalls := make(chan struct{}, 1) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-1", nil + }, + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + stopCalls <- struct{}{} + return nil + }, + } + exec := newTestExecutor(t, manager, repo) + + result, err := exec.StopSessionDetailed(context.Background(), session, "test stop", false) + + if !errors.Is(err, writeFailure) { + t.Fatalf("error = %v, want %v", err, writeFailure) + } + if result.Changed { + t.Error("failed persistence reported a changed session") + } + select { + case <-stopCalls: + t.Error("runtime teardown ran after persistence failure") + default: + } +} + +func TestStopSessionDetailed_PersistsBeforeDetachedTeardown(t *testing.T) { + type stopCall struct { + executionID string + reason string + force bool + state models.TaskSessionState + contextErr error + } + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + } + repo.sessions[session.ID] = session + stopCalls := make(chan stopCall, 1) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-1", nil + }, + stopAgentWithReasonFunc: func(ctx context.Context, executionID, reason string, force bool) error { + stored, _ := repo.GetTaskSession(context.Background(), session.ID) + stopCalls <- stopCall{ + executionID: executionID, + reason: reason, + force: force, + state: stored.State, + contextErr: ctx.Err(), + } + return nil + }, + } + exec := newTestExecutor(t, manager, repo) + ctx, cancel := context.WithCancel(context.Background()) + + result, err := exec.StopSessionDetailed(ctx, session, "coordinator stop", false) + cancel() + + if err != nil { + t.Fatalf("StopSessionDetailed: %v", err) + } + if !result.Changed || result.FinalState != models.TaskSessionStateCancelled { + t.Fatalf("result = %#v, want accepted CANCELLED", result) + } + select { + case <-stopCalls: + t.Fatal("teardown started before caller explicitly scheduled it") + default: + } + if !result.ScheduleTeardown() { + t.Fatal("accepted stop did not expose teardown scheduling") + } + if result.ScheduleTeardown() { + t.Fatal("teardown was scheduled more than once") + } + select { + case call := <-stopCalls: + if call.executionID != "execution-1" || call.reason != "coordinator stop" || call.force { + t.Errorf("stop call = %#v", call) + } + if call.state != models.TaskSessionStateCancelled { + t.Errorf("state at teardown = %q, want CANCELLED", call.state) + } + if call.contextErr != nil { + t.Errorf("detached teardown context error = %v", call.contextErr) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for asynchronous teardown") + } +} + +func TestStopWithSession_PreservesLegacyBestEffortPersistence(t *testing.T) { + writeFailure := errors.New("database is read-only") + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + } + repo.sessions[session.ID] = session + repo.updateTaskSessionStateFunc = func(context.Context, string, models.TaskSessionState, string) error { + return writeFailure + } + stopCalls := make(chan struct{}, 1) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-1", nil + }, + stopAgentWithReasonFunc: func(_ context.Context, executionID, reason string, force bool) error { + if executionID != "execution-1" || reason != "legacy reason" || !force { + t.Errorf("legacy stop arguments = %q, %q, %v", executionID, reason, force) + } + stopCalls <- struct{}{} + return nil + }, + } + exec := newTestExecutor(t, manager, repo) + + if err := exec.stopWithSession(context.Background(), session, "legacy reason", true); err != nil { + t.Fatalf("legacy stop returned persistence error: %v", err) + } + select { + case <-stopCalls: + case <-time.After(time.Second): + t.Fatal("timed out waiting for legacy teardown") + } +} + +func TestStopWithSession_PreservesLegacyLookupFailureClassification(t *testing.T) { + lookupFailure := errors.New("lifecycle store unavailable") + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + } + repo.sessions[session.ID] = session + exec := newTestExecutor(t, &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "", lookupFailure + }, + }, repo) + + err := exec.stopWithSession(context.Background(), session, "legacy reason", true) + if !errors.Is(err, lookupFailure) { + t.Fatalf("error = %v, want lookup failure", err) + } + if !errors.Is(err, ErrExecutionNotFound) { + t.Fatalf("error = %v, want legacy ErrExecutionNotFound", err) + } + if errors.Is(err, runtimeapi.ErrNotFound) { + t.Fatalf("error = %v, real lookup failure must not look like exact runtime absence", err) + } +} + +func TestStopWithSession_ClaimSuppressesLateCancelledLaunchCleanup(t *testing.T) { + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-legacy-race", TaskID: "task-legacy-race", State: models.TaskSessionStateRunning, + } + repo.sessions[session.ID] = session + stopCalls := make(chan bool, 2) + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-legacy-race", nil + }, + stopAgentWithReasonFunc: func(_ context.Context, _ string, _ string, force bool) error { + stopCalls <- force + return nil + }, + } + exec := newTestExecutor(t, manager, repo) + var claimed bool + exec.SetOnExecutionStopOwnerRegistration(func(sessionID, executionID string, force bool) { + if sessionID != session.ID || executionID != "execution-legacy-race" || force { + t.Fatalf("stop claim = (%q, %q, %v)", sessionID, executionID, force) + } + claimed = true + }) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + if sessionID != session.ID || executionID != "execution-legacy-race" { + t.Fatalf("cleanup claim = (%q, %q)", sessionID, executionID) + } + return !claimed + }) + + if err := exec.stopWithSession(context.Background(), session, "legacy graceful stop", false); err != nil { + t.Fatalf("stopWithSession: %v", err) + } + exec.handleAgentProcessStartFailure( + context.Background(), session.TaskID, session.ID, "execution-legacy-race", + errors.New("start failed after legacy cancellation"), true, false, + ) + + select { + case force := <-stopCalls: + if force { + t.Fatal("legacy owner was replaced by forced launch cleanup") + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for legacy graceful stop") + } + select { + case force := <-stopCalls: + t.Fatalf("duplicate teardown reached runtime (force=%v)", force) + case <-time.After(100 * time.Millisecond): + } +} + +func TestStopExecution_PreservesRuntimeFailureClassification(t *testing.T) { + runtimeFailure := errors.New("runtime teardown failed") + tests := []struct { + name string + stopErr error + wantErr error + }{ + { + name: "exact absence remains classifiable", + stopErr: fmt.Errorf("missing: %w", lifecycle.ErrExecutionNotFound), + wantErr: lifecycle.ErrExecutionNotFound, + }, + { + name: "real runtime failure is preserved", + stopErr: runtimeFailure, + wantErr: runtimeFailure, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentWithReasonFunc: func(context.Context, string, string, bool) error { + return tt.stopErr + }, + }, newMockRepository()) + + err := exec.StopExecution(context.Background(), "execution-1", "cleanup", true) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("StopExecution error = %v, want %v", err, tt.wantErr) + } + if !errors.Is(err, ErrExecutionNotFound) { + t.Fatalf("StopExecution error = %v, want legacy ErrExecutionNotFound", err) + } + if errors.Is(tt.stopErr, lifecycle.ErrExecutionNotFound) && + !errors.Is(err, runtimeapi.ErrNotFound) { + t.Fatalf("StopExecution error = %v, want public runtime ErrNotFound", err) + } + if !errors.Is(tt.stopErr, lifecycle.ErrExecutionNotFound) && + errors.Is(err, runtimeapi.ErrNotFound) { + t.Fatalf("StopExecution error = %v, real failure must not look like exact runtime absence", err) + } + }) + } +} + +func TestLaunchModelSwitchAgent_CleansStartedExecutionAfterTerminalRace(t *testing.T) { + ctx := context.Background() + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-model-race", TaskID: "task-model-race", State: models.TaskSessionStateWaitingForInput, + } + repo.sessions[session.ID] = session + stopCalls := make(chan struct{}, 1) + manager := &mockAgentManager{ + launchAgentFunc: func(context.Context, *LaunchAgentRequest) (*LaunchAgentResponse, error) { + return &LaunchAgentResponse{AgentExecutionID: "execution-model-race"}, nil + }, + startAgentProcessFunc: func(context.Context, string) error { + return repo.UpdateTaskSessionState( + ctx, + session.ID, + models.TaskSessionStateCancelled, + "stopped during model switch", + ) + }, + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "execution-model-race" || !force { + t.Fatalf("StopAgent = (%q, %v), want (execution-model-race, true)", executionID, force) + } + stopCalls <- struct{}{} + return nil + }, + } + exec := newTestExecutor(t, manager, repo) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + return sessionID == session.ID && executionID == "execution-model-race" + }) + + err := exec.launchModelSwitchAgent( + ctx, + session.TaskID, + session.ID, + "new-model", + session, + &LaunchAgentRequest{}, + nil, + ) + + if !errors.Is(err, ErrSessionStateSuperseded) { + t.Fatalf("launchModelSwitchAgent error = %v, want ErrSessionStateSuperseded", err) + } + select { + case <-stopCalls: + case <-time.After(time.Second): + t.Fatal("timed out waiting for model-switch race cleanup") + } +} + +func TestPersistInPlaceModelSwitch_DoesNotRestoreStaleActiveStateAfterCancellation(t *testing.T) { + repo := newMockRepository() + session := &models.TaskSession{ + ID: "session-model-race", + TaskID: "task-model-race", + State: models.TaskSessionStateRunning, + AgentProfileSnapshot: map[string]interface{}{"model": "old-model"}, + Metadata: map[string]interface{}{}, + } + repo.sessions[session.ID] = session + repo.getTaskSessionFunc = func(context.Context, string) (*models.TaskSession, error) { + stale := *session + stale.AgentProfileSnapshot = map[string]interface{}{"model": "old-model"} + stale.Metadata = map[string]interface{}{} + + repo.mu.Lock() + cancelled := *session + cancelled.State = models.TaskSessionStateCancelled + cancelled.ErrorMessage = "stopped by parent task via MCP" + repo.sessions[session.ID] = &cancelled + repo.mu.Unlock() + return &stale, nil + } + exec := newTestExecutor(t, &mockAgentManager{}, repo) + + exec.persistInPlaceModelSwitch(context.Background(), session.ID, "new-model") + + repo.mu.Lock() + stored := repo.sessions[session.ID] + repo.mu.Unlock() + if stored.State != models.TaskSessionStateCancelled { + t.Fatalf("session state = %q, want CANCELLED", stored.State) + } + if stored.ErrorMessage != "stopped by parent task via MCP" { + t.Fatalf("error message = %q, want coordinator stop reason", stored.ErrorMessage) + } + if got, _ := stored.AgentProfileSnapshot["model"].(string); got != "new-model" { + t.Fatalf("snapshot model = %q, want new-model", got) + } +} + +func TestSwitchModelFallback_PreservesRestrictedMCPMode(t *testing.T) { + tests := []struct { + name string + metadata map[string]interface{} + wantMode string + }{ + {name: "Office task", wantMode: McpModeOffice}, + { + name: "Config session takes precedence for Office task", + metadata: map[string]interface{}{"config_mode": true}, + wantMode: McpModeConfig, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newMockRepository() + repo.tasks["task-office"] = &models.Task{ + ID: "task-office", + WorkspaceID: "workspace-1", + Title: "Office task", + AssigneeAgentProfileID: "office-agent", + } + repo.sessions["session-office"] = &models.TaskSession{ + ID: "session-office", + TaskID: "task-office", + AgentProfileID: "office-agent", + State: models.TaskSessionStateRunning, + Metadata: tt.metadata, + } + + var capturedReq *LaunchAgentRequest + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "execution-old", nil + }, + launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) { + capturedReq = req + return &LaunchAgentResponse{AgentExecutionID: "execution-new"}, nil + }, + } + exec := newTestExecutor(t, manager, repo) + + if _, err := exec.SwitchModel( + context.Background(), "task-office", "session-office", "new-model", "continue", + ); err != nil { + t.Fatalf("SwitchModel: %v", err) + } + if capturedReq == nil { + t.Fatal("fallback did not call LaunchAgent") + } + if capturedReq.McpMode != tt.wantMode { + t.Fatalf("McpMode = %q, want %q", capturedReq.McpMode, tt.wantMode) + } + }) + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index e976003054..86fae1c3ba 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -20,6 +20,7 @@ type mockAgentManager struct { launchAgentFunc func(ctx context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) startAgentProcessFunc func(ctx context.Context, agentExecutionID string) error stopAgentFunc func(ctx context.Context, agentExecutionID string, force bool) error + stopAgentWithReasonFunc func(ctx context.Context, agentExecutionID string, reason string, force bool) error resolveAgentProfileFunc func(ctx context.Context, profileID string) (*AgentProfileInfo, error) setExecutionDescriptionFunc func(ctx context.Context, agentExecutionID string, description string) error getExecutionIDForSessionFunc func(ctx context.Context, sessionID string) (string, error) @@ -94,6 +95,9 @@ func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID strin } func (m *mockAgentManager) StopAgentWithReason(ctx context.Context, agentExecutionID string, reason string, force bool) error { + if m.stopAgentWithReasonFunc != nil { + return m.stopAgentWithReasonFunc(ctx, agentExecutionID, reason, force) + } return m.StopAgent(ctx, agentExecutionID, force) } @@ -256,8 +260,10 @@ type mockRepository struct { // Optional hook to inject behavior into GetTaskSession (e.g. simulate a // transient DB error); if nil, the default map lookup is used. - getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error) - createTaskSessionFunc func(ctx context.Context, session *models.TaskSession) error + getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error) + createTaskSessionFunc func(ctx context.Context, session *models.TaskSession) error + updateTaskSessionStateFunc func(ctx context.Context, sessionID string, state models.TaskSessionState, errorMessage string) error + listActiveTaskSessionsByTaskIDFunc func(ctx context.Context, taskID string) ([]*models.TaskSession, error) // Optional hook invoked at the top of UpdateTaskStateIfCurrentIn, before // it reads task state/archived_at. Lets tests simulate the exact TOCTOU // window this CAS closes: an earlier (non-transactional) archived-state @@ -378,6 +384,40 @@ func (m *mockRepository) UpdateTaskSession(ctx context.Context, session *models. return nil } +func (m *mockRepository) UpdateTaskSessionIfCurrentState( + _ context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + current, ok := m.sessions[session.ID] + if !ok { + return false, models.ErrTaskSessionNotFound + } + if current.State != expected { + return false, nil + } + m.updateTaskSessionCalls = append(m.updateTaskSessionCalls, session) + m.sessions[session.ID] = session + return true, nil +} + +func (m *mockRepository) UpdateTaskSessionAgentProfileSnapshot( + _ context.Context, + sessionID string, + snapshot map[string]interface{}, +) error { + m.mu.Lock() + defer m.mu.Unlock() + session, ok := m.sessions[sessionID] + if !ok { + return fmt.Errorf("session not found: %s", sessionID) + } + session.AgentProfileSnapshot = snapshot + return nil +} + func (m *mockRepository) SetSessionPrimary(ctx context.Context, sessionID string) error { m.mu.Lock() defer m.mu.Unlock() @@ -386,6 +426,12 @@ func (m *mockRepository) SetSessionPrimary(ctx context.Context, sessionID string } func (m *mockRepository) UpdateTaskSessionState(ctx context.Context, sessionID string, state models.TaskSessionState, errorMessage string) error { + m.mu.Lock() + fn := m.updateTaskSessionStateFunc + m.mu.Unlock() + if fn != nil { + return fn(ctx, sessionID, state, errorMessage) + } m.mu.Lock() defer m.mu.Unlock() if session, ok := m.sessions[sessionID]; ok { @@ -694,6 +740,12 @@ func (m *mockRepository) ListActiveTaskSessions(ctx context.Context) ([]*models. return nil, nil } func (m *mockRepository) ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error) { + m.mu.Lock() + fn := m.listActiveTaskSessionsByTaskIDFunc + m.mu.Unlock() + if fn != nil { + return fn(ctx, taskID) + } return nil, nil } func (m *mockRepository) HasActiveTaskSessionsByAgentProfile(ctx context.Context, agentProfileID string) (bool, error) { diff --git a/apps/backend/internal/orchestrator/executor/executor_office.go b/apps/backend/internal/orchestrator/executor/executor_office.go index 9b825d9433..e1e48f67c8 100644 --- a/apps/backend/internal/orchestrator/executor/executor_office.go +++ b/apps/backend/internal/orchestrator/executor/executor_office.go @@ -77,13 +77,37 @@ func (e *Executor) rebindOfficeSessionExecutionProfile( return nil } snapshot, isPassthrough := e.resolveAgentProfileSnapshot(ctx, executionProfileID) - session.ExecutionProfileID = executionProfileID - session.AgentProfileSnapshot = snapshot - session.IsPassthrough = isPassthrough - if err := e.repo.UpdateTaskSession(ctx, session); err != nil { - return fmt.Errorf("update office execution profile: %w", err) + for { + if isStopTerminalSessionState(session.State) { + return nil + } + + expectedState := session.State + updated := *session + updated.ExecutionProfileID = executionProfileID + updated.AgentProfileSnapshot = snapshot + updated.IsPassthrough = isPassthrough + changed, err := e.repo.UpdateTaskSessionIfCurrentState(ctx, &updated, expectedState) + if err != nil { + return fmt.Errorf("update office execution profile: %w", err) + } + if changed { + *session = updated + return nil + } + + fresh, err := e.repo.GetTaskSession(ctx, session.ID) + if err != nil { + return fmt.Errorf("reload office session after execution profile race: %w", err) + } + if fresh == nil { + return fmt.Errorf("reload office session after execution profile race: %w", models.ErrTaskSessionNotFound) + } + *session = *fresh + if session.ExecutionProfileID == executionProfileID || isStopTerminalSessionState(session.State) { + return nil + } } - return nil } // reuseDecision describes what tryReuseExistingSession did with an existing diff --git a/apps/backend/internal/orchestrator/executor/executor_office_test.go b/apps/backend/internal/orchestrator/executor/executor_office_test.go index 51c3d393c7..07b481ad5a 100644 --- a/apps/backend/internal/orchestrator/executor/executor_office_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_office_test.go @@ -20,6 +20,24 @@ func officeTestTask() *v1.Task { } } +type officeRebindRaceRepository struct { + *mockRepository + beforeGuardedUpdate func() +} + +func (r *officeRebindRaceRepository) UpdateTaskSessionIfCurrentState( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) (bool, error) { + if r.beforeGuardedUpdate != nil { + hook := r.beforeGuardedUpdate + r.beforeGuardedUpdate = nil + hook() + } + return r.mockRepository.UpdateTaskSessionIfCurrentState(ctx, session, expected) +} + func TestEnsureSessionForAgent_CreatesWhenMissing(t *testing.T) { repo := newMockRepository() exec := newTestExecutor(t, &mockAgentManager{}, repo) @@ -74,6 +92,53 @@ func TestEnsureSessionForAgent_RebindsExecutionProfileOnReuse(t *testing.T) { } } +func TestEnsureSessionForAgent_RebindDoesNotOverwriteConcurrentCancellation(t *testing.T) { + baseRepo := newMockRepository() + existing := &models.TaskSession{ + ID: "sess-existing", + TaskID: "task-office", + AgentProfileID: "agent-1", + ExecutionProfileID: "codex-profile", + State: models.TaskSessionStateIdle, + StartedAt: time.Now().UTC(), + } + baseRepo.sessions[existing.ID] = existing + raceRepo := &officeRebindRaceRepository{mockRepository: baseRepo} + raceRepo.beforeGuardedUpdate = func() { + baseRepo.mu.Lock() + cancelled := *baseRepo.sessions[existing.ID] + cancelled.State = models.TaskSessionStateCancelled + cancelled.ErrorMessage = "stopped by parent task via MCP" + baseRepo.sessions[existing.ID] = &cancelled + baseRepo.mu.Unlock() + } + exec := newTestExecutor(t, &mockAgentManager{}, baseRepo) + exec.repo = raceRepo + + got, err := exec.EnsureSessionForAgent( + context.Background(), officeTestTask(), "agent-1", "claude-profile", "exec-1", "", + ) + if err != nil { + t.Fatalf("EnsureSessionForAgent: %v", err) + } + if got.ID == existing.ID { + t.Fatalf("cancelled session %q was reused", existing.ID) + } + + baseRepo.mu.Lock() + stored := baseRepo.sessions[existing.ID] + baseRepo.mu.Unlock() + if stored.State != models.TaskSessionStateCancelled { + t.Fatalf("existing session state = %q, want CANCELLED", stored.State) + } + if stored.ErrorMessage != "stopped by parent task via MCP" { + t.Fatalf("existing session error = %q, want coordinator stop reason", stored.ErrorMessage) + } + if stored.ExecutionProfileID != "codex-profile" { + t.Fatalf("cancelled session profile = %q, want original profile", stored.ExecutionProfileID) + } +} + func TestEnsureSessionForAgent_RebindsExecutionProfileAfterCreateRace(t *testing.T) { repo := newMockRepository() exec := newTestExecutor(t, &mockAgentManager{}, repo) diff --git a/apps/backend/internal/orchestrator/executor/executor_resume.go b/apps/backend/internal/orchestrator/executor/executor_resume.go index df0257fcdf..74b742df54 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume.go @@ -233,6 +233,7 @@ func (e *Executor) backfillRepoDefaultBranch(ctx context.Context, repo *models.R // metadata merge, both of which are session-row concerns the lifecycle manager // doesn't know about. func (e *Executor) persistLaunchState(ctx context.Context, taskID, sessionID string, session *models.TaskSession, resp *LaunchAgentResponse, startAgent bool, now time.Time) error { + expectedState := session.State if startAgent { session.State = models.TaskSessionStateStarting } @@ -253,7 +254,7 @@ func (e *Executor) persistLaunchState(ctx context.Context, taskID, sessionID str if startAgent { updateErr = e.updateSessionStarting(ctx, taskID, session, true) } else { - updateErr = e.repo.UpdateTaskSession(ctx, session) + updateErr = e.persistSessionFullRowIfCurrentState(ctx, session, expectedState) } if updateErr != nil { e.logger.Error("failed to update agent session after launch", @@ -400,7 +401,7 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio } if err := e.persistResumeState(ctx, task.ID, session, startAgent); err != nil { - e.stopUnstartedExecution(ctx, session.ID, resp.AgentExecutionID) + e.cleanupUnstartedExecutionAfterPersistError(ctx, session.ID, resp.AgentExecutionID, err) return nil, err } e.persistWorktreeAssociation(ctx, task.ID, session, repositoryID, resp) @@ -569,9 +570,9 @@ func (e *Executor) buildResumeRequest(ctx context.Context, task *v1.Task, sessio e.reuseExistingEnvironment(ctx, req, existingEnv) - // Activate config-mode MCP tools when config_mode is set in session metadata. - if isConfigModeSession(session) { - req.McpMode = McpModeConfig + req.McpMode, err = e.resolveTaskSessionMCPMode(ctx, task.ID, session) + if err != nil { + return nil, "", execConfig, nil, nil, err } existingRunning := e.applyRunningRecordToResumeRequest(ctx, req, task, session, startAgent) @@ -851,6 +852,7 @@ func resolveResumeTaskDirName(existingEnv *models.TaskEnvironment, task *v1.Task // orchestrator's only remaining responsibility is the session-row state // machine (STARTING / CompletedAt-clear). func (e *Executor) persistResumeState(ctx context.Context, taskID string, session *models.TaskSession, startAgent bool) error { + expectedState := session.State session.ErrorMessage = "" if startAgent { session.State = models.TaskSessionStateStarting @@ -861,7 +863,7 @@ func (e *Executor) persistResumeState(ctx context.Context, taskID string, sessio if startAgent { updateErr = e.updateSessionStarting(ctx, taskID, session, false) } else { - updateErr = e.repo.UpdateTaskSession(ctx, session) + updateErr = e.persistSessionFullRowIfCurrentState(ctx, session, expectedState) } if updateErr != nil { e.logger.Error("failed to update task session for resume", @@ -921,6 +923,9 @@ func (e *Executor) startAgentProcessOnResume(ctx context.Context, taskID string, } func (e *Executor) writeTaskInProgressForRuntime(ctx context.Context, taskID, sessionID string) error { + if e.onTaskRuntimeStateReconcile != nil { + return e.onTaskRuntimeStateReconcile(ctx, taskID, sessionID, v1.TaskStateInProgress) + } task, err := e.repo.GetTask(ctx, taskID) if err != nil { return err @@ -954,3 +959,17 @@ func (e *Executor) writeTaskInProgressForRuntime(ctx context.Context, taskID, se } return e.updateTaskState(ctx, taskID, v1.TaskStateInProgress) } + +func (e *Executor) writeTaskFailedForRuntime(ctx context.Context, taskID, sessionID string) error { + if e.onTaskRuntimeStateReconcile != nil { + return e.onTaskRuntimeStateReconcile(ctx, taskID, sessionID, v1.TaskStateFailed) + } + session, err := e.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if session == nil || session.State != models.TaskSessionStateFailed { + return nil + } + return e.updateTaskState(ctx, taskID, v1.TaskStateFailed) +} diff --git a/apps/backend/internal/orchestrator/executor/executor_resume_test.go b/apps/backend/internal/orchestrator/executor/executor_resume_test.go index f54c6bcf10..a3daae8501 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume_test.go @@ -70,6 +70,53 @@ func setupLiveResumeTestFixture(repo *mockRepository) { } } +func TestResumeSession_PassesResolvedTaskSessionMCPModeToAgentManager(t *testing.T) { + tests := []struct { + name string + officeAssignee string + metadata map[string]interface{} + wantMode string + }{ + {name: "regular task", wantMode: ""}, + {name: "Office task", officeAssignee: "office-agent", wantMode: McpModeOffice}, + {name: "Config session", metadata: map[string]interface{}{"config_mode": true}, wantMode: McpModeConfig}, + { + name: "Config session takes precedence for Office task", + officeAssignee: "office-agent", + metadata: map[string]interface{}{"config_mode": true}, + wantMode: McpModeConfig, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.tasks["task-1"].AssigneeAgentProfileID = tt.officeAssignee + repo.sessions["sess-1"].Metadata = tt.metadata + + var capturedReq *LaunchAgentRequest + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) { + capturedReq = req + return &LaunchAgentResponse{AgentExecutionID: "exec-new", Status: v1.AgentStatusStarting}, nil + }, + } + exec := newTestExecutor(t, agentMgr, repo) + + if _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], false); err != nil { + t.Fatalf("ResumeSession: %v", err) + } + if capturedReq == nil { + t.Fatal("LaunchAgent was not called") + } + if capturedReq.McpMode != tt.wantMode { + t.Fatalf("McpMode = %q, want %q", capturedReq.McpMode, tt.wantMode) + } + }) + } +} + // TestResumeSession_LiveAgentReturnsAlreadyRunning ensures ResumeSession returns // ErrExecutionAlreadyRunning instead of killing the live agent subprocess when // a live agent is already registered for the session. diff --git a/apps/backend/internal/orchestrator/executor/executor_test.go b/apps/backend/internal/orchestrator/executor/executor_test.go index 0561e64da7..a638d71d12 100644 --- a/apps/backend/internal/orchestrator/executor/executor_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_test.go @@ -1313,6 +1313,326 @@ verified: } } +func TestHandleAgentProcessStartFailure_CancelledWithoutTeardownStopsExecution(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-123"] = &models.TaskSession{ + ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateCancelled, + } + repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateReview} + + var stopCalls atomic.Int32 + var recoveryCalls atomic.Int32 + var sessionStateCalls atomic.Int32 + var taskStateCalls atomic.Int32 + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "exec-456" { + t.Fatalf("execution ID = %q, want exec-456", executionID) + } + if !force { + t.Fatal("cancelled execution cleanup must force stop") + } + stopCalls.Add(1) + return nil + }, + }, repo) + exec.SetOnAgentStartFailed(func(context.Context, string, string, string, error, bool) bool { + recoveryCalls.Add(1) + return false + }) + exec.SetOnSessionStateChange(func(context.Context, string, string, models.TaskSessionState, string) error { + sessionStateCalls.Add(1) + return nil + }) + exec.SetOnTaskRuntimeStateReconcile(func(context.Context, string, string, v1.TaskState) error { + taskStateCalls.Add(1) + return nil + }) + + exec.handleAgentProcessStartFailure( + context.Background(), + "task-123", + "session-123", + "exec-456", + errors.New("start failed after stop"), + true, + false, + ) + + if got := stopCalls.Load(); got != 1 { + t.Fatalf("StopAgent calls = %d, want 1", got) + } + if got := recoveryCalls.Load(); got != 0 { + t.Fatalf("start-failure recovery calls = %d, want 0", got) + } + if got := sessionStateCalls.Load(); got != 0 { + t.Fatalf("session-state calls = %d, want 0", got) + } + if got := taskStateCalls.Load(); got != 0 { + t.Fatalf("task-state calls = %d, want 0", got) + } +} + +func TestHandleAgentProcessStartFailure_CancelledWithTeardownSkipsExecutionCleanup(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-123"] = &models.TaskSession{ + ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateCancelled, + } + repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateReview} + + var stopCalls atomic.Int32 + var claimCalls atomic.Int32 + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentFunc: func(context.Context, string, bool) error { + stopCalls.Add(1) + return nil + }, + }, repo) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + claimCalls.Add(1) + if sessionID != "session-123" || executionID != "exec-456" { + t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID) + } + return false + }) + + exec.handleAgentProcessStartFailure( + context.Background(), + "task-123", + "session-123", + "exec-456", + errors.New("start failed after stop"), + true, + false, + ) + + if got := claimCalls.Load(); got != 1 { + t.Fatalf("cleanup claim calls = %d, want 1", got) + } + if got := stopCalls.Load(); got != 0 { + t.Fatalf("StopAgent calls = %d, want 0", got) + } +} + +func TestHandleAgentProcessStartFailure_CancellationDuringCallbackStopsUnclaimedExecution(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-123"] = &models.TaskSession{ + ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting, + } + + var stopCalls atomic.Int32 + var callbackCalls atomic.Int32 + var transitionCalls atomic.Int32 + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "exec-456" || !force { + t.Fatalf("StopAgent = (%q, %v), want (exec-456, true)", executionID, force) + } + stopCalls.Add(1) + return nil + }, + }, repo) + exec.SetOnAgentStartFailed(func(context.Context, string, string, string, error, bool) bool { + callbackCalls.Add(1) + return false + }) + exec.SetOnSessionStateTransition(func( + context.Context, + string, + string, + models.TaskSessionState, + string, + ) (bool, models.TaskSessionState, error) { + transitionCalls.Add(1) + return false, models.TaskSessionStateCancelled, nil + }) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + if sessionID != "session-123" || executionID != "exec-456" { + t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID) + } + return true + }) + + exec.handleAgentProcessStartFailure( + context.Background(), + "task-123", + "session-123", + "exec-456", + errors.New("start failed while cancellation landed"), + true, + false, + ) + + if got := callbackCalls.Load(); got != 1 { + t.Fatalf("start-failure callback calls = %d, want 1", got) + } + if got := transitionCalls.Load(); got != 1 { + t.Fatalf("state transition calls = %d, want 1", got) + } + if got := stopCalls.Load(); got != 1 { + t.Fatalf("StopAgent calls = %d, want 1", got) + } +} + +func TestCleanupUnstartedExecutionAfterPersistError_CancelledWithoutTeardownStopsExecution(t *testing.T) { + tests := []struct { + name string + state models.TaskSessionState + wantStops int32 + }{ + {name: "cancelled", state: models.TaskSessionStateCancelled, wantStops: 1}, + {name: "failed", state: models.TaskSessionStateFailed, wantStops: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stopCalls atomic.Int32 + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "exec-456" { + t.Fatalf("execution ID = %q, want exec-456", executionID) + } + if !force { + t.Fatal("cleanup must force a non-cancelled unstarted execution") + } + stopCalls.Add(1) + return nil + }, + }, newMockRepository()) + + exec.cleanupUnstartedExecutionAfterPersistError( + context.Background(), + "session-123", + "exec-456", + &SessionStateSupersededError{SessionID: "session-123", State: tt.state}, + ) + + if got := stopCalls.Load(); got != tt.wantStops { + t.Fatalf("StopAgent calls = %d, want %d", got, tt.wantStops) + } + }) + } +} + +func TestCleanupUnstartedExecutionAfterPersistError_CancelledWithTeardownSkipsExecutionCleanup(t *testing.T) { + var stopCalls atomic.Int32 + exec := newTestExecutor(t, &mockAgentManager{ + stopAgentFunc: func(context.Context, string, bool) error { + stopCalls.Add(1) + return nil + }, + }, newMockRepository()) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + if sessionID != "session-123" || executionID != "exec-456" { + t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID) + } + return false + }) + + exec.cleanupUnstartedExecutionAfterPersistError( + context.Background(), + "session-123", + "exec-456", + &SessionStateSupersededError{ + SessionID: "session-123", + State: models.TaskSessionStateCancelled, + }, + ) + + if got := stopCalls.Load(); got != 0 { + t.Fatalf("StopAgent calls = %d, want 0", got) + } +} + +func TestStartAgentProcessAsync_StopWinningStartRacePreservesReview(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-123"] = &models.TaskSession{ + ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting, + } + repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateScheduling} + + startEntered := make(chan struct{}) + releaseStart := make(chan struct{}) + cleanupClaimed := make(chan struct{}) + stopDone := make(chan struct{}) + var reconcileCalls atomic.Int32 + agentManager := &mockAgentManager{ + startAgentProcessFunc: func(context.Context, string) error { + close(startEntered) + <-releaseStart + return nil + }, + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "exec-456" || !force { + t.Fatalf("StopAgent = (%q, %v), want (exec-456, true)", executionID, force) + } + close(stopDone) + return nil + }, + } + exec := newTestExecutor(t, agentManager, repo) + exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool { + if sessionID != "session-123" || executionID != "exec-456" { + t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID) + } + close(cleanupClaimed) + return true + }) + exec.SetOnTaskRuntimeStateReconcile(func( + ctx context.Context, + taskID, sessionID string, + state v1.TaskState, + ) error { + reconcileCalls.Add(1) + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if isRuntimeWorkingSessionState(session.State) { + _, _, err = repo.UpdateTaskStateIfNotArchived(ctx, taskID, state) + } + return err + }) + + exec.startAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456") + select { + case <-startEntered: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for process start") + } + if err := repo.UpdateTaskSessionState( + context.Background(), + "session-123", + models.TaskSessionStateCancelled, + "stopped by parent task via MCP", + ); err != nil { + t.Fatalf("cancel session: %v", err) + } + repo.mu.Lock() + repo.tasks["task-123"].State = v1.TaskStateReview + repo.mu.Unlock() + close(releaseStart) + + select { + case <-cleanupClaimed: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for post-start cleanup claim") + } + select { + case <-stopDone: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for post-start runtime cleanup") + } + if got := reconcileCalls.Load(); got != 0 { + t.Fatalf("runtime task reconcile calls = %d, want 0", got) + } + repo.mu.Lock() + gotTaskState := repo.tasks["task-123"].State + repo.mu.Unlock() + if gotTaskState != v1.TaskStateReview { + t.Fatalf("task state = %q, want REVIEW", gotTaskState) + } +} + // runAgentProcessAsyncFailureFixture builds an Executor configured to fail // StartAgentProcess, with task/session-state-change recorders for assertions. // stopCh is closed when StopAgent is invoked — since the failure path calls @@ -2068,6 +2388,55 @@ func TestPersistResumeState_SetsStartingState(t *testing.T) { t.Errorf("expected state WAITING_FOR_INPUT, got %s", session.State) } }) + + t.Run("prepare-only resume cannot overwrite coordinator cancellation", func(t *testing.T) { + stale := &models.TaskSession{ + ID: "session-resume-stop-race", + TaskID: "task-1", + State: models.TaskSessionStateWaitingForInput, + UpdatedAt: now, + } + cancelled := *stale + cancelled.State = models.TaskSessionStateCancelled + cancelled.ErrorMessage = "stopped by parent task via MCP" + repo.sessions[stale.ID] = &cancelled + + err := executor.persistResumeState(context.Background(), "task-1", stale, false) + if !errors.Is(err, ErrSessionStateSuperseded) { + t.Fatalf("persistResumeState error = %v, want ErrSessionStateSuperseded", err) + } + stored := repo.sessions[stale.ID] + if stored.State != models.TaskSessionStateCancelled { + t.Fatalf("session state = %q, want CANCELLED", stored.State) + } + }) +} + +func TestPersistLaunchState_PrepareOnlyCannotOverwriteCoordinatorCancellation(t *testing.T) { + repo := newMockRepository() + executor := newTestExecutor(t, &mockAgentManager{}, repo) + now := time.Now().UTC() + stale := &models.TaskSession{ + ID: "session-launch-stop-race", + TaskID: "task-launch-stop-race", + State: models.TaskSessionStateCreated, + UpdatedAt: now, + } + cancelled := *stale + cancelled.State = models.TaskSessionStateCancelled + cancelled.ErrorMessage = "stopped by parent task via MCP" + repo.sessions[stale.ID] = &cancelled + + err := executor.persistLaunchState( + context.Background(), stale.TaskID, stale.ID, stale, &LaunchAgentResponse{}, false, now, + ) + if !errors.Is(err, ErrSessionStateSuperseded) { + t.Fatalf("persistLaunchState error = %v, want ErrSessionStateSuperseded", err) + } + stored := repo.sessions[stale.ID] + if stored.State != models.TaskSessionStateCancelled { + t.Fatalf("session state = %q, want CANCELLED", stored.State) + } } // Regression: PrepareTaskSession launches the workspace in a background diff --git a/apps/backend/internal/orchestrator/scheduler/scheduler.go b/apps/backend/internal/orchestrator/scheduler/scheduler.go index 87133a7c53..0fd8f8a0d3 100644 --- a/apps/backend/internal/orchestrator/scheduler/scheduler.go +++ b/apps/backend/internal/orchestrator/scheduler/scheduler.go @@ -11,6 +11,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/orchestrator/queue" + "github.com/kandev/kandev/internal/task/models" taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" v1 "github.com/kandev/kandev/pkg/api/v1" "go.uber.org/zap" @@ -54,6 +55,14 @@ type TaskRepository interface { // need the archived_at IS NULL guarantee. Returns whether a row was // modified. UpdateTaskStateIfNotArchived(ctx context.Context, taskID string, state v1.TaskState) (bool, error) + // UpdateTaskStateIfSessionState additionally pins the owning session's + // current state, closing races with clarification and terminal transitions. + UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, + ) (bool, error) } // QueueStatus contains queue statistics diff --git a/apps/backend/internal/orchestrator/scheduler/scheduler_test.go b/apps/backend/internal/orchestrator/scheduler/scheduler_test.go index af55ca2337..a213c2d89e 100644 --- a/apps/backend/internal/orchestrator/scheduler/scheduler_test.go +++ b/apps/backend/internal/orchestrator/scheduler/scheduler_test.go @@ -17,6 +17,7 @@ import ( "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/orchestrator/queue" + "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/task/repository" taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" v1 "github.com/kandev/kandev/pkg/api/v1" @@ -246,6 +247,15 @@ func (r *testTaskRepository) UpdateTaskStateIfNotArchived( return true, nil } +func (r *testTaskRepository) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, _ string, + _ models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + return r.UpdateTaskStateIfNotArchived(ctx, taskID, state) +} + func createTestLogger() *logger.Logger { log, _ := logger.NewLogger(logger.LoggingConfig{ Level: "error", // Suppress logs during tests diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index 133fc0404e..179409f314 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -165,6 +165,7 @@ type sessionExecutorStore interface { SetSessionPrimary(ctx context.Context, sessionID string) error RenameTaskSession(ctx context.Context, id, name string) error UpdateTaskSession(ctx context.Context, session *models.TaskSession) error + UpdateTaskSessionIfCurrentState(ctx context.Context, session *models.TaskSession, expected models.TaskSessionState) (bool, error) UpdateTaskSessionState(ctx context.Context, id string, state models.TaskSessionState, errorMessage string) error UpdateTaskSessionBaseCommit(ctx context.Context, id string, baseCommitSHA string) error GetTaskSessionByTaskAndAgent(ctx context.Context, taskID, agentInstanceID string) (*models.TaskSession, error) @@ -425,6 +426,14 @@ type Service struct { // guard does not grow without bound in long-running backend processes. completedExecutions sync.Map + // executionTeardownClaims arbitrates detached runtime teardown by + // "::". Coordinator stop requests graceful + // teardown while terminal-event cleanup requests force teardown; the first + // intent accepted under the session's cancelInFlight guard owns that + // execution. Claims expire with the same bounded grace period used for + // completed-execution stream markers. + executionTeardownClaims sync.Map + // Session reset flags: sessionID -> true while resetAgentContext is restarting process. // Used to suppress stale ready events and avoid draining queued prompts mid-reset. resetInProgressSessions sync.Map @@ -570,13 +579,17 @@ func NewService( s.processParentChildrenCompletedForTaskState(ctx, taskID, state) return nil }) + exec.SetOnTaskRuntimeStateReconcile(s.reconcileTaskStateForRuntime) exec.SetOnSessionStateChange(func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) error { s.updateTaskSessionState(ctx, taskID, sessionID, state, errorMessage, true) return nil }) + exec.SetOnSessionStateTransition(s.transitionTaskSessionState) exec.SetOnSessionStarting(func(ctx context.Context, taskID string, session *models.TaskSession, promoteTask bool) error { return s.setSessionStarting(ctx, taskID, session, promoteTask) }) + exec.SetOnExecutionCleanupClaim(s.claimForcedExecutionCleanup) + exec.SetOnExecutionStopOwnerRegistration(s.RegisterExecutionStopOwner) exec.SetOnTaskReviewStateReconcile(func(ctx context.Context, taskID, completedSessionID string) { s.writeTaskReviewState(ctx, taskID, completedSessionID) }) diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index b3cc269ab7..6a7065bc47 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "os" + "sort" "strings" "sync" "time" @@ -32,6 +33,22 @@ type PromptResult struct { AgentMessage string // The agent's accumulated response message } +// CoordinatorTaskStopStatus is the idempotent product result returned to a +// parent task that asks to halt a direct child. +type CoordinatorTaskStopStatus string + +const ( + CoordinatorTaskStopStatusStopped CoordinatorTaskStopStatus = "stopped" + CoordinatorTaskStopStatusNotRunning CoordinatorTaskStopStatus = "not_running" + coordinatorMCPStopReason = "stopped by parent task via MCP" +) + +// CoordinatorTaskStopResult reports whether at least one live child session +// accepted logical cancellation. Runtime teardown continues asynchronously. +type CoordinatorTaskStopResult struct { + Status CoordinatorTaskStopStatus `json:"status"` +} + // resumeReasonErrorRecovery is the resume reason returned when a session is in // error-recovery state (WAITING_FOR_INPUT with a non-empty ErrorMessage). const resumeReasonErrorRecovery = "error_recovery" @@ -310,14 +327,12 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag // If the workflow step overrode the profile, update the session record in DB // so the frontend tab displays the correct agent (it reads session.agent_profile_id). if effectiveProfileID != session.AgentProfileID { + observedState := session.State s.logger.Info("updating session agent profile for workflow step override", zap.String("session_id", sessionID), zap.String("old_profile", session.AgentProfileID), zap.String("new_profile", effectiveProfileID)) session.AgentProfileID = effectiveProfileID - // Tag as workflow-spawned provenance: the in-place profile mutation - // came from a workflow step override, not direct user selection. - s.tagSessionAsWorkflowSwitched(ctx, sessionID) // Re-resolve the agent profile snapshot so the tab shows the correct agent logo/name. // Set a minimal snapshot first so stale data is never persisted if resolution fails. session.AgentProfileSnapshot = map[string]interface{}{"id": effectiveProfileID} @@ -337,12 +352,13 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag "config_options": maps.Clone(profileInfo.ConfigOptions), } } - s.promoteSessionIfTaskHasNoPrimary(ctx, taskID, session) - if err := s.repo.UpdateTaskSession(ctx, session); err != nil { - s.logger.Warn("failed to update session agent profile", - zap.String("session_id", sessionID), - zap.Error(err)) + if err := s.persistFullTaskSessionIfCurrent(ctx, session, observedState); err != nil { + return nil, fmt.Errorf("persist workflow session profile: %w", err) } + // Tag as workflow-spawned provenance only after the guarded profile + // write succeeds; a concurrent stop owns a rejected session. + s.tagSessionAsWorkflowSwitched(ctx, sessionID) + s.promoteSessionIfTaskHasNoPrimary(ctx, taskID, session) } // Transition task state: CREATED → SCHEDULING → (IN_PROGRESS via executor). @@ -354,10 +370,11 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag if isOfficeTask { s.logger.Debug("skipping SCHEDULING transition for office task", zap.String("task_id", taskID)) - } else if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateScheduling); err != nil { + } else if err := s.scheduleTaskForSession(ctx, taskID, sessionID); err != nil { s.logger.Warn("failed to update task state to SCHEDULING", zap.String("task_id", taskID), zap.Error(err)) + return nil, err } task, err := s.scheduler.GetTask(ctx, taskID) @@ -408,7 +425,9 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag effectivePrompt, planModeActive := s.applyWorkflowAndPlanMode(ctx, effectivePrompt, taskID, sessionID, dbTask.WorkflowStepID, planMode, task.IsEphemeral) // Inject config context for config-mode sessions (dedicated settings chat) + configMode := false if cm, ok := session.Metadata["config_mode"].(bool); ok && cm { + configMode = true effectivePrompt = sysprompt.InjectConfigContext(sessionID, effectivePrompt) } @@ -421,7 +440,10 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag // agent CLI's TTY and the user sees it verbatim — they don't want a wall of // MCP-tool boilerplate prepended to "hello". if (effectivePrompt != "" || len(attachments) > 0) && !sysprompt.HasKandevContext(effectivePrompt) && !session.IsPassthrough { - effectivePrompt = sysprompt.InjectKandevContext(taskID, sessionID, effectivePrompt, s.WorkflowStepRequiresCompletionSignal(ctx, dbTask.WorkflowStepID)) + effectivePrompt = sysprompt.InjectKandevContextWithOptions(taskID, sessionID, effectivePrompt, sysprompt.KandevContextOptions{ + RequiresCompletionSignal: s.WorkflowStepRequiresCompletionSignal(ctx, dbTask.WorkflowStepID), + IncludeCoordinatorTaskControls: !isOfficeTask && !configMode, + }) } executorID := session.ExecutorID @@ -430,7 +452,11 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag // re-drive this first turn — initial launches bypass PromptTask. s.rememberTurnPrompt(sessionID, prompt, "", planMode, attachments) - execution, err := s.executor.LaunchPreparedSession(ctx, task, sessionID, executor.LaunchOptions{AgentProfileID: effectiveProfileID, ExecutorID: executorID, Prompt: effectivePrompt, StartAgent: true, Attachments: attachments}) + mcpMode := "" + if isOfficeTask { + mcpMode = executor.McpModeOffice + } + execution, err := s.executor.LaunchPreparedSession(ctx, task, sessionID, executor.LaunchOptions{AgentProfileID: effectiveProfileID, ExecutorID: executorID, Prompt: effectivePrompt, StartAgent: true, McpMode: mcpMode, Attachments: attachments}) if err != nil { return nil, err } @@ -448,6 +474,27 @@ func (s *Service) StartCreatedSession(ctx context.Context, taskID, sessionID, ag return execution, nil } +func (s *Service) scheduleTaskForSession(ctx context.Context, taskID, sessionID string) error { + s.taskRuntimeStateMu.Lock() + defer s.taskRuntimeStateMu.Unlock() + + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return err + } + if session == nil { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, sessionID) + } + if session.State != models.TaskSessionStateCreated && + session.State != models.TaskSessionStateWaitingForInput { + if isTerminalSessionState(session.State) { + return &executor.SessionStateSupersededError{SessionID: session.ID, State: session.State} + } + return fmt.Errorf("session %s is %s; cannot schedule task", session.ID, session.State) + } + return s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateScheduling) +} + func (s *Service) promoteSessionIfTaskHasNoPrimary(ctx context.Context, taskID string, session *models.TaskSession) { if session == nil || session.IsPrimary { return @@ -669,7 +716,10 @@ func (s *Service) startTask(ctx context.Context, taskID string, agentProfileID s // directly is wrong because it can be empty on manual user-initiated starts // while the task is already bound to a signal-gated step in the DB. if (effectivePrompt != "" || len(attachments) > 0) && !sysprompt.HasKandevContext(effectivePrompt) && !skipKandevMCPWrap { - effectivePrompt = sysprompt.InjectKandevContext(task.ID, sessionID, effectivePrompt, s.StepRequiresCompletionSignal(ctx, task.ID)) + effectivePrompt = sysprompt.InjectKandevContextWithOptions(task.ID, sessionID, effectivePrompt, sysprompt.KandevContextOptions{ + RequiresCompletionSignal: s.StepRequiresCompletionSignal(ctx, task.ID), + IncludeCoordinatorTaskControls: !isOfficeTask && !configMode, + }) } // Office tasks restrict the MCP toolset: kanban tools (move/update/list @@ -1400,6 +1450,13 @@ func (s *Service) reapPromptUnreadyExecution(ctx context.Context, sessionID stri } return fmt.Errorf("execution id for session %s not found", sessionID) } + if !s.claimForcedExecutionCleanup(sessionID, executionID) { + return fmt.Errorf( + "execution %s teardown is already owned for session %s", + executionID, + sessionID, + ) + } s.logger.Warn("stopping prompt-unready agent execution before resume", zap.String("session_id", sessionID), @@ -1408,6 +1465,19 @@ func (s *Service) reapPromptUnreadyExecution(ctx context.Context, sessionID stri if err := s.executor.StopExecution(ctx, executionID, promptReadinessRecoveryStopReason, true); err != nil { return err } + refreshed, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return fmt.Errorf("reload session after prompt-readiness teardown: %w", err) + } + if refreshed == nil { + return fmt.Errorf("reload session after prompt-readiness teardown: session %s not found", sessionID) + } + if isTerminalSessionState(refreshed.State) { + return &executor.SessionStateSupersededError{ + SessionID: refreshed.ID, + State: refreshed.State, + } + } return nil } @@ -1818,6 +1888,136 @@ func (s *Service) StopTask(ctx context.Context, taskID string, reason string, fo return nil } +// StopTaskForCoordinator gracefully halts every currently-observed live +// execution for taskID without accepting caller-controlled lifecycle options. +// The operation is idempotent: no accepted cancellation is not_running. +func (s *Service) StopTaskForCoordinator(ctx context.Context, taskID string) (CoordinatorTaskStopResult, error) { + if s.executor == nil { + return CoordinatorTaskStopResult{}, errors.New("coordinator stop: executor is not configured") + } + sessions, err := s.repo.ListActiveTaskSessionsByTaskID(ctx, taskID) + if err != nil { + return CoordinatorTaskStopResult{}, fmt.Errorf("coordinator stop: list active sessions for task %q: %w", taskID, err) + } + + // Repository ordering is launch-oriented. Coordinator control uses stable + // identity ordering so partial failures and retries are deterministic. + sort.SliceStable(sessions, func(i, j int) bool { + return coordinatorStopSessionID(sessions[i]) < coordinatorStopSessionID(sessions[j]) + }) + + accepted := 0 + failures := make([]error, 0) + for _, candidate := range sessions { + if candidate == nil || candidate.ID == "" { + failures = append(failures, errors.New("coordinator stop: active session candidate is nil or has an empty ID")) + continue + } + changed, stopErr := s.stopTaskSessionForCoordinator(ctx, taskID, candidate.ID) + if stopErr != nil { + failures = append(failures, stopErr) + continue + } + if changed { + accepted++ + } + } + if accepted > 0 { + // This helper owns Office/archive/active-state guards and publishes + // through the task-service adapter. Remaining working sessions still + // block REVIEW on a partial stop. + s.writeTaskReviewState(ctx, taskID, "") + } + if len(failures) > 0 { + s.logger.Warn("coordinator stop partially failed", + zap.String("task_id", taskID), + zap.Int("accepted", accepted), + zap.Int("failed", len(failures))) + return CoordinatorTaskStopResult{}, errors.Join(failures...) + } + if accepted == 0 { + return CoordinatorTaskStopResult{Status: CoordinatorTaskStopStatusNotRunning}, nil + } + return CoordinatorTaskStopResult{Status: CoordinatorTaskStopStatusStopped}, nil +} + +func coordinatorStopSessionID(session *models.TaskSession) string { + if session == nil { + return "" + } + return session.ID +} + +func (s *Service) stopTaskSessionForCoordinator(ctx context.Context, taskID, sessionID string) (bool, error) { + lock, release := s.acquireCancelInFlightGuard(sessionID) + lock.Lock() + + result, teardownClaimed, err := s.stopTaskSessionForCoordinatorLocked(ctx, taskID, sessionID) + lock.Unlock() + release() + if err != nil { + return false, err + } + if result.Changed && teardownClaimed { + result.ScheduleTeardown() + } + return result.Changed, nil +} + +// stopTaskSessionForCoordinatorLocked commits cancellation and claims teardown +// ownership while sessionID's cancelInFlight guard is held. The caller must +// release that guard before scheduling the returned teardown. +func (s *Service) stopTaskSessionForCoordinatorLocked( + ctx context.Context, + taskID, sessionID string, +) (executor.SessionStopResult, bool, error) { + + // Re-read after acquiring the shared cancel/ready/queue guard. A candidate + // may have become terminal while this call waited for another decision. + session, err := s.repo.GetTaskSession(ctx, sessionID) + if err != nil { + return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: reload session %q: %w", sessionID, err) + } + if session == nil { + return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: reload session %q returned nil", sessionID) + } + if session.TaskID != taskID { + return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: session %q belongs to task %q, not %q", sessionID, session.TaskID, taskID) + } + if !isCoordinatorStoppableSessionState(session.State) { + return executor.SessionStopResult{FinalState: session.State}, false, nil + } + + s.taskRuntimeStateMu.Lock() + defer s.taskRuntimeStateMu.Unlock() + // Halt-only intent also disarms any provider-backoff retry. This must run + // even when the failed execution has already disappeared and the result is + // therefore not_running; otherwise its timer can launch replacement work. + s.resetTransientRetry(sessionID) + result, err := s.executor.StopSessionDetailed(ctx, session, coordinatorMCPStopReason, false) + if err != nil { + return result, false, fmt.Errorf("coordinator stop: session %q: %w", sessionID, err) + } + teardownClaimed := result.Changed && s.claimExecutionTeardown( + sessionID, + result.ExecutionID, + executionTeardownIntentGraceful, + ) + return result, teardownClaimed, nil +} + +func isCoordinatorStoppableSessionState(state models.TaskSessionState) bool { + switch state { + case models.TaskSessionStateCreated, + models.TaskSessionStateStarting, + models.TaskSessionStateRunning, + models.TaskSessionStateWaitingForInput: + return true + default: + return false + } +} + // CancelTaskExecution stops active sessions for a task without mutating task state. // It is used by office tree controls where pause/cancel state transitions are // handled by the office service itself. @@ -2409,10 +2609,10 @@ func (s *Service) effectivePromptForSession(sessionID, prompt string, planMode b // promptTask dispatches to the agent, and returns the session to use for // the rest of the call (possibly reloaded). // -// When claimEntryID is empty (a direct, non-queued prompt) this is a bare, -// unguarded transition — preserving PromptTask's pre-existing direct-call -// behavior as-is; this branch takes no guard and makes no claim about -// races with a concurrent queued-dispatch decision on the same session. +// Direct and queued prompts both claim RUNNING under the per-session +// cancelInFlight guard. This makes the final "still promptable, now owned by +// this turn" decision atomic with coordinator stop; the potentially blocking +// executor.Prompt call remains outside the guard. // // When claimEntryID is set (a queued dispatch claiming its reserved token — // see the Service.dispatchingQueued field doc comment), the claim happens @@ -2428,17 +2628,12 @@ func (s *Service) effectivePromptForSession(sessionID, prompt string, planMode b // checkSessionPromptable here makes "is it actually still safe to mark this // session RUNNING right now" an atomic fact, not a stale read. func (s *Service) claimSessionRunningForPrompt(ctx context.Context, taskID, sessionID, claimEntryID string, session *models.TaskSession) (*models.TaskSession, error) { - if claimEntryID == "" { - s.setSessionRunning(ctx, taskID, sessionID, session) - return session, nil - } - lock, release := s.acquireCancelInFlightGuard(sessionID) defer release() lock.Lock() defer lock.Unlock() - if !s.isCurrentQueuedDispatch(sessionID, claimEntryID) { + if claimEntryID != "" && !s.isCurrentQueuedDispatch(sessionID, claimEntryID) { return nil, errQueuedDispatchSuperseded } freshSession, sessErr := s.repo.GetTaskSession(ctx, sessionID) @@ -2452,6 +2647,15 @@ func (s *Service) claimSessionRunningForPrompt(ctx context.Context, taskID, sess return nil, promptErr } s.setSessionRunning(ctx, taskID, sessionID, freshSession) + // setSessionRunning refreshes freshSession in place when its guarded write + // loses, so a cancellation landing after the promptability check is visible + // here as a terminal state. + if isTerminalSessionState(freshSession.State) && freshSession.State != models.TaskSessionStateCompleted { + return nil, &executor.SessionStateSupersededError{ + SessionID: freshSession.ID, + State: freshSession.State, + } + } // Clear the token now, inside the same lock, rather than deferring to // executeQueuedMessage's cleanup after this entire (potentially // long-blocking) call returns: from this point session.State is the @@ -2459,7 +2663,9 @@ func (s *Service) claimSessionRunningForPrompt(ctx context.Context, taskID, sess // any longer would only risk the natural agent.ready firing when *this* // turn completes seeing it still set and skipping the *next* queued // entry — see the Service.dispatchingQueued field doc comment. - s.clearQueuedDispatchInFlightIfCurrent(sessionID, claimEntryID) + if claimEntryID != "" { + s.clearQueuedDispatchInFlightIfCurrent(sessionID, claimEntryID) + } return freshSession, nil } diff --git a/apps/backend/internal/orchestrator/task_operations_test.go b/apps/backend/internal/orchestrator/task_operations_test.go index b3c1733cd5..2c22a63392 100644 --- a/apps/backend/internal/orchestrator/task_operations_test.go +++ b/apps/backend/internal/orchestrator/task_operations_test.go @@ -65,6 +65,129 @@ func seedTaskAndSession(t *testing.T, repo *sqliterepo.Repository, taskID, sessi } } +func newCoordinatorStopTestService( + repo repoStore, + taskRepo scheduler.TaskRepository, + agentManager executor.AgentManagerClient, +) *Service { + log := testLogger() + exec := executor.NewExecutor(agentManager, repo, log, executor.ExecutorConfig{}) + svc := &Service{ + logger: log, + repo: repo, + taskRepo: taskRepo, + agentManager: agentManager, + executor: exec, + messageQueue: messagequeue.NewServiceMemory(log), + } + exec.SetOnSessionStateTransition(svc.transitionTaskSessionState) + exec.SetOnExecutionCleanupClaim(svc.claimForcedExecutionCleanup) + exec.SetOnExecutionStopOwnerRegistration(svc.RegisterExecutionStopOwner) + return svc +} + +func TestStopTaskForCoordinator_StopsAndIsIdempotent(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + seedExecutorRunning(t, repo, "session1", "task1", "execution1") + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task1", v1.TaskStateInProgress) + agentManager := &mockAgentManager{repoForExecutionLookup: repo} + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + if _, err := svc.messageQueue.QueueMessage( + ctx, "session1", "task1", "preserve me", "", messagequeue.QueuedByAgent, false, nil, + ); err != nil { + t.Fatalf("queue message: %v", err) + } + + result, err := svc.StopTaskForCoordinator(ctx, "task1") + + if err != nil { + t.Fatalf("StopTaskForCoordinator: %v", err) + } + if result.Status != CoordinatorTaskStopStatusStopped { + t.Fatalf("status = %q, want %q", result.Status, CoordinatorTaskStopStatusStopped) + } + session, err := repo.GetTaskSession(ctx, "session1") + if err != nil { + t.Fatalf("get stopped session: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Fatalf("session state = %q, want CANCELLED", session.State) + } + if got := taskRepo.updatedStates["task1"]; got != v1.TaskStateReview { + t.Fatalf("task state = %q, want REVIEW", got) + } + if got := svc.messageQueue.GetStatus(ctx, "session1").Count; got != 1 { + t.Fatalf("queued message count = %d, want 1", got) + } + waitForStopCall(t, agentManager) + agentManager.mu.Lock() + stopCall := agentManager.stopAgentWithReasonArgs[0] + agentManager.mu.Unlock() + if stopCall.ExecutionID != "execution1" || stopCall.Reason != coordinatorMCPStopReason || stopCall.Force { + t.Fatalf("stop call = %#v", stopCall) + } + + repeat, err := svc.StopTaskForCoordinator(ctx, "task1") + if err != nil { + t.Fatalf("repeat StopTaskForCoordinator: %v", err) + } + if repeat.Status != CoordinatorTaskStopStatusNotRunning { + t.Fatalf("repeat status = %q, want %q", repeat.Status, CoordinatorTaskStopStatusNotRunning) + } +} + +func TestStopTaskForCoordinator_AggregatesAbsentAndFailure(t *testing.T) { + lookupFailure := errors.New("lifecycle store unavailable") + tests := []struct { + name string + lookupErr error + wantStatus CoordinatorTaskStopStatus + wantErr error + }{ + { + name: "all absent is not running", + lookupErr: fmt.Errorf("wrapped: %w", lifecycle.ErrNoExecutionForSession), + wantStatus: CoordinatorTaskStopStatusNotRunning, + }, + { + name: "genuine lookup failure is returned", + lookupErr: lookupFailure, + wantErr: lookupFailure, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + taskRepo := newMockTaskRepo() + seedMockTaskState(taskRepo, "task1", v1.TaskStateInProgress) + agentManager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "", tt.lookupErr + }, + } + svc := newCoordinatorStopTestService(repo, taskRepo, agentManager) + + result, err := svc.StopTaskForCoordinator(ctx, "task1") + + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if result.Status != tt.wantStatus { + t.Fatalf("status = %q, want %q", result.Status, tt.wantStatus) + } + if _, changed := taskRepo.updatedStates["task1"]; changed { + t.Fatal("task state changed without an accepted clean stop") + } + }) + } +} + // --- PromptTask --- func TestPromptTask_EmptySessionID(t *testing.T) { @@ -2199,11 +2322,19 @@ func TestStartCreatedSession_OfficeTaskSkipsSchedulingState(t *testing.T) { stepGetter := newMockStepGetter() stepGetter.steps["step-office"] = &wfmodels.WorkflowStep{ID: "step-office", WorkflowID: "wf1"} svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr) - svc.messageCreator = &mockMessageCreator{} + messages := &mockMessageCreator{} + svc.messageCreator = messages - if _, err := svc.StartCreatedSession(ctx, "task1", "session1", "profile1", "Do the work", true, false, true, nil); err != nil { + if _, err := svc.StartCreatedSession(ctx, "task1", "session1", "profile1", "Do the work", false, false, true, nil); err != nil { t.Fatalf("StartCreatedSession: %v", err) } + require.Len(t, messages.userMessages, 1) + assert.NotContains(t, messages.userMessages[0].content, "stop_task_kandev", + "Office first-turn context must not advertise a task-mode-only tool") + agentMgr.mu.Lock() + mcpModeCalls := append([]sessionModeCall(nil), agentMgr.mcpModeCalls...) + agentMgr.mu.Unlock() + require.Equal(t, []sessionModeCall{{SessionID: "exec-1", ModeID: executor.McpModeOffice}}, mcpModeCalls) if writes := taskRepo.stateWrites["task1"]; writes != 0 { t.Fatalf("office task should not write SCHEDULING, got %d state writes", writes) @@ -2213,6 +2344,29 @@ func TestStartCreatedSession_OfficeTaskSkipsSchedulingState(t *testing.T) { } } +func TestStartCreatedSession_ConfigModeOmitsCoordinatorTaskControls(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateCreated) + seedExecutorRunning(t, repo, "session1", "task1", "exec-1") + + require.NoError(t, repo.UpdateSessionMetadata(ctx, "session1", map[string]interface{}{"config_mode": true})) + + taskRepo := newMockTaskRepo() + taskRepo.tasks["task1"] = &v1.Task{ID: "task1", Title: "Config chat", State: v1.TaskStateInProgress} + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + messages := &mockMessageCreator{} + svc.messageCreator = messages + + _, err := svc.StartCreatedSession(ctx, "task1", "session1", "profile1", "Configure Kandev", false, false, false, nil) + require.NoError(t, err) + require.Len(t, messages.userMessages, 1) + assert.Contains(t, messages.userMessages[0].content, "KANDEV CONFIG MCP TOOLS") + assert.NotContains(t, messages.userMessages[0].content, "stop_task_kandev", + "Config first-turn context must not advertise a task-mode-only tool") +} + // --- recordInitialMessage --- // mockMessageCreator implements MessageCreator for testing. @@ -2891,6 +3045,18 @@ func (c *ctxAwareTaskRepo) UpdateTaskStateIfNotArchived( return c.inner.UpdateTaskStateIfNotArchived(ctx, taskID, state) } +func (c *ctxAwareTaskRepo) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return c.inner.UpdateTaskStateIfSessionState(ctx, taskID, sessionID, expectedSessionState, state) +} + // TestResumeTaskSession_FailedStateWriteSurvivesCancelledCallerCtx verifies the // fix for the WS-disconnect cascade: when the caller's ctx was already // cancelled (e.g. the user navigated away mid-resume) and the launch then @@ -3721,12 +3887,21 @@ func TestReconcileSessionsOnStartup(t *testing.T) { // --- ensureSessionRunning: prepared workspace --- -func TestEnsureSessionRunning_PreparedWorkspace(t *testing.T) { +func TestEnsureSessionRunning_PreparedOfficeWorkspaceSetsMCPMode(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) // Seed task and session in CREATED state (workspace prepared, agent not started) seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateCreated) + dbTask, err := repo.GetTask(ctx, "task1") + if err != nil { + t.Fatalf("failed to load task: %v", err) + } + dbTask.WorkflowStepID = "step-office" + dbTask.AssigneeAgentProfileID = "office-agent" + if err := repo.UpdateTask(ctx, dbTask); err != nil { + t.Fatalf("failed to mark task as Office-owned: %v", err) + } // Set AgentExecutionID to simulate a prepared workspace session, err := repo.GetTaskSession(ctx, "session1") @@ -3790,6 +3965,10 @@ func TestEnsureSessionRunning_PreparedWorkspace(t *testing.T) { if !startAgentProcessCalled { t.Fatal("expected StartAgentProcess to be called (prepared workspace path)") } + wrappedMgr.mu.Lock() + mcpModeCalls := append([]sessionModeCall(nil), wrappedMgr.mcpModeCalls...) + wrappedMgr.mu.Unlock() + require.Equal(t, []sessionModeCall{{SessionID: "exec-prepare-1", ModeID: executor.McpModeOffice}}, mcpModeCalls) // Verify the session transitioned through STARTING updated, err := repo.GetTaskSession(ctx, "session1") diff --git a/apps/backend/internal/sysprompt/sysprompt.go b/apps/backend/internal/sysprompt/sysprompt.go index 6ac92d044a..94a3f4dcb6 100644 --- a/apps/backend/internal/sysprompt/sysprompt.go +++ b/apps/backend/internal/sysprompt/sysprompt.go @@ -75,10 +75,15 @@ func HasKandevContext(text string) bool { // It instructs agents to collaborate on the plan without implementing changes. func PlanMode() string { return prompts.Get("plan-mode") } -// KandevContext returns the system prompt template that provides Kandev-specific -// instructions and session context to agents. Contains {task_id}, {session_id}, -// and {step_complete_section} placeholders — use [FormatKandevContext] to inject values. -func KandevContext() string { return prompts.Get("kandev-context") } +// KandevContext returns the task-mode system prompt template that provides +// Kandev-specific instructions and session context to agents. Contains +// {task_id}, {session_id}, and {step_complete_section} placeholders — use +// [FormatKandevContext] to inject values. +func KandevContext() string { + return Resolve("kandev-context", map[string]string{ + "coordinator_task_control_section": coordinatorTaskControlSection, + }) +} // stepCompleteSection is the description + instruction block for the // step_complete_kandev MCP tool. Only injected when the current workflow step @@ -97,18 +102,51 @@ const stepCompleteSection = "- step_complete_kandev: Signal that every user-stat "Do NOT call when asking a question, mid-conversation, or on partial progress. " + "Required params: summary (one-paragraph plain text). Optional: handoff, blockers.\n" +// coordinatorTaskControlSection documents task-mode-only parent/child controls. +// Restricted MCP modes omit the section because neither message_task_kandev nor +// stop_task_kandev is registered there. The baseline message-tool sentence stays +// in the template to avoid broadening this feature into a cleanup of older mode +// mismatches. +const coordinatorTaskControlSection = " Optional: session_id, delivery_mode. " + + "Use delivery_mode=\"queued\" or omit it for information that can wait. " + + "Use delivery_mode=\"interrupt\" for urgent replacement work on a running direct child; " + + "if immediate cancel-and-dispatch cannot be confirmed safely, the message remains queued. " + + "For halt-only work, use stop_task_kandev.\n" + + "- stop_task_kandev: Halt all live sessions observed for a direct child, with no prompt and no replacement turn. " + + "Only the target task's direct parent may call it. Required params: task_id." + +// KandevContextOptions controls capability-dependent sections in the first-turn +// Kandev context. +type KandevContextOptions struct { + RequiresCompletionSignal bool + IncludeCoordinatorTaskControls bool +} + // FormatKandevContext returns the Kandev context prompt with task and session IDs injected. // When requiresCompletionSignal is true, the step_complete_kandev tool description is // included; otherwise the placeholder is collapsed to an empty string. func FormatKandevContext(taskID, sessionID string, requiresCompletionSignal bool) string { + return FormatKandevContextWithOptions(taskID, sessionID, KandevContextOptions{ + RequiresCompletionSignal: requiresCompletionSignal, + IncludeCoordinatorTaskControls: true, + }) +} + +// FormatKandevContextWithOptions returns capability-aware Kandev context. +func FormatKandevContextWithOptions(taskID, sessionID string, options KandevContextOptions) string { section := "" - if requiresCompletionSignal { + if options.RequiresCompletionSignal { section = stepCompleteSection } + coordinatorControls := "" + if options.IncludeCoordinatorTaskControls { + coordinatorControls = coordinatorTaskControlSection + } return Resolve("kandev-context", map[string]string{ - "task_id": taskID, - "session_id": sessionID, - "step_complete_section": section, + "task_id": taskID, + "session_id": sessionID, + "step_complete_section": section, + "coordinator_task_control_section": coordinatorControls, }) } @@ -137,6 +175,11 @@ func InjectKandevContext(taskID, sessionID, prompt string, requiresCompletionSig return Wrap(FormatKandevContext(taskID, sessionID, requiresCompletionSignal)) + "\n\n" + prompt } +// InjectKandevContextWithOptions prepends capability-aware Kandev context. +func InjectKandevContextWithOptions(taskID, sessionID, prompt string, options KandevContextOptions) string { + return Wrap(FormatKandevContextWithOptions(taskID, sessionID, options)) + "\n\n" + prompt +} + // DefaultPlanPrefix returns the planning instruction prompt used when plan mode // is requested but no workflow step provides its own prompt prefix. func DefaultPlanPrefix() string { return prompts.Get("default-plan-prefix") } diff --git a/apps/backend/internal/sysprompt/sysprompt_test.go b/apps/backend/internal/sysprompt/sysprompt_test.go index 9807394b86..d69a5a5941 100644 --- a/apps/backend/internal/sysprompt/sysprompt_test.go +++ b/apps/backend/internal/sysprompt/sysprompt_test.go @@ -123,6 +123,21 @@ func TestFormatKandevContext_IncludesStepCompleteToolWhenRequired(t *testing.T) "step_complete_kandev must be exposed when the step requires an explicit signal") } +func TestFormatKandevContext_CoordinatorTaskControlsFollowCapability(t *testing.T) { + taskMode := FormatKandevContext("task-abc", "session-xyz", false) + assert.Contains(t, taskMode, `delivery_mode="interrupt"`) + assert.Contains(t, taskMode, "stop_task_kandev") + + for _, mode := range []string{"office", "config"} { + t.Run(mode, func(t *testing.T) { + context := FormatKandevContextWithOptions("task-abc", "session-xyz", KandevContextOptions{}) + assert.NotContains(t, context, "delivery_mode") + assert.NotContains(t, context, "stop_task_kandev") + assert.NotContains(t, context, "{coordinator_task_control_section}") + }) + } +} + func TestFormatKandevContext_InjectsIDs(t *testing.T) { result := FormatKandevContext("task-abc", "session-xyz", false) assert.Contains(t, result, "Kandev Task ID: task-abc") diff --git a/apps/backend/internal/task/handlers/message_handlers.go b/apps/backend/internal/task/handlers/message_handlers.go index 2cad1afaeb..de531eea87 100644 --- a/apps/backend/internal/task/handlers/message_handlers.go +++ b/apps/backend/internal/task/handlers/message_handlers.go @@ -292,8 +292,17 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w // wall of MCP-tool boilerplate prepended to "hello". storedContent := req.Content if isCreatedSession && !sessionResp.Session.IsPassthrough && (req.Content != "" || len(req.Attachments) > 0) { + task, err := h.service.GetTask(ctx, req.TaskID) + if err != nil { + h.logger.Error("failed to resolve first-turn MCP capabilities", zap.String("task_id", req.TaskID), zap.Error(err)) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to get task", nil) + } + configMode, _ := sessionResp.Session.Metadata["config_mode"].(bool) requiresSignal := h.orchestrator != nil && h.orchestrator.StepRequiresCompletionSignal(ctx, req.TaskID) - storedContent = sysprompt.InjectKandevContext(req.TaskID, req.TaskSessionID, req.Content, requiresSignal) + storedContent = sysprompt.InjectKandevContextWithOptions(req.TaskID, req.TaskSessionID, req.Content, sysprompt.KandevContextOptions{ + RequiresCompletionSignal: requiresSignal, + IncludeCoordinatorTaskControls: task.AssigneeAgentProfileID == "" && !configMode, + }) req.Content = storedContent } diff --git a/apps/backend/internal/task/handlers/message_handlers_test.go b/apps/backend/internal/task/handlers/message_handlers_test.go index 99f421b003..08226bc19a 100644 --- a/apps/backend/internal/task/handlers/message_handlers_test.go +++ b/apps/backend/internal/task/handlers/message_handlers_test.go @@ -311,6 +311,73 @@ func (r *messageAddSwitchRepo) CreateTurn(_ context.Context, turn *models.Turn) return nil } +func TestWSAddMessage_CreatedOfficeSessionOmitsCoordinatorTaskControls(t *testing.T) { + now := time.Now().UTC() + content := runCreatedMessageContextTest(t, &models.Task{ + ID: "t1", + State: v1.TaskStateInProgress, + AssigneeAgentProfileID: "office-agent", + UpdatedAt: now, + }, &models.TaskSession{ + ID: "s1", + TaskID: "t1", + State: models.TaskSessionStateCreated, + AgentProfileID: "profile-1", + UpdatedAt: now, + }) + assert.NotContains(t, content, "stop_task_kandev", + "Office pre-wrap must not persist a task-mode-only tool") +} + +func TestWSAddMessage_CreatedConfigSessionOmitsCoordinatorTaskControls(t *testing.T) { + now := time.Now().UTC() + content := runCreatedMessageContextTest(t, &models.Task{ + ID: "t1", + State: v1.TaskStateInProgress, + UpdatedAt: now, + }, &models.TaskSession{ + ID: "s1", + TaskID: "t1", + State: models.TaskSessionStateCreated, + AgentProfileID: "profile-1", + Metadata: map[string]interface{}{"config_mode": true}, + UpdatedAt: now, + }) + assert.NotContains(t, content, "stop_task_kandev", + "Config pre-wrap must not persist a task-mode-only tool") +} + +func runCreatedMessageContextTest(t *testing.T, task *models.Task, session *models.TaskSession) string { + t.Helper() + repo := &messageAddSwitchRepo{ + tasks: map[string]*models.Task{task.ID: task}, + sessions: map[string]*models.TaskSession{session.ID: session}, + primaryID: session.ID, + } + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + require.NoError(t, err) + svc := service.NewService(service.Repos{ + Workspaces: repo, Tasks: repo, TaskRepos: repo, + Workflows: repo, Messages: repo, Turns: repo, + Sessions: repo, GitSnapshots: repo, RepoEntities: repo, + Executors: repo, Environments: repo, TaskEnvironments: repo, + Reviews: repo, + }, nil, log, service.RepositoryDiscoveryConfig{}) + h := NewMessageHandlers(svc, nil, log) + + req, err := ws.NewRequest("req-office", ws.ActionMessageAdd, map[string]interface{}{ + "task_id": task.ID, + "session_id": session.ID, + "content": "Do the work", + }) + require.NoError(t, err) + resp, err := h.wsAddMessage(context.Background(), req) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, resp.Type) + require.Len(t, repo.messages, 1) + return repo.messages[0].Content +} + type switchingTurnStartOrchestrator struct { mu sync.Mutex startOnce sync.Once diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index c809ed0f25..e40aee9384 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -95,6 +95,11 @@ func (m *mockRepository) CountOpenWatcherCreatedTasks(_ context.Context, _, _ st func (m *mockRepository) UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error { return nil } +func (m *mockRepository) UpdateTaskStateIfSessionState( + _ context.Context, _, _ string, _ models.TaskSessionState, _ v1.TaskState, +) (v1.TaskState, bool, error) { + return "", false, nil +} func (m *mockRepository) UpdateTaskStateIfCurrentIn(_ context.Context, _ string, _ v1.TaskState, _ []v1.TaskState) (v1.TaskState, bool, error) { return "", false, nil } diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index d77a015758..16d7b270d5 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -63,6 +63,15 @@ type TaskRepository interface { // which integrations exist. CountOpenWatcherCreatedTasks(ctx context.Context, metadataKey, watchID string) (int, error) UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error + // UpdateTaskStateIfSessionState atomically transitions task state only while + // the named owning session remains in expectedSessionState and the task is + // not archived. Returns the pre-update task state and whether a row changed. + UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, + ) (v1.TaskState, bool, error) // UpdateTaskStateIfCurrentIn atomically transitions state only when the // task's current state is in allowed AND the task is not archived // (archived_at IS NULL). The archived check is enforced inside the same diff --git a/apps/backend/internal/task/repository/sqlite/session.go b/apps/backend/internal/task/repository/sqlite/session.go index e9bc753684..0f20b226b7 100644 --- a/apps/backend/internal/task/repository/sqlite/session.go +++ b/apps/backend/internal/task/repository/sqlite/session.go @@ -439,6 +439,19 @@ func (r *Repository) UpdateTaskSession(ctx context.Context, session *models.Task return r.updateTaskSession(ctx, r.db, session) } +// UpdateTaskSessionIfCurrentState persists a full session row only while the +// stored state still matches expected. Runtime launch/resume paths use this to +// retain their non-state fields without reviving a concurrently cancelled +// session from a stale snapshot. +func (r *Repository) UpdateTaskSessionIfCurrentState( + ctx context.Context, + session *models.TaskSession, + expected models.TaskSessionState, +) (bool, error) { + session.UpdatedAt = time.Now().UTC() + return r.updateTaskSessionWithStateGuard(ctx, r.db, session, &expected) +} + // UpdateTaskSessionWithMetadata updates the session row and metadata column in // one transaction so callers cannot observe a partially-applied update. func (r *Repository) UpdateTaskSessionWithMetadata( @@ -470,21 +483,37 @@ func (r *Repository) updateTaskSession( exec taskSessionExecutor, session *models.TaskSession, ) error { + changed, err := r.updateTaskSessionWithStateGuard(ctx, exec, session, nil) + if err != nil { + return err + } + if !changed { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, session.ID) + } + return nil +} + +func (r *Repository) updateTaskSessionWithStateGuard( + ctx context.Context, + exec taskSessionExecutor, + session *models.TaskSession, + expected *models.TaskSessionState, +) (bool, error) { agentProfileSnapshotJSON, err := json.Marshal(session.AgentProfileSnapshot) if err != nil { - return fmt.Errorf("failed to serialize agent profile snapshot: %w", err) + return false, fmt.Errorf("failed to serialize agent profile snapshot: %w", err) } executorSnapshotJSON, err := json.Marshal(session.ExecutorSnapshot) if err != nil { - return fmt.Errorf("failed to serialize executor snapshot: %w", err) + return false, fmt.Errorf("failed to serialize executor snapshot: %w", err) } environmentSnapshotJSON, err := json.Marshal(session.EnvironmentSnapshot) if err != nil { - return fmt.Errorf("failed to serialize environment snapshot: %w", err) + return false, fmt.Errorf("failed to serialize environment snapshot: %w", err) } repositorySnapshotJSON, err := json.Marshal(session.RepositorySnapshot) if err != nil { - return fmt.Errorf("failed to serialize repository snapshot: %w", err) + return false, fmt.Errorf("failed to serialize repository snapshot: %w", err) } // metadata is NOT written here — callers wanting to change it must use @@ -498,30 +527,34 @@ func (r *Repository) updateTaskSession( if session.AgentProfileID != "" { agentProfileID = session.AgentProfileID } - result, err := exec.ExecContext(ctx, r.db.Rebind(` + query := ` UPDATE task_sessions SET agent_profile_id = ?, execution_profile_id = ?, executor_id = ?, executor_profile_id = ?, environment_id = ?, repository_id = ?, base_branch = ?, base_commit_sha = ?, workspace_path = ?, agent_profile_snapshot = ?, executor_snapshot = ?, environment_snapshot = ?, repository_snapshot = ?, state = ?, error_message = ?, completed_at = ?, updated_at = ?, is_primary = ?, review_status = ?, is_passthrough = ?, task_environment_id = ? - WHERE id = ? - `), agentProfileID, session.ExecutionProfileID, session.ExecutorID, session.ExecutorProfileID, session.EnvironmentID, + WHERE id = ?` + args := []interface{}{agentProfileID, session.ExecutionProfileID, session.ExecutorID, session.ExecutorProfileID, session.EnvironmentID, session.RepositoryID, session.BaseBranch, session.BaseCommitSHA, session.WorkspacePath, string(agentProfileSnapshotJSON), string(executorSnapshotJSON), string(environmentSnapshotJSON), string(repositorySnapshotJSON), string(session.State), session.ErrorMessage, session.CompletedAt, session.UpdatedAt, dialect.BoolToInt(session.IsPrimary), session.ReviewStatus, dialect.BoolToInt(session.IsPassthrough), session.TaskEnvironmentID, - session.ID) + session.ID} + if expected != nil { + query += " AND state = ?" + args = append(args, string(*expected)) + } + result, err := exec.ExecContext(ctx, r.db.Rebind(query), args...) if err != nil { - return err + return false, err } - - rows, _ := result.RowsAffected() - if rows == 0 { - return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, session.ID) + rows, err := result.RowsAffected() + if err != nil { + return false, err } - return nil + return rows > 0, nil } // RenameTaskSession updates just the user-supplied name of an agent session. @@ -541,14 +574,37 @@ func (r *Repository) RenameTaskSession(ctx context.Context, id, name string) err return nil } +// UpdateTaskSessionAgentProfileSnapshot updates only the profile snapshot. +// Runtime capability events may carry a stale full session row, so keeping +// this write narrow prevents them from overwriting terminal state. +func (r *Repository) UpdateTaskSessionAgentProfileSnapshot( + ctx context.Context, + id string, + snapshot map[string]interface{}, +) error { + snapshotJSON, err := json.Marshal(snapshot) + if err != nil { + return fmt.Errorf("failed to serialize agent profile snapshot: %w", err) + } + result, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE task_sessions + SET agent_profile_snapshot = ?, updated_at = ? + WHERE id = ? + `), string(snapshotJSON), time.Now().UTC(), id) + if err != nil { + return err + } + rows, _ := result.RowsAffected() + if rows == 0 { + return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, id) + } + return nil +} + // UpdateTaskSessionState updates just the state and error message of an agent session func (r *Repository) UpdateTaskSessionState(ctx context.Context, id string, status models.TaskSessionState, errorMessage string) error { now := time.Now().UTC() - - var completedAt *time.Time - if status == models.TaskSessionStateCompleted || status == models.TaskSessionStateFailed || status == models.TaskSessionStateCancelled { - completedAt = &now - } + completedAt := completedAtForTaskSessionState(status, now) result, err := r.db.ExecContext(ctx, r.db.Rebind(` UPDATE task_sessions SET state = ?, error_message = ?, completed_at = ?, updated_at = ? WHERE id = ? @@ -564,6 +620,65 @@ func (r *Repository) UpdateTaskSessionState(ctx context.Context, id string, stat return nil } +// UpdateTaskSessionStateIfCurrent transitions a session only when its state +// still matches the caller's observation. The returned timestamp belongs to +// the committed write and remains authoritative even if a later read fails. +func (r *Repository) UpdateTaskSessionStateIfCurrent( + ctx context.Context, + id string, + expected, status models.TaskSessionState, + errorMessage string, +) (bool, time.Time, error) { + now := time.Now().UTC() + completedAt := completedAtForTaskSessionState(status, now) + result, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE task_sessions + SET state = ?, error_message = ?, completed_at = ?, updated_at = ? + WHERE id = ? AND state = ? + `), string(status), errorMessage, completedAt, now, id, string(expected)) + if err != nil { + return false, time.Time{}, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, time.Time{}, err + } + return rows > 0, now, nil +} + +// CancelActiveTaskSession atomically transitions one active session to +// CANCELLED. A false result means the row exists in a non-active state or was +// concurrently changed before this conditional write; callers re-read to +// distinguish those cases from a missing row. The returned timestamp belongs +// to the committed cancellation, so accepting callers never need a fallible +// post-write read before scheduling teardown. +func (r *Repository) CancelActiveTaskSession(ctx context.Context, id, reason string) (bool, time.Time, error) { + now := time.Now().UTC() + result, err := r.db.ExecContext(ctx, r.db.Rebind(` + UPDATE task_sessions + SET state = ?, error_message = ?, completed_at = ?, updated_at = ? + WHERE id = ? + AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') + `), string(models.TaskSessionStateCancelled), reason, now, now, id) + if err != nil { + return false, time.Time{}, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, time.Time{}, err + } + return rows > 0, now, nil +} + +func completedAtForTaskSessionState(status models.TaskSessionState, now time.Time) *time.Time { + if status == models.TaskSessionStateCompleted || + status == models.TaskSessionStateFailed || + status == models.TaskSessionStateCancelled { + return &now + } + return nil +} + // CancelActiveTaskSessionsByTaskID transitions every active session of a task // (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT) to CANCELLED, returning the // number of rows changed. The transition is a pure DB state change and does not diff --git a/apps/backend/internal/task/repository/sqlite/session_test.go b/apps/backend/internal/task/repository/sqlite/session_test.go index 0143c104e3..03f842c125 100644 --- a/apps/backend/internal/task/repository/sqlite/session_test.go +++ b/apps/backend/internal/task/repository/sqlite/session_test.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/db/dialect" "github.com/kandev/kandev/internal/task/models" + "github.com/stretchr/testify/require" ) func newRepoForSessionTests(t *testing.T) *Repository { @@ -736,6 +737,101 @@ func sessionState(t *testing.T, repo *Repository, sessionID string) string { return state } +func TestCancelActiveTaskSessionIsTerminalSafe(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-cas", "session-running", "turn-cas") + if err := repo.UpdateTaskSessionState(ctx, "session-running", models.TaskSessionStateRunning, ""); err != nil { + t.Fatalf("seed running state: %v", err) + } + insertSession(t, repo, "session-completed", "task-cas", string(models.TaskSessionStateCompleted)) + + changed, cancelledAt, err := repo.CancelActiveTaskSession(ctx, "session-running", "coordinator stop") + if err != nil { + t.Fatalf("cancel running session: %v", err) + } + if !changed { + t.Fatal("running session was not cancelled") + } + if got := sessionState(t, repo, "session-running"); got != string(models.TaskSessionStateCancelled) { + t.Fatalf("running session state = %q, want CANCELLED", got) + } + cancelled, err := repo.GetTaskSession(ctx, "session-running") + if err != nil { + t.Fatalf("read cancelled session: %v", err) + } + if !cancelled.UpdatedAt.Equal(cancelledAt) { + t.Fatalf("cancel timestamp = %s, stored updated_at = %s", cancelledAt, cancelled.UpdatedAt) + } + + changed, _, err = repo.CancelActiveTaskSession(ctx, "session-completed", "coordinator stop") + if err != nil { + t.Fatalf("cancel completed session: %v", err) + } + if changed { + t.Fatal("completed session reported a cancellation") + } + if got := sessionState(t, repo, "session-completed"); got != string(models.TaskSessionStateCompleted) { + t.Fatalf("completed session state = %q, want COMPLETED", got) + } +} + +func TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-state-cas", "session-state-cas", "turn-state-cas") + if err := repo.UpdateTaskSessionState(ctx, "session-state-cas", models.TaskSessionStateRunning, ""); err != nil { + t.Fatalf("seed running state: %v", err) + } + if changed, _, err := repo.CancelActiveTaskSession(ctx, "session-state-cas", "coordinator stop"); err != nil || !changed { + t.Fatalf("cancel session: changed=%v err=%v", changed, err) + } + + changed, _, err := repo.UpdateTaskSessionStateIfCurrent( + ctx, + "session-state-cas", + models.TaskSessionStateRunning, + models.TaskSessionStateWaitingForInput, + "", + ) + if err != nil { + t.Fatalf("stale conditional update: %v", err) + } + if changed { + t.Fatal("stale RUNNING writer changed a CANCELLED session") + } + if got := sessionState(t, repo, "session-state-cas"); got != string(models.TaskSessionStateCancelled) { + t.Fatalf("session state = %q, want CANCELLED", got) + } +} + +func TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-full-row-cas", "session-full-row-cas", "turn-full-row-cas") + require.NoError(t, repo.UpdateTaskSessionState( + ctx, "session-full-row-cas", models.TaskSessionStateRunning, "", + )) + stale, err := repo.GetTaskSession(ctx, "session-full-row-cas") + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateRunning, stale.State) + changed, _, err := repo.CancelActiveTaskSession(ctx, stale.ID, "stopped by parent task via MCP") + require.NoError(t, err) + require.True(t, changed) + + stale.State = models.TaskSessionStateStarting + stale.ExecutorID = "late-executor" + changed, err = repo.UpdateTaskSessionIfCurrentState( + ctx, stale, models.TaskSessionStateRunning, + ) + require.NoError(t, err) + require.False(t, changed) + stored, err := repo.GetTaskSession(ctx, stale.ID) + require.NoError(t, err) + require.Equal(t, models.TaskSessionStateCancelled, stored.State) + require.Empty(t, stored.ExecutorID) +} + func TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index 204a1c538d..ac1d0f6e74 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -1233,6 +1233,152 @@ func (r *Repository) UpdateTaskState(ctx context.Context, id string, state v1.Ta return nil } +// UpdateTaskStateIfSessionState atomically ties a task-state write to the +// current state of one owning session and to the task remaining unarchived. +// The task-state predicate makes the returned old state authoritative for +// task.state_changed events; a concurrent task or session write retries with +// a fresh snapshot or makes this update a no-op. +func (r *Repository) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (v1.TaskState, bool, error) { + for attempt := range updateTaskStateIfNotArchivedMaxAttempts { + if attempt > 0 { + select { + case <-ctx.Done(): + return "", false, ctx.Err() + case <-time.After(updateTaskStateIfNotArchivedBackoff): + } + } + oldState, updated, retry, err := r.tryUpdateTaskStateIfSessionState( + ctx, taskID, sessionID, expectedSessionState, state, + ) + if err != nil || !retry { + return oldState, updated, err + } + } + return "", false, fmt.Errorf("update task state if session state: exceeded %d attempts for task %s", + updateTaskStateIfNotArchivedMaxAttempts, taskID) +} + +func (r *Repository) tryUpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (oldState v1.TaskState, updated, retry bool, err error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return "", false, false, err + } + defer func() { _ = tx.Rollback() }() + + var archivedAt sql.NullTime + var currentSessionState models.TaskSessionState + err = tx.QueryRowContext(ctx, r.db.Rebind(` + SELECT tasks.state, tasks.archived_at, task_sessions.state + FROM tasks + JOIN task_sessions ON task_sessions.task_id = tasks.id + WHERE tasks.id = ? AND task_sessions.id = ? + `), taskID, sessionID).Scan(&oldState, &archivedAt, ¤tSessionState) + if errors.Is(err, sql.ErrNoRows) { + return "", false, false, nil + } + if err != nil { + return "", false, false, err + } + if archivedAt.Valid || currentSessionState != expectedSessionState { + return oldState, false, false, nil + } + + result, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE tasks + SET state = ?, updated_at = ? + WHERE id = ? + AND state = ? + AND archived_at IS NULL + AND EXISTS ( + SELECT 1 + FROM task_sessions + WHERE task_sessions.id = ? + AND task_sessions.task_id = tasks.id + AND task_sessions.state = ? + ) + `), state, time.Now().UTC(), taskID, oldState, sessionID, expectedSessionState) + if err != nil { + if isRetryableStateRaceError(err) { + return oldState, false, true, nil + } + return "", false, false, err + } + rows, err := result.RowsAffected() + if err != nil { + return "", false, false, err + } + if rows == 0 { + return oldState, false, true, nil + } + if err := tx.Commit(); err != nil { + return "", false, false, err + } + return oldState, true, false, nil +} + +// RestoreTaskMessageRollbackIfSessionState atomically restores the two task +// fields changed by message_task's on-turn-start preparation, but only while +// the owning session remains in the state restored by the same rollback. A +// coordinator cancellation therefore prevents a late dispatch-failure +// rollback from moving the task out of REVIEW or rewinding its workflow step. +func (r *Repository) RestoreTaskMessageRollbackIfSessionState( + ctx context.Context, + task *models.Task, + sessionID string, + expectedSessionState models.TaskSessionState, +) (bool, error) { + if task == nil { + return false, errors.New("restore task message rollback: task is nil") + } + updatedAt := time.Now().UTC() + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + result, err := tx.ExecContext(ctx, r.db.Rebind(` + UPDATE tasks + SET state = ?, workflow_step_id = ?, updated_at = ? + WHERE id = ? + AND EXISTS ( + SELECT 1 + FROM task_sessions + WHERE task_sessions.id = ? + AND task_sessions.task_id = tasks.id + AND task_sessions.state = ? + ) + `), task.State, task.WorkflowStepID, updatedAt, task.ID, sessionID, expectedSessionState) + if err != nil { + return false, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, err + } + if rows == 0 { + return false, tx.Commit() + } + if err := syncRunnerInTx(ctx, tx, task.WorkflowStepID, task.ID, task.AssigneeAgentProfileID); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + task.UpdatedAt = updatedAt + return true, nil +} + // UpdateTaskStateIfCurrentIn transitions state inside a transaction, re-checking // the current state on write so concurrent handlers cannot clobber a task that // moved out of allowed between read and update. The write is also scoped to diff --git a/apps/backend/internal/task/repository/sqlite/task_state_cas_test.go b/apps/backend/internal/task/repository/sqlite/task_state_cas_test.go index 8036a2be5a..fb2de2ec9e 100644 --- a/apps/backend/internal/task/repository/sqlite/task_state_cas_test.go +++ b/apps/backend/internal/task/repository/sqlite/task_state_cas_test.go @@ -9,9 +9,196 @@ import ( "github.com/jmoiron/sqlx" "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) +func TestUpdateTaskStateIfSessionState_RequiresCurrentOwningSession(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-session-cas") + insertSession(t, repo, "session-cas", "task-session-cas", string(models.TaskSessionStateWaitingForInput)) + + if err := repo.UpdateTaskState(ctx, "task-session-cas", v1.TaskStateReview); err != nil { + t.Fatalf("seed REVIEW: %v", err) + } + oldState, updated, err := repo.UpdateTaskStateIfSessionState( + ctx, + "task-session-cas", + "session-cas", + models.TaskSessionStateStarting, + v1.TaskStateInProgress, + ) + if err != nil { + t.Fatalf("UpdateTaskStateIfSessionState: %v", err) + } + if updated { + t.Fatal("expected stale STARTING writer to be rejected after clarification paused the session") + } + if oldState != v1.TaskStateReview { + t.Fatalf("old state = %q, want %q", oldState, v1.TaskStateReview) + } + task, err := repo.GetTask(ctx, "task-session-cas") + if err != nil { + t.Fatalf("get task: %v", err) + } + if task.State != v1.TaskStateReview { + t.Fatalf("task state = %q, want %q", task.State, v1.TaskStateReview) + } +} + +func TestUpdateTaskStateIfSessionState_SkipsArchivedTask(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-session-archived") + insertSession(t, repo, "session-archived", "task-session-archived", string(models.TaskSessionStateStarting)) + + if err := repo.UpdateTaskState(ctx, "task-session-archived", v1.TaskStateReview); err != nil { + t.Fatalf("seed REVIEW: %v", err) + } + if err := repo.ArchiveTask(ctx, "task-session-archived"); err != nil { + t.Fatalf("archive task: %v", err) + } + oldState, updated, err := repo.UpdateTaskStateIfSessionState( + ctx, + "task-session-archived", + "session-archived", + models.TaskSessionStateStarting, + v1.TaskStateInProgress, + ) + if err != nil { + t.Fatalf("UpdateTaskStateIfSessionState: %v", err) + } + if updated { + t.Fatal("expected archived task write to be rejected") + } + if oldState != v1.TaskStateReview { + t.Fatalf("old state = %q, want %q", oldState, v1.TaskStateReview) + } +} + +func TestUpdateTaskStateIfSessionState_UpdatesMatchingSession(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-session-match") + insertSession(t, repo, "session-match", "task-session-match", string(models.TaskSessionStateStarting)) + + if err := repo.UpdateTaskState(ctx, "task-session-match", v1.TaskStateReview); err != nil { + t.Fatalf("seed REVIEW: %v", err) + } + oldState, updated, err := repo.UpdateTaskStateIfSessionState( + ctx, + "task-session-match", + "session-match", + models.TaskSessionStateStarting, + v1.TaskStateInProgress, + ) + if err != nil { + t.Fatalf("UpdateTaskStateIfSessionState: %v", err) + } + if !updated { + t.Fatal("expected matching session to permit task update") + } + if oldState != v1.TaskStateReview { + t.Fatalf("old state = %q, want %q", oldState, v1.TaskStateReview) + } + task, err := repo.GetTask(ctx, "task-session-match") + if err != nil { + t.Fatalf("get task: %v", err) + } + if task.State != v1.TaskStateInProgress { + t.Fatalf("task state = %q, want %q", task.State, v1.TaskStateInProgress) + } +} + +func TestRestoreTaskMessageRollbackIfSessionState_RejectionDoesNotMutateCandidate(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-rollback-rejected") + insertSession(t, repo, "session-rollback-rejected", "task-rollback-rejected", string(models.TaskSessionStateWaitingForInput)) + if _, err := repo.db.Exec(` + UPDATE tasks SET state = ?, workflow_step_id = ? WHERE id = ? + `, v1.TaskStateInProgress, "current-step", "task-rollback-rejected"); err != nil { + t.Fatalf("seed current task fields: %v", err) + } + + originalUpdatedAt := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC) + candidate := &models.Task{ + ID: "task-rollback-rejected", + State: v1.TaskStateReview, + WorkflowStepID: "restored-step", + UpdatedAt: originalUpdatedAt, + } + updated, err := repo.RestoreTaskMessageRollbackIfSessionState( + ctx, + candidate, + "session-rollback-rejected", + models.TaskSessionStateRunning, + ) + if err != nil { + t.Fatalf("RestoreTaskMessageRollbackIfSessionState: %v", err) + } + if updated { + t.Fatal("expected session-state mismatch to reject rollback") + } + if !candidate.UpdatedAt.Equal(originalUpdatedAt) { + t.Fatalf("rejected candidate UpdatedAt = %v, want unchanged %v", candidate.UpdatedAt, originalUpdatedAt) + } + persisted, err := repo.GetTask(ctx, candidate.ID) + if err != nil { + t.Fatalf("get task: %v", err) + } + if persisted.State != v1.TaskStateInProgress || persisted.WorkflowStepID != "current-step" { + t.Fatalf("rejected rollback persisted state/step = %q/%q", persisted.State, persisted.WorkflowStepID) + } +} + +func TestRestoreTaskMessageRollbackIfSessionState_RestoresFieldsAndRunner(t *testing.T) { + repo := newRepoForHealTests(t) + ctx := context.Background() + insertTask(t, repo.db, "task-rollback-success") + insertSession(t, repo, "session-rollback-success", "task-rollback-success", string(models.TaskSessionStateRunning)) + if _, err := repo.db.Exec(` + UPDATE tasks SET state = ?, workflow_step_id = ? WHERE id = ? + `, v1.TaskStateInProgress, "current-step", "task-rollback-success"); err != nil { + t.Fatalf("seed current task fields: %v", err) + } + + originalUpdatedAt := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC) + candidate := &models.Task{ + ID: "task-rollback-success", + State: v1.TaskStateReview, + WorkflowStepID: "restored-step", + AssigneeAgentProfileID: "profile-restored", + UpdatedAt: originalUpdatedAt, + } + updated, err := repo.RestoreTaskMessageRollbackIfSessionState( + ctx, + candidate, + "session-rollback-success", + models.TaskSessionStateRunning, + ) + if err != nil { + t.Fatalf("RestoreTaskMessageRollbackIfSessionState: %v", err) + } + if !updated { + t.Fatal("expected matching session state to restore task") + } + if !candidate.UpdatedAt.After(originalUpdatedAt) { + t.Fatalf("successful candidate UpdatedAt = %v, want after %v", candidate.UpdatedAt, originalUpdatedAt) + } + persisted, err := repo.GetTask(ctx, candidate.ID) + if err != nil { + t.Fatalf("get task: %v", err) + } + if persisted.State != v1.TaskStateReview || persisted.WorkflowStepID != "restored-step" { + t.Fatalf("restored state/step = %q/%q", persisted.State, persisted.WorkflowStepID) + } + if persisted.AssigneeAgentProfileID != "profile-restored" { + t.Fatalf("restored runner = %q, want profile-restored", persisted.AssigneeAgentProfileID) + } +} + // TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask reproduces the TOCTOU race // flagged in PR #1706 review: a caller's earlier archived-state guard (a // plain GetTask read) can observe archived_at == NULL, then have ArchiveTask diff --git a/apps/backend/internal/task/service/handoff_workspace_test.go b/apps/backend/internal/task/service/handoff_workspace_test.go index 672534de65..1b108a35bb 100644 --- a/apps/backend/internal/task/service/handoff_workspace_test.go +++ b/apps/backend/internal/task/service/handoff_workspace_test.go @@ -395,6 +395,12 @@ func (r *phase4TaskRepo) UpdateTaskState(context.Context, string, v1.TaskState) r.panicNotUsed("UpdateTaskState") return nil } +func (r *phase4TaskRepo) UpdateTaskStateIfSessionState( + context.Context, string, string, models.TaskSessionState, v1.TaskState, +) (v1.TaskState, bool, error) { + r.panicNotUsed("UpdateTaskStateIfSessionState") + return "", false, nil +} func (r *phase4TaskRepo) UpdateTaskStateIfCurrentIn(context.Context, string, v1.TaskState, []v1.TaskState) (v1.TaskState, bool, error) { r.panicNotUsed("UpdateTaskStateIfCurrentIn") return "", false, nil diff --git a/apps/backend/internal/task/service/message_rollback_test.go b/apps/backend/internal/task/service/message_rollback_test.go new file mode 100644 index 0000000000..01d32ab869 --- /dev/null +++ b/apps/backend/internal/task/service/message_rollback_test.go @@ -0,0 +1,45 @@ +package service + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" +) + +func TestRestoreTaskMessageRollback_RejectionReturnsPersistedTask(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + sessionID := setupTestSession(t, repo) + require.NoError(t, repo.UpdateTaskSessionState( + ctx, + sessionID, + models.TaskSessionStateWaitingForInput, + "", + )) + + before, err := repo.GetTask(ctx, "task-123") + require.NoError(t, err) + returned, updated, err := svc.RestoreTaskMessageRollback( + ctx, + before.ID, + sessionID, + models.TaskSessionStateRunning, + v1.TaskStateReview, + "restored-step", + ) + require.NoError(t, err) + require.False(t, updated) + require.Equal(t, before.State, returned.State) + require.Equal(t, before.WorkflowStepID, returned.WorkflowStepID) + require.Equal(t, before.UpdatedAt, returned.UpdatedAt) + + persisted, err := repo.GetTask(ctx, before.ID) + require.NoError(t, err) + require.Equal(t, before.State, persisted.State) + require.Equal(t, before.WorkflowStepID, persisted.WorkflowStepID) + require.Empty(t, eventBus.GetPublishedEvents()) +} diff --git a/apps/backend/internal/task/service/resource_cleanup_jobs.go b/apps/backend/internal/task/service/resource_cleanup_jobs.go index 693e428479..aa53e28171 100644 --- a/apps/backend/internal/task/service/resource_cleanup_jobs.go +++ b/apps/backend/internal/task/service/resource_cleanup_jobs.go @@ -373,7 +373,15 @@ func (s *Service) executeTaskResourceCleanupJob( job *models.TaskResourceCleanupJob, snapshot taskResourceCleanupSnapshot, ) error { - targets := restoreStopTargets(snapshot.StopTargets) + targets, err := s.refreshTaskRuntimeStopTargets( + ctx, + job.TaskID, + restoreStopTargets(snapshot.StopTargets), + ) + if err != nil { + return fmt.Errorf("refresh task cleanup runtime inventory: %w", err) + } + s.registerTaskRuntimeStopOwners(targets, true) failedStops := s.stopTaskRuntimeTargets(ctx, job.TaskID, targets, taskResourceCleanupStopReason(job.Trigger), "task cleanup runtime stop failed") if cancelled, err := s.cancelIfTaskUnarchived(ctx, job); err != nil || cancelled { return err diff --git a/apps/backend/internal/task/service/resource_cleanup_jobs_test.go b/apps/backend/internal/task/service/resource_cleanup_jobs_test.go index 9ba63aa6bd..db8fb65ea4 100644 --- a/apps/backend/internal/task/service/resource_cleanup_jobs_test.go +++ b/apps/backend/internal/task/service/resource_cleanup_jobs_test.go @@ -75,6 +75,7 @@ func (g coordinatorCleanupActivityGate) AcquireTaskResourceCleanup( } func (s *activityCleanupStopper) StopTask(context.Context, string, string, bool) error { return nil } +func (s *activityCleanupStopper) RegisterExecutionStopOwner(string, string, bool) {} func (s *activityCleanupStopper) StopSession(context.Context, string, string, bool) error { return nil } diff --git a/apps/backend/internal/task/service/service.go b/apps/backend/internal/task/service/service.go index cdcf52c7b7..25dc052686 100644 --- a/apps/backend/internal/task/service/service.go +++ b/apps/backend/internal/task/service/service.go @@ -41,6 +41,9 @@ type TaskExecutionStopper interface { StopTask(ctx context.Context, taskID, reason string, force bool) error StopSession(ctx context.Context, sessionID, reason string, force bool) error StopExecution(ctx context.Context, executionID, reason string, force bool) error + // RegisterExecutionStopOwner records exact teardown ownership before a + // terminal session mutation. It never replaces the explicit stop call. + RegisterExecutionStopOwner(sessionID, executionID string, force bool) } // TaskResourceCleanupActivityGate serializes durable cleanup with install-wide maintenance. diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index 80cda1ef0d..b681c31c40 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" v1 "github.com/kandev/kandev/pkg/api/v1" "go.uber.org/zap" @@ -894,6 +895,56 @@ func (s *Service) UpdateTask(ctx context.Context, id string, req *UpdateTaskRequ return task, nil } +type taskMessageRollbackRepository interface { + RestoreTaskMessageRollbackIfSessionState( + ctx context.Context, + task *models.Task, + sessionID string, + expectedSessionState models.TaskSessionState, + ) (bool, error) +} + +// RestoreTaskMessageRollback restores message_task's task-state/workflow-step +// snapshot only while ownerSessionID still has expectedSessionState. It is a +// narrow compensation API: the repository predicate and both task-field +// writes share one SQL statement, so coordinator cancellation cannot be +// overwritten between a state check and the rollback write. +func (s *Service) RestoreTaskMessageRollback( + ctx context.Context, + taskID, ownerSessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, + workflowStepID string, +) (*models.Task, bool, error) { + repo, ok := s.tasks.(taskMessageRollbackRepository) + if !ok { + return nil, false, errors.New("task repository does not support guarded message rollback") + } + task, err := s.tasks.GetTask(ctx, taskID) + if err != nil { + return nil, false, err + } + oldState := task.State + restoredTask := *task + restoredTask.State = state + restoredTask.WorkflowStepID = workflowStepID + updated, err := repo.RestoreTaskMessageRollbackIfSessionState( + ctx, + &restoredTask, + ownerSessionID, + expectedSessionState, + ) + if err != nil || !updated { + return task, updated, err + } + + if restoredTask.State != oldState { + s.publishTaskEvent(ctx, events.TaskStateChanged, &restoredTask, &oldState) + } + s.publishTaskEvent(ctx, events.TaskUpdated, &restoredTask, nil) + return &restoredTask, true, nil +} + // ArchiveTask archives a task by setting its archived_at timestamp. // The task remains in the DB but is excluded from active board views. // Active agent sessions are stopped and worktrees cleaned up in background. @@ -970,6 +1021,11 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error { return err } + // Register the exact inventory before CANCELLED becomes visible. A launch + // persistence loser can then distinguish these owned executions from one + // that raced in after the snapshot and must clean itself up. + s.registerTaskRuntimeStopOwners(stopTargets, true) + // 3b. Finalize active sessions in the DB. The async cleanup below tears down // the agent processes; this records the terminal session state, which // process teardown does not persist on its own. @@ -1009,6 +1065,22 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error { return nil } +func (s *Service) registerTaskRuntimeStopOwners(stopTargets []taskStopTarget, force bool) { + if s.executionStopper == nil { + return + } + for _, target := range stopTargets { + if target.sessionID == "" || target.executionID == "" { + continue + } + s.executionStopper.RegisterExecutionStopOwner( + target.sessionID, + target.executionID, + force, + ) + } +} + // DeleteTask deletes a task and publishes a task.deleted event. // For fast UI response, the DB delete and event publish happen synchronously, // while agent stopping and worktree cleanup happen asynchronously. @@ -1282,6 +1354,16 @@ func (s *Service) runTaskCleanup( cleanupStart := time.Now() cleanupCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() + refreshedTargets, err := s.refreshTaskRuntimeStopTargets(cleanupCtx, id, stopTargets) + if err != nil { + s.logger.Warn(cleanupMsg+" deferred because runtime inventory refresh failed", + zap.String("task_id", id), + zap.Error(err)) + s.signalCleanupDoneForTest() + return + } + stopTargets = refreshedTargets + s.registerTaskRuntimeStopOwners(stopTargets, true) failedStops := s.stopTaskRuntimeTargets(cleanupCtx, id, stopTargets, stopReason, stopFailMsg) @@ -1375,6 +1457,87 @@ func (s *Service) buildStopTargets(ctx context.Context, taskID string, activeSes return targets, nil } +// refreshTaskRuntimeStopTargets merges the durable/pre-mutation snapshot with +// the live runtime inventory immediately before teardown. A launch may rotate +// an execution after the snapshot but before archive/delete becomes visible; +// keeping every exact ID makes cleanup safe across that race and worker +// restarts. A session-only fallback is retained only when no exact ID exists +// for that session; otherwise it would observe absence after the exact stop and +// keep a durable cleanup retrying forever. +func (s *Service) refreshTaskRuntimeStopTargets( + ctx context.Context, + taskID string, + snapshot []taskStopTarget, +) ([]taskStopTarget, error) { + if s.executionStopper == nil || s.executors == nil { + return snapshot, nil + } + runningRows, err := s.executors.ListExecutorsRunningByTaskID(ctx, taskID) + if err != nil { + return nil, err + } + return mergeTaskStopTargets(taskStopTargetsFromRunningRows(runningRows), snapshot), nil +} + +func taskStopTargetsFromRunningRows(runningRows []*models.ExecutorRunning) []taskStopTarget { + targets := make([]taskStopTarget, 0, len(runningRows)) + for _, running := range runningRows { + if running == nil { + continue + } + targets = append(targets, taskStopTarget{ + sessionID: strings.TrimSpace(running.SessionID), + executionID: strings.TrimSpace(running.AgentExecutionID), + }) + } + return targets +} + +func mergeTaskStopTargets(live, snapshot []taskStopTarget) []taskStopTarget { + sessionsWithExactTarget := exactTaskStopTargetSessions(live, snapshot) + targets := make([]taskStopTarget, 0, len(live)+len(snapshot)) + seen := make(map[string]int, len(live)+len(snapshot)) + appendTarget := func(target taskStopTarget) { + target.sessionID = strings.TrimSpace(target.sessionID) + target.executionID = strings.TrimSpace(target.executionID) + if target.sessionID == "" { + return + } + if target.executionID == "" { + if _, hasExact := sessionsWithExactTarget[target.sessionID]; hasExact { + return + } + } + key := target.sessionID + "\x00" + target.executionID + if index, exists := seen[key]; exists { + targets[index].terminal = targets[index].terminal || target.terminal + return + } + seen[key] = len(targets) + targets = append(targets, target) + } + for _, target := range live { + appendTarget(target) + } + for _, target := range snapshot { + appendTarget(target) + } + return targets +} + +func exactTaskStopTargetSessions(targetSets ...[]taskStopTarget) map[string]struct{} { + exact := make(map[string]struct{}) + for _, targets := range targetSets { + for _, target := range targets { + sessionID := strings.TrimSpace(target.sessionID) + if sessionID != "" && strings.TrimSpace(target.executionID) != "" { + exact[sessionID] = struct{}{} + } + } + } + return exact +} + func (s *Service) stopTaskRuntimeTargets(ctx context.Context, taskID string, stopTargets []taskStopTarget, stopReason, stopFailMsg string) map[string]struct{} { failedStops := make(map[string]struct{}) if s.executionStopper == nil || len(stopTargets) == 0 { @@ -1386,6 +1549,9 @@ func (s *Service) stopTaskRuntimeTargets(ctx context.Context, taskID string, sto } if target.executionID != "" { if err := s.executionStopper.StopExecution(ctx, target.executionID, stopReason, true); err != nil { + if runtimeStopAlreadyComplete(err) { + continue + } if target.terminal { s.logger.Debug("stop failed for terminal session execution (expected), proceeding with cleanup", zap.String("task_id", taskID), @@ -1420,6 +1586,10 @@ func (s *Service) stopTaskRuntimeTargets(ctx context.Context, taskID string, sto return failedStops } +func runtimeStopAlreadyComplete(err error) bool { + return errors.Is(err, runtimeapi.ErrNotFound) +} + // performTaskCleanup handles post-deletion cleanup operations. // Handles worktree cleanup, executor_running records, and quick-chat workspace directories. // Agent stopping is handled separately in the DeleteTask background goroutine. diff --git a/apps/backend/internal/task/service/service_tasks_stop_test.go b/apps/backend/internal/task/service/service_tasks_stop_test.go index 002d4543ae..de12e4338c 100644 --- a/apps/backend/internal/task/service/service_tasks_stop_test.go +++ b/apps/backend/internal/task/service/service_tasks_stop_test.go @@ -3,8 +3,10 @@ package service import ( "context" "errors" + "fmt" "testing" + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/task/repository" @@ -144,16 +146,91 @@ func TestBuildStopTargets_TerminalSessionWithoutExecutorRow(t *testing.T) { } } +func TestRefreshTaskRuntimeStopTargets_MergesLateExecutionBeforeSnapshot(t *testing.T) { + svc, _, _ := createTestService(t) + svc.executionStopper = &stubStopper{} + svc.executors = &stubExecutors{ + runningByTaskID: []*models.ExecutorRunning{ + {SessionID: "sess-1", AgentExecutionID: "exec-late"}, + }, + } + + targets, err := svc.refreshTaskRuntimeStopTargets( + context.Background(), + "task-1", + []taskStopTarget{{sessionID: "sess-1", executionID: "exec-snapshot", terminal: true}}, + ) + if err != nil { + t.Fatalf("refreshTaskRuntimeStopTargets error: %v", err) + } + if len(targets) != 2 { + t.Fatalf("targets = %#v, want late and snapshot executions", targets) + } + if targets[0].sessionID != "sess-1" || targets[0].executionID != "exec-late" || targets[0].terminal { + t.Fatalf("first target = %#v, want live late execution", targets[0]) + } + if targets[1].sessionID != "sess-1" || targets[1].executionID != "exec-snapshot" || !targets[1].terminal { + t.Fatalf("second target = %#v, want retained snapshot execution", targets[1]) + } +} + +func TestRefreshTaskRuntimeStopTargets_ExactRuntimeSupersedesEmptySessionFallback(t *testing.T) { + svc, _, _ := createTestService(t) + stopper := &stubStopper{ + stopSessionErr: fmt.Errorf("not registered: %w", runtimeapi.ErrNotFound), + } + svc.executionStopper = stopper + svc.executors = &stubExecutors{ + runningByTaskID: []*models.ExecutorRunning{ + {SessionID: "sess-1", AgentExecutionID: "exec-late"}, + }, + } + + targets, err := svc.refreshTaskRuntimeStopTargets( + context.Background(), + "task-1", + []taskStopTarget{{sessionID: "sess-1"}}, + ) + if err != nil { + t.Fatalf("refreshTaskRuntimeStopTargets error: %v", err) + } + failed := svc.stopTaskRuntimeTargets( + context.Background(), + "task-1", + targets, + "archive", + "stop failed", + ) + if len(failed) != 0 { + t.Fatalf("late exact execution left retryable fallback failure: %v", failed) + } + if stopper.stopExecutionCalls != 1 || stopper.stopSessionCalls != 0 { + t.Fatalf( + "stop calls = execution:%d session:%d, want execution:1 session:0", + stopper.stopExecutionCalls, + stopper.stopSessionCalls, + ) + } +} + // --- stopTaskRuntimeTargets --- // stubStopper is a minimal TaskExecutionStopper for tests. type stubStopper struct { - stopSessionErr error + stopSessionErr error + stopExecutionErr error + stopSessionCalls int + stopExecutionCalls int } -func (s *stubStopper) StopTask(_ context.Context, _, _ string, _ bool) error { return nil } -func (s *stubStopper) StopExecution(_ context.Context, _, _ string, _ bool) error { return nil } +func (s *stubStopper) StopTask(_ context.Context, _, _ string, _ bool) error { return nil } +func (s *stubStopper) StopExecution(_ context.Context, _, _ string, _ bool) error { + s.stopExecutionCalls++ + return s.stopExecutionErr +} +func (s *stubStopper) RegisterExecutionStopOwner(string, string, bool) {} func (s *stubStopper) StopSession(_ context.Context, _, _ string, _ bool) error { + s.stopSessionCalls++ return s.stopSessionErr } @@ -166,6 +243,8 @@ func (s *cancelAfterFirstStopper) StopTask(context.Context, string, string, bool return nil } +func (s *cancelAfterFirstStopper) RegisterExecutionStopOwner(string, string, bool) {} + func (s *cancelAfterFirstStopper) StopExecution(_ context.Context, id, _ string, _ bool) error { s.calls = append(s.calls, id) if len(s.calls) == 1 { @@ -197,6 +276,42 @@ func TestStopTaskRuntimeTargets_TerminalStopFailureDoesNotBlockCleanup(t *testin } } +func TestStopTaskRuntimeTargets_ExactRuntimeAbsenceIsIdempotent(t *testing.T) { + svc, _, _ := createTestService(t) + svc.executors = &stubExecutors{} + svc.executionStopper = &stubStopper{ + stopExecutionErr: fmt.Errorf("already stopped: %w", runtimeapi.ErrNotFound), + } + failed := svc.stopTaskRuntimeTargets( + context.Background(), + "task-a", + []taskStopTarget{{sessionID: "sess-running", executionID: "exec-running"}}, + "archive", + "stop failed", + ) + if len(failed) != 0 { + t.Errorf("exact runtime absence must be idempotent; got %v", failed) + } +} + +func TestStopTaskRuntimeTargets_SessionRuntimeAbsenceRemainsRetryable(t *testing.T) { + svc, _, _ := createTestService(t) + svc.executors = &stubExecutors{} + svc.executionStopper = &stubStopper{ + stopSessionErr: fmt.Errorf("not registered yet: %w", runtimeapi.ErrNotFound), + } + failed := svc.stopTaskRuntimeTargets( + context.Background(), + "task-a", + []taskStopTarget{{sessionID: "sess-running"}}, + "archive", + "stop failed", + ) + if _, ok := failed["sess-running"]; !ok { + t.Errorf("session-level absence must remain retryable; got %v", failed) + } +} + func TestStopTaskRuntimeTargets_NonTerminalStopFailureBlocksCleanup(t *testing.T) { svc, _, _ := createTestService(t) svc.executors = &stubExecutors{} diff --git a/apps/backend/internal/task/service/service_test.go b/apps/backend/internal/task/service/service_test.go index 9c664e2ab4..89d0e8cf73 100644 --- a/apps/backend/internal/task/service/service_test.go +++ b/apps/backend/internal/task/service/service_test.go @@ -1744,6 +1744,61 @@ func TestService_ArchiveTaskStopsExecutorRunningForTerminalSession(t *testing.T) } } +func TestService_ArchiveTaskClaimsExactExecutionBeforeCancellingSession(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + stopper := newRecordingTaskExecutionStopper() + stateAtClaim := make(chan models.TaskSessionState, 1) + stopper.claimExecutionFunc = func(sessionID, executionID string, force bool) bool { + if sessionID != "session-running" || executionID != "exec-running" || !force { + t.Fatalf("teardown claim = (%q, %q, %v)", sessionID, executionID, force) + } + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("load session at teardown claim: %v", err) + } + stateAtClaim <- session.State + return true + } + svc.SetExecutionStopper(stopper) + svc.setCleanupDoneForTestHook(make(chan struct{}, 1)) + + _ = repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}) + _ = repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-123", WorkspaceID: "ws-1", Name: "Workflow"}) + _ = repo.CreateTask(ctx, &models.Task{ID: "task-123", WorkspaceID: "ws-1", WorkflowID: "wf-123", WorkflowStepID: "step-123", Title: "Test", Priority: "medium"}) + _ = repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-running", TaskID: "task-123", State: models.TaskSessionStateRunning, + }) + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "session-running", + SessionID: "session-running", + TaskID: "task-123", + ExecutorID: "executor-1", + Runtime: agentruntime.RuntimeStandalone, + Status: models.ExecutorRunningStatusStarting, + AgentExecutionID: "exec-running", + }); err != nil { + t.Fatalf("seed executor running: %v", err) + } + + if err := svc.ArchiveTask(ctx, "task-123"); err != nil { + t.Fatalf("ArchiveTask: %v", err) + } + select { + case state := <-stateAtClaim: + if state != models.TaskSessionStateRunning { + t.Fatalf("session state at teardown claim = %q, want RUNNING", state) + } + case <-time.After(time.Second): + t.Fatal("archive did not claim its exact execution") + } + call := stopper.waitForStopExecution(t) + if call.executionID != "exec-running" || !call.force { + t.Fatalf("StopExecution call = %#v", call) + } + waitForCleanupDone(t, svc) +} + func TestService_DeleteTaskFailsClosedWhenRuntimeInventoryFails(t *testing.T) { svc, _, repo := createTestService(t) ctx := context.Background() @@ -1902,6 +1957,7 @@ type recordingTaskExecutionStopper struct { stopExecutionCh chan stopExecutionCall stopExecutionErr error stopExecutionErrByID map[string]error + claimExecutionFunc func(sessionID, executionID string, force bool) bool } func newRecordingTaskExecutionStopper() *recordingTaskExecutionStopper { @@ -1926,6 +1982,13 @@ func (s *recordingTaskExecutionStopper) StopExecution(_ context.Context, executi return s.stopExecutionErr } +func (s *recordingTaskExecutionStopper) RegisterExecutionStopOwner(sessionID, executionID string, force bool) { + if s.claimExecutionFunc == nil { + return + } + s.claimExecutionFunc(sessionID, executionID, force) +} + func (s *recordingTaskExecutionStopper) waitForStopExecution(t *testing.T) stopExecutionCall { t.Helper() select { diff --git a/apps/backend/internal/task/service/service_workflow.go b/apps/backend/internal/task/service/service_workflow.go index ba66f54cd9..3bfaa190bd 100644 --- a/apps/backend/internal/task/service/service_workflow.go +++ b/apps/backend/internal/task/service/service_workflow.go @@ -259,6 +259,36 @@ func (s *Service) UpdateTaskStateIfNotArchived( return true, nil } +// UpdateTaskStateIfSessionState transitions task state only while its owning +// session remains in the expected state and the task remains unarchived. +// Publishes task.state_changed only when the guarded write changes state. +func (s *Service) UpdateTaskStateIfSessionState( + ctx context.Context, + taskID, sessionID string, + expectedSessionState models.TaskSessionState, + state v1.TaskState, +) (bool, error) { + oldState, updated, err := s.tasks.UpdateTaskStateIfSessionState( + ctx, taskID, sessionID, expectedSessionState, state, + ) + if err != nil || !updated { + return false, err + } + if oldState == state { + return false, nil + } + + task, err := s.tasks.GetTask(ctx, taskID) + if err != nil { + return true, err + } + // Pin the state written by the guarded CAS so a later transition between + // commit and reload cannot produce a mismatched event payload. + task.State = state + s.publishTaskEvent(ctx, events.TaskStateChanged, task, &oldState) + return true, nil +} + // UpdateTaskMetadata updates only the metadata of a task (merges with existing) func (s *Service) UpdateTaskMetadata(ctx context.Context, id string, metadata map[string]interface{}) (*models.Task, error) { task, err := s.tasks.GetTask(ctx, id) diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index 4ca0f3ea30..e366f141a5 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -388,6 +388,7 @@ const ( ActionMCPArchiveTask = "mcp.archive_task" ActionMCPUpdateTaskState = "mcp.update_task_state" ActionMCPMessageTask = "mcp.message_task" + ActionMCPStopTask = "mcp.stop_task" ActionMCPSpawnSession = "mcp.spawn_session" ActionMCPGetTaskConversation = "mcp.get_task_conversation" ) diff --git a/apps/web/e2e/tests/office/onboarding-task-launch.spec.ts b/apps/web/e2e/tests/office/onboarding-task-launch.spec.ts index 99373b14ab..e15984f03d 100644 --- a/apps/web/e2e/tests/office/onboarding-task-launch.spec.ts +++ b/apps/web/e2e/tests/office/onboarding-task-launch.spec.ts @@ -68,43 +68,34 @@ const test = base.extend<{ testPage: Page }, OnboardingTaskFixtures>({ test.describe("Onboarding task launch", () => { test("task created during onboarding does not fail to launch", async ({ - officeApi, + apiClient, onboardingSeed, }) => { test.setTimeout(30_000); - // Poll the task state — it should progress past CREATED/SCHEDULING - // and NOT end up in FAILED. The scheduler picks up the run within - // ~5 seconds, so we give it up to 15 seconds. + // Office task status is workflow-owned and intentionally stays CREATED + // until the agent changes it. Poll the session to prove runtime launch. let lastState = ""; const deadline = Date.now() + 15_000; while (Date.now() < deadline) { - const issue = await officeApi.getTask(onboardingSeed.taskId); - const raw = issue as Record; - const inner = (raw.task as Record) ?? raw; - lastState = (inner.state as string) ?? (inner.status as string) ?? ""; + const { sessions } = await apiClient.listTaskSessions(onboardingSeed.taskId); + lastState = sessions[0]?.state ?? ""; - // FAILED means the agent launch failed — this is the regression. - // Status comes back as the canonical office lowercase form via - // dbStateToOfficeStatus (in_progress, in_review, done, …); legacy - // SCREAMING_SNAKE_CASE values are accepted for forward compatibility. - expect(lastState, "task should not enter FAILED state").not.toMatch(/^(failed|FAILED)$/); + expect(lastState, "session should not terminate before launch").not.toMatch( + /^(FAILED|CANCELLED)$/, + ); - // Any of these states means the agent launched successfully. - const launched = ["in_progress", "scheduling", "in_review", "done", "completed"]; - if (launched.includes(lastState.toLowerCase())) { - return; // Success — agent launched without workspace_path error + if (["RUNNING", "WAITING_FOR_INPUT", "IDLE", "COMPLETED"].includes(lastState)) { + return; } await new Promise((r) => setTimeout(r, 1_000)); } - // If we're still in CREATED after 15s, the scheduler didn't pick it up. - // That's a different issue, but not the regression we're testing. expect( - ["in_progress", "scheduling", "in_review", "done", "completed"], - `expected task to be scheduled, got "${lastState}"`, - ).toContain(lastState.toLowerCase()); + ["RUNNING", "WAITING_FOR_INPUT", "IDLE", "COMPLETED"], + `expected an agent session to launch, got "${lastState}"`, + ).toContain(lastState); }); }); diff --git a/docs/decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md b/docs/decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md new file mode 100644 index 0000000000..e45739dcf5 --- /dev/null +++ b/docs/decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md @@ -0,0 +1,56 @@ +# ADR-2026-07-19-reject-mcp-actions-on-raw-websocket: Reject MCP Actions on the Raw WebSocket + +**Status:** accepted +**Date:** 2026-07-19 +**Area:** backend, protocol + +## Context + +Kandev's agent stream, external MCP endpoint, and browser WebSocket gateway share +one backend action dispatcher. Task-mode MCP adapters inject task and session +identity into internal `mcp.*` action payloads, but a raw `/ws` client can +currently invoke those same actions and forge the injected fields. A +direct-parent-only stop or interrupt therefore cannot treat payload identity as +trusted unless the transport boundary is enforced. + +## Decision + +The browser/raw WebSocket client path in +`apps/backend/internal/gateway/websocket/client.go` rejects every action whose +name starts with `mcp.` before it reaches the shared dispatcher. Internal MCP +actions are reachable only through an MCP adapter: + +- mode-scoped MCP requests arriving through an authenticated agentctl agent + stream; +- the external MCP server's mode-specific registered tool set; or +- trusted in-process backend callers. + +Handlers may treat identity fields injected by the task MCP server as +transport-trusted only under this boundary. New `mcp.*` actions must remain +unavailable through raw `/ws`, and gateway regression tests must prove that a +forged payload cannot reach its handler. The public WebSocket catalog may list +these action names as internal transport shims, but must state that raw clients +cannot dispatch them. + +## Consequences + +Task-scoped authorization can use server-injected identity without accepting a +browser-forged task ID. Frontend code must continue using public non-MCP +WebSocket actions; it cannot reuse internal MCP action names. Agent-stream and +external MCP behavior stays unchanged because those adapters do not traverse +the raw gateway client dispatch path. + +This is a transport-origin boundary, not user authentication. Kandev's external +MCP endpoint retains its separately documented open-network policy and remains +limited by the tools registered for its mode. + +## Alternatives Considered + +An exported context provenance marker carrying session/execution identity was +rejected for this change because it would require threading a new security +principal through the agent stream, dispatcher, and every identity-sensitive +handler. A per-action gateway denylist was rejected because each new MCP +mutation could be accidentally omitted. Trusting `sender_task_id` after only +database relationship checks was rejected because the raw caller controls that +field. A separate dispatcher for every MCP mode was rejected as a larger +routing refactor than the required prefix boundary. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index c5c7412ee8..f37f6f0ef6 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -55,3 +55,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 0045 | [Install-wide storage maintenance uses typed ownership providers and quarantine](0045-install-wide-storage-maintenance.md) | accepted | backend, frontend, infra | 2026-07-14 | | 0046 | [Settings route save coordinator](0046-settings-route-save-coordinator.md) | accepted | frontend | 2026-07-14 | | 2026-07-18-turn-configuration-snapshots | [Attribute runtime configuration to turns](2026-07-18-turn-configuration-snapshots.md) | accepted | backend, frontend | 2026-07-18 | +| 2026-07-19-reject-mcp-actions-on-raw-websocket | [Reject MCP Actions on the Raw WebSocket](2026-07-19-reject-mcp-actions-on-raw-websocket.md) | accepted | backend, protocol | 2026-07-19 | diff --git a/docs/plans/parent-child-task-stop/plan.md b/docs/plans/parent-child-task-stop/plan.md new file mode 100644 index 0000000000..4ecc41b7e7 --- /dev/null +++ b/docs/plans/parent-child-task-stop/plan.md @@ -0,0 +1,433 @@ +--- +spec: docs/specs/tasks/parent-child-task-stop.md +created: 2026-07-19 +status: complete +--- + +# Implementation Plan: Parent-Child Task Control + +## Overview + +Keep both coordinator controls and make their intent obvious: + +- prefer `message_task_kandev(delivery_mode="interrupt")` when a busy child + should stop its current approach and immediately process a replacement + instruction; +- use new `stop_task_kandev` when the child should halt with no replacement + prompt or turn. + +Improve interrupt discoverability in the tool description and injected MCP +context, then expose Kandev's existing task-level stop through a direct-parent- +only task-mode MCP tool. Give the coordinator stop path a structured, truthful +result, add the trusted backend action and server adapter, and finish with an +integrated long-running child regression and public contract docs. No frontend +UI changes are required. + +The user confirmed both controls should exist, with interrupt preferred for the +usual stop-and-steer case. Planning defaults retained from the timed-out +clarification prompt: stop the whole child task, authorize only its direct +parent, and treat an already-idle child as idempotent `not_running` success. + +Transport decision: +[ADR-2026-07-19-reject-mcp-actions-on-raw-websocket](../../decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md). + +--- + +## Backend + +### Structured coordinator-stop semantics + +Files: + +- `apps/backend/internal/orchestrator/executor/executor_interaction.go` +- `apps/backend/internal/orchestrator/executor/executor_interaction_test.go` +- `apps/backend/internal/orchestrator/executor/executor_mocks_test.go` +- `apps/backend/internal/orchestrator/task_operations.go` +- `apps/backend/internal/orchestrator/task_operations_test.go` +- `apps/backend/internal/orchestrator/service.go` +- `apps/backend/internal/orchestrator/event_handlers_streaming.go` +- `apps/backend/internal/orchestrator/event_handlers_streaming_test.go` + +Changes: + +- Add an orchestrator-level coordinator stop operation that accepts only + `taskID` and returns a structured `stopped` / `not_running` result. Keep + executor sentinels and force/reason parameters behind that boundary. +- Define one exact semantic reason constant: + `coordinatorMCPStopReason = "stopped by parent task via MCP"`, with graceful + `force=false`. Do not pass caller-controlled text into lifecycle cleanup + because some executor providers interpret semantic reason strings. +- List active candidate sessions, then inspect them sequentially. For each + candidate, acquire the existing per-session `cancelInFlightGuard`, re-read + session/lifecycle state, and issue the logical stop before releasing the + guard. Process candidates in stable ID order and never hold more than one + guard or hold it across asynchronous runtime teardown. +- Treat only `errors.Is(err, lifecycle.ErrNoExecutionForSession)` as normal + absence. Any repository error, any other lifecycle error, or + `executionID == ""` with no absence sentinel is an internal failure. +- Refactor the stop-specific session transition to return an outcome containing + whether `CANCELLED` was written and the observed final state, plus error. + A same-state/terminal race must be re-read: already-`CANCELLED` is + idempotent, natural `COMPLETED`/other terminal state is no longer live, and + only an actual `CANCELLED` acceptance counts as stopped. +- Persist `CANCELLED` and publish its existing state event before scheduling + asynchronous teardown. If persistence fails, return the error and do not + teardown that candidate; earlier successful candidates remain cancelled and + retry targets the rest. +- Aggregate coordinator outcomes explicitly: zero accepted stops with every + candidate naturally absent/terminal gives `not_running`; one or more + accepted stops with the remainder naturally absent gives `stopped`; any + genuine lookup/persistence failure gives an internal error, even after partial + success. +- Reuse the detailed per-session stop primitive, but preserve legacy adapter + contracts. UI `Service.StopTask`, `CompleteTask`, Office/tree/workspace + `CancelTaskExecution`, and handoff cleanup keep their caller-supplied + reason/force values and current error-only policy + (`not_running -> executor.ErrExecutionNotFound`; task-wide legacy stop may + still return nil when any session stopped). Add regression coverage rather + than silently making those callers strict. +- On a structured `stopped` result, attempt a guarded `REVIEW` transition + only for non-Office, unarchived tasks currently in `IN_PROGRESS` or + `SCHEDULING`, using `UpdateTaskStateIfCurrentIn`. Rely on the production + task-service adapter to publish the state event; do not publish a second event + manually. Log a reconciliation write failure without changing the accepted + stop result. On `not_running`, leave task state untouched. +- Preserve worktrees, task environments, executor resume records, and queued + messages. Do not expose or use force-stop behavior. +- Scope the first version to executions registered live when each candidate is + inspected. Do not add a second lock beside `cancelInFlightGuard`, a durable + pause flag, or a task-wide launch fence; a launch registering after its + candidate lookup can escape even if it began before the stop call. + +Reason: + +The handler needs an idempotent product result, not a guess based on +`executor.ErrExecutionNotFound`. The existing lifecycle seam already provides +immediate logical cancellation plus asynchronous teardown, but it currently +collapses repository errors, swallows session-state persistence errors, tolerates +partial success, and writes `REVIEW` unconditionally. The structured boundary +makes the new agent-facing contract truthful without claiming process-exit or +concurrent-launch guarantees. + +### MCP backend action and authorization + +Files: + +- `apps/backend/pkg/websocket/actions.go` +- `apps/backend/internal/mcp/handlers/handlers.go` +- `apps/backend/internal/mcp/handlers/stop_task.go` +- `apps/backend/internal/mcp/handlers/stop_task_test.go` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/backend/internal/backendapp/mcp_stop_composition_test.go` +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/gateway/websocket/client_mcp_actions_test.go` + +Changes: + +- Add `ActionMCPStopTask = "mcp.stop_task"`. +- Add a narrow `TaskStopper` dependency and wire the production orchestrator + service through `registerMCPAndDebugRoutes`; keep stop semantics out of the + MCP handler itself. +- Parse `task_id` and trusted injected `sender_task_id`. Reject malformed JSON + as `BAD_REQUEST`; require both IDs with `VALIDATION_ERROR`. +- Load sender and target tasks and require both same-workspace membership and + `target.ParentID == sender.ID` before invoking the stopper. Do not reuse the + broader handoff/document relationship policy. +- Invoke the structured coordinator stop operation and return its + `status="stopped"` or `status="not_running"` result directly. The handler + must not map raw executor sentinels or accept reason/force fields. +- Return typed `VALIDATION_ERROR`, `NOT_FOUND`, `FORBIDDEN`, and + `INTERNAL_ERROR` responses without leaking a caller-controlled sender ID. +- Register the new backend action and update the always-registered MCP handler + count from 24 to 25. +- Add a production-composition test that dispatches `mcp.stop_task` through the + backendapp wiring and proves a non-nil orchestrator stopper is installed; unit + tests with an injected fake alone are insufficient. +- Enforce the ADR transport boundary in the raw gateway client: reject every + `mcp.*` action before shared-dispatcher invocation. Agent-stream MCP and the + external MCP server keep their existing direct adapter paths; the latter does + not register `stop_task_kandev`. +- Add a bypass regression proving a raw gateway request with forged + `sender_task_id` receives `FORBIDDEN` and never reaches the stop handler, + while the agent-stream/direct MCP dispatch path remains functional. + +Reason: + +Authorization must be enforced before runtime mutation, and injected identity is +trustworthy only when raw WebSocket clients cannot reach internal MCP actions. A +dedicated action also keeps user-originated `orchestrator.stop` separate from +agent-originated, direct-parent-authorized requests. + +### Task-mode MCP server adapter + +Files: + +- `apps/backend/internal/mcp/server/server.go` +- `apps/backend/internal/mcp/server/handlers.go` +- `apps/backend/internal/mcp/server/server_test.go` +- `apps/backend/internal/mcp/server/handlers_test.go` +- `apps/backend/internal/mcp/server/sysprompt_sync_test.go` +- `apps/backend/config/prompts/kandev-context.md` + +Changes: + +- Register `stop_task_kandev` only in `ModeTask`, with required `task_id` + only. Describe all-session stop, direct-child authority, + eligible active-Kanban `REVIEW`, idempotency, async teardown, and the + difference from interrupting `message_task_kandev`. +- Forward to `mcp.stop_task` while injecting only trusted `s.taskID` as + `sender_task_id`; expose neither identity, reason, nor `force` as tool + arguments. +- Rewrite the opening guidance for `message_task_kandev` as the reciprocal + three-way choice: ordinary information stays queued; urgent direct-parent + replacement work uses `delivery_mode="interrupt"`; halt-only intent uses + `stop_task_kandev`. Remove the ambiguous current phrase “steering/stop + messages,” because interrupt always supplies a replacement message. +- Expand `apps/backend/config/prompts/kandev-context.md` to list the optional + `delivery_mode` values and the three-way choice: queue, interrupt-and-steer, + or halt-only stop. This fixes the shortened runtime context that currently + hides the interrupt parameter from agents. +- Update `registerKanbanTools` accounting from 14 to 15 and total task-mode + registration from 26 to 27. Assert the stop tool remains absent from Config, + Office, and External modes. +- Extend schema-to-system-prompt sync coverage so the shortened injected context + cannot again omit the message tool's `delivery_mode` enum/default, + direct-parent restriction, or companion stop tool. + +--- + +## Frontend + +No frontend code changes. This feature adds an agent-only task MCP operation and +reuses existing session/task state events already rendered by the web client. + +--- + +## Tests + +- **What:** the per-session primitive distinguishes accepted cancellation, + exact absence sentinel, terminal race, lookup failure, persistence failure, + and empty-ID invariant. + **File:** `apps/backend/internal/orchestrator/executor/executor_interaction_test.go` + **How:** table-driven unit tests with recording lifecycle/repository fakes. + +- **What:** coordinator traversal attempts candidates in stable order and applies + the accepted/vanished/failure aggregation table, including idempotent repeat. + **File:** `apps/backend/internal/orchestrator/task_operations_test.go` + **How:** table-driven orchestrator tests with detailed stop outcomes. + +- **What:** repository/lifecycle lookup errors do not masquerade as + `not_running`; terminal races do not get overwritten; session-state + persistence failure prevents teardown for that candidate; the fixed stop + reason and `force=false` reach the lifecycle manager. + **Files:** `apps/backend/internal/orchestrator/executor/executor_interaction_test.go`, + `apps/backend/internal/orchestrator/executor/executor_mocks_test.go`, + `apps/backend/internal/orchestrator/event_handlers_streaming_test.go` + **How:** injected errors and recording callbacks; no sleeps. + +- **What:** stop serializes with ready-event/pending-move/queue-drain decisions + through the existing `cancelInFlightGuard`; no queued message is popped or + replacement turn dispatched after cancellation acceptance. + **File:** `apps/backend/internal/orchestrator/task_operations_test.go` + **How:** channel-synchronized race tests with no sleeps. + +- **What:** a launch that registers after its candidate lookup is explicitly not + fenced in v1 and can escape; the test documents this boundary instead of + accidentally claiming stronger semantics. + **File:** `apps/backend/internal/orchestrator/task_operations_test.go` + **How:** channel-synchronized pre-registration launch test. + +- **What:** runtime teardown is truly asynchronous and detached from caller + cancellation. + **File:** `apps/backend/internal/orchestrator/executor/executor_interaction_test.go` + **How:** block `StopAgentWithReason`, assert structured stop returns after the + state event, cancel caller context, then unblock and assert teardown completes. + +- **What:** successful task stop writes session `CANCELLED`, uses archive-safe + guarded `REVIEW` through the existing event-producing adapter for active + Kanban state only, and preserves workspace/queue data; Office, archived, + terminal, and `not_running` cases change no task state. + **File:** `apps/backend/internal/orchestrator/task_operations_test.go` + **How:** SQLite-backed orchestrator test with fake lifecycle manager and event + subscriber. + +- **What:** direct parent succeeds; self, sibling, child, grandparent, unrelated, + and cross-workspace callers are forbidden before stopper invocation. + **File:** `apps/backend/internal/mcp/handlers/stop_task_test.go` + **How:** table-driven handler test with seeded tasks and recording stopper. + +- **What:** malformed JSON, missing/unknown identities, `not_running`, and + stopper errors map to the specified response codes/statuses; unknown payload + fields cannot influence reason or force. + **File:** `apps/backend/internal/mcp/handlers/stop_task_test.go` + **How:** focused unit tests. + +- **What:** task-not-found sentinels map to `NOT_FOUND`, while arbitrary sender + or target repository failures map to `INTERNAL_ERROR`; neither invokes the + stopper. + **File:** `apps/backend/internal/mcp/handlers/stop_task_test.go` + **How:** injected task-service failures for both lookups. + +- **What:** task MCP schema registers and forwards trusted sender identity while + other server modes omit the tool; the message schema description and injected + context prominently explain `delivery_mode="interrupt"`. + **Files:** `apps/backend/internal/mcp/server/server_test.go`, + `apps/backend/internal/mcp/server/handlers_test.go`, + `apps/backend/internal/mcp/server/sysprompt_sync_test.go` + **How:** tool-list/schema/description tests plus exact fake-backend payload + assertions. + +- **What:** production backendapp composition installs the orchestrator-backed + stopper before handler registration. + **File:** `apps/backend/internal/backendapp/mcp_stop_composition_test.go` + **How:** construct the production handler graph and dispatch the new action + through its dispatcher. + +- **What:** raw `/ws` dispatch rejects all `mcp.*` actions, including a forged + direct-parent stop payload, before the shared dispatcher; ordinary public + actions still dispatch. + **File:** `apps/backend/internal/gateway/websocket/client_mcp_actions_test.go` + **How:** gateway client test with a recording dispatcher handler and captured + error response. + +- **What:** a parent stops a long-running child through the real orchestrator and + MCP backend action without + queuing a replacement prompt; the call returns promptly, session becomes + `CANCELLED`, task becomes `REVIEW`, and a repeat returns `not_running`. + **File:** `apps/backend/internal/integration/mcp_stop_task_test.go` + **How:** integration test with the mock agent and real + orchestrator/dispatcher/MCP-handler wiring. The local task-mode MCP + tool-to-action forwarding remains covered by server unit tests. + +--- + +## E2E Tests + +Skipped. No browser-visible control or new frontend behavior is introduced. The +backend integration test covers authorization through lifecycle state changes; +task-mode MCP registration/forwarding is covered by server tests. + +--- + +## Public documentation + +Files: + +- `docs/public/automation-and-mcp.md` +- `docs/public/coordination.md` +- `docs/public/sessions-and-review.md` +- `docs/public/feature-status.md` +- `docs/public/websocket-api.md` +- `docs/public/coverage.json` + +Changes: + +- Add `stop_task_kandev` to task MCP lifecycle/coordination references. +- Make `delivery_mode="interrupt"` the documented recommendation when the + coordinator has replacement instructions for a busy direct child. +- Explain direct-child authorization, all-session scope, task `REVIEW`, preserved + workspace, idempotent response, and asynchronous teardown; qualify `REVIEW` + as the normal active-Kanban transition. +- Contrast halt-only stop with `message_task_kandev(delivery_mode="interrupt")`. +- Add internal `mcp.stop_task` to the WebSocket action catalog and state that + raw `/ws` clients are rejected for every `mcp.*` action; supported access is + through the agent-stream or mode-scoped MCP adapters. +- Assign `stop_task_kandev` exactly once to the `task-lifecycle` coverage area + and add its backend handler plus raw-gateway guard source/test files to that + area's coverage mapping. +- Update External MCP feature-status wording to name stop among the omitted + live-session tools. + +--- + +## Implementation Waves + +Wave 1: + +- [x] [task-01-execution-stop-semantics](task-01-execution-stop-semantics.md) + +Wave 2: + +- [x] [task-02-coordinator-stop-operation](task-02-coordinator-stop-operation.md) + +Wave 3: + +- [x] [task-03-mcp-stop-handler](task-03-mcp-stop-handler.md) + +Wave 4: + +- [x] [task-04-task-mcp-tool](task-04-task-mcp-tool.md) + +Wave 5 (parallel): + +- [x] [task-05-integration-regression](task-05-integration-regression.md) +- [x] [task-06-public-docs](task-06-public-docs.md) + +--- + +## Verification Commands + +Format first: + +```bash +make fmt +``` + +Targeted backend tests: + +```bash +cd apps/backend && go test ./internal/orchestrator/executor ./internal/orchestrator ./internal/mcp/handlers ./internal/mcp/server ./internal/backendapp ./internal/gateway/websocket +``` + +Focused integration test: + +```bash +cd apps/backend && go test ./internal/integration -run TestMCPStopTask +``` + +Public docs validation: + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` + +Full verification: + +```bash +make fmt +make typecheck test lint +``` + +--- + +## Risks + +- Runtime teardown remains asynchronous by design; tests must distinguish + logical stop acceptance from confirmed process exit. +- Stop inspects candidate sessions sequentially, not behind a task-wide launch + fence. An execution registering after its candidate lookup can run; closing + that race requires a separately designed task-level stop fence. +- Async teardown failure is logged after the response; the first version adds no + retry/reconciliation guarantee for a lingering external process. +- Existing queued messages intentionally survive stop and may be delivered if + the child is later restarted. +- Multi-session partial failure can leave some sessions already cancelled; + returning an error and making retries idempotent is required. + +## Approved implementation assumptions + +On 2026-07-19, the user approved implementation after confirming both controls +should remain, with interrupt preferred for stop-and-steer. The approved +defaults are: + +- stop every current session on the direct child, not only its primary session; +- authorize only the direct parent; +- return idempotent `not_running` for an already-idle child; +- attempt `REVIEW` only for active Kanban state; preserve Office, archived, and + non-active task state; +- preserve existing queued messages and descendants; +- accept the documented pre-registration launch race in v1 instead of adding a + durable task-level stop fence; +- enforce the accepted ADR by rejecting the entire internal `mcp.*` namespace + on raw `/ws` rather than adding per-action caller provenance. diff --git a/docs/plans/parent-child-task-stop/task-01-execution-stop-semantics.md b/docs/plans/parent-child-task-stop/task-01-execution-stop-semantics.md new file mode 100644 index 0000000000..41ba1a1a37 --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-01-execution-stop-semantics.md @@ -0,0 +1,76 @@ +--- +id: "01-execution-stop-semantics" +title: "Execution stop outcomes" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 01: Execution stop outcomes + +## Acceptance + +- A detailed per-session stop primitive distinguishes accepted cancellation, + natural terminal/absent execution, and genuine lookup/persistence failure. +- Only `lifecycle.ErrNoExecutionForSession` counts as natural absence. Other + lookup errors and empty-ID/nil are failures. +- The stop-specific state transition reports changed/final state. Natural + terminal races do not get overwritten or count as stopped; successful + acceptance persists `CANCELLED` before async teardown. +- State-write failure prevents teardown for that candidate. Runtime teardown is + asynchronous and detached from caller cancellation. +- Existing UI, completion, Office/tree/workspace, and handoff stop callers retain + supplied reason/force and legacy `ErrExecutionNotFound` / partial-success + behavior. + +## Verification + +```bash +cd apps/backend && go test ./internal/orchestrator/executor ./internal/orchestrator +``` + +## Files Likely Touched + +- `apps/backend/internal/orchestrator/executor/executor_interaction.go` +- `apps/backend/internal/orchestrator/executor/executor_interaction_test.go` +- `apps/backend/internal/orchestrator/executor/executor_mocks_test.go` +- `apps/backend/internal/orchestrator/service.go` +- `apps/backend/internal/orchestrator/event_handlers_streaming.go` +- `apps/backend/internal/orchestrator/event_handlers_streaming_test.go` + +## Dependencies + +None. + +## Inputs + +- Spec: `What`, `State and persistence`, `Failure modes`, and `Scenarios`. +- Existing task-stop seam: `Service.StopTask` and `Executor.StopByTaskID`. +- Normal lifecycle absence sentinel: `lifecycle.ErrNoExecutionForSession`. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report per-session outcomes, +legacy compatibility, files changed, tests run, blockers, and remaining +lifecycle risks. + +## Completion + +- Added a strict per-session outcome with accepted/final-state reporting. +- Added exact execution teardown ownership across coordinator stops, terminal + cleanup, launch/recovery races, and durable task-resource cleanup. +- Preserved legacy stop sentinels while exposing exact runtime absence as + idempotent; real lookup and persistence failures remain retryable. +- Added an atomic session-owned task-state CAS so clarification/terminal state + cannot be overwritten by delayed runtime reconciliation. +- Changed areas: runtime/lifecycle stop contracts, orchestrator executor and + recovery paths, task cleanup jobs, MCP clarification state, and focused E2E + coverage. +- Tests run: `make fmt`, `make typecheck`, `make test`, `make lint`, scoped Go + race suites, and the sidebar-clarification and Office-onboarding Playwright + specs against production builds. +- Blockers: none. +- Remaining lifecycle risks: none known; session-scoped lookup absence remains + retryable until an exact execution identity is available. diff --git a/docs/plans/parent-child-task-stop/task-02-coordinator-stop-operation.md b/docs/plans/parent-child-task-stop/task-02-coordinator-stop-operation.md new file mode 100644 index 0000000000..374a1add9b --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-02-coordinator-stop-operation.md @@ -0,0 +1,73 @@ +--- +id: "02-coordinator-stop-operation" +title: "Coordinator stop operation" +status: done +wave: 2 +depends_on: ["01-execution-stop-semantics"] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 02: Coordinator stop operation + +## Acceptance + +- A coordinator-specific orchestrator operation accepts only a task ID, uses the + exact fixed reason `"stopped by parent task via MCP"` with + `force=false`, and returns structured `stopped` / `not_running` status. +- Candidate sessions are processed in stable order under the existing + per-session `cancelInFlightGuard`; ready, pending-move, and queue-drain races + cannot dispatch replacement work after cancellation acceptance. +- Aggregation follows the spec: all absent/terminal is `not_running`; at least + one accepted with the rest absent is `stopped`; any genuine synchronous + failure is an error even after partial success. +- An accepted stop attempts guarded `REVIEW` only for active Kanban state. + Office, archived, non-active, and `not_running` targets retain task state. +- Queues, workspaces, task environments, descendants, and existing legacy stop + entrypoints remain intact. +- A deterministic test documents that an execution registering after its + candidate lookup can escape because v1 has no task-wide launch fence. + +## Verification + +```bash +cd apps/backend && go test ./internal/orchestrator +``` + +## Files Likely Touched + +- `apps/backend/internal/orchestrator/task_operations.go` +- `apps/backend/internal/orchestrator/task_operations_test.go` +- `apps/backend/internal/orchestrator/service.go` + +## Dependencies + +- Task 01 supplies the detailed per-session stop outcome. + +## Inputs + +- Spec: `Choosing the control`, `State and persistence`, `Failure modes`, + and `Scenarios`. +- Existing ready/interrupt/drain serialization: `cancelInFlightGuard`. +- Guarded task-state path: `UpdateTaskStateIfCurrentIn` for + `IN_PROGRESS` / `SCHEDULING` through the production task-service adapter; + do not publish a duplicate event. +- Scope boundary: lifecycle registration after a candidate lookup is not fenced. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report aggregation rules, +race coverage, task-state behavior, files changed, tests run, blockers, and +remaining launch-fence risk. + +## Completion + +- Added deterministic stable-ID traversal and explicit `stopped` / + `not_running` aggregation with joined synchronous failures. +- Reused `cancelInFlightGuard` through final re-read and cancellation + acceptance; channel tests cover the ready/queue race under `-race`. +- Added atomic active-session cancellation in SQLite so a concurrent terminal + transition cannot be overwritten. +- Guarded REVIEW reconciliation preserves Office, archived, non-active, and + idle targets; queues and task resources are untouched. +- The approved late-registration escape remains documented and tested. diff --git a/docs/plans/parent-child-task-stop/task-03-mcp-stop-handler.md b/docs/plans/parent-child-task-stop/task-03-mcp-stop-handler.md new file mode 100644 index 0000000000..00aa3a8157 --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-03-mcp-stop-handler.md @@ -0,0 +1,80 @@ +--- +id: "03-mcp-stop-handler" +title: "MCP stop handler" +status: done +wave: 3 +depends_on: ["02-coordinator-stop-operation"] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 03: MCP stop handler + +## Acceptance + +- `mcp.stop_task` accepts only trusted direct-parent requests and rejects every + broader relationship before stop side effects. +- Handler returns `stopped` or idempotent `not_running` with the target ID and + maps validation, existence, authorization, and stop failures to typed errors. +- Production wiring supplies the structured orchestrator operation through a + narrow `TaskStopper` dependency. Handler accepts neither reason nor force and + never maps raw executor sentinels. +- Malformed JSON is `BAD_REQUEST`; required-ID, not-found, and forbidden cases + produce their specified typed errors. Backend handler count becomes 25. +- Task-not-found sentinels and arbitrary task repository failures remain + distinct. Production composition proves the orchestrator stopper is wired. +- Raw gateway clients cannot invoke any `mcp.*` action. A forged + `sender_task_id` is rejected before dispatcher/handler side effects, while + agent-stream MCP dispatch remains supported. + +## Verification + +```bash +cd apps/backend && go test ./internal/mcp/handlers ./internal/backendapp ./internal/gateway/websocket +``` + +## Files Likely Touched + +- `apps/backend/pkg/websocket/actions.go` +- `apps/backend/internal/mcp/handlers/handlers.go` +- `apps/backend/internal/mcp/handlers/stop_task.go` +- `apps/backend/internal/mcp/handlers/stop_task_test.go` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/backend/internal/backendapp/mcp_stop_composition_test.go` +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/gateway/websocket/client_mcp_actions_test.go` + +## Dependencies + +- Task 02 defines the structured stop result consumed by this handler. + +## Inputs + +- Spec: `API surface`, `Permissions`, and `Failure modes`. +- Direct-parent precedent: `handleMessageTask` interrupt authorization. +- Production dependency wiring: `registerMCPAndDebugRoutes`. +- Authorization uses the same direct-parent predicate plus a same-workspace + defense-in-depth check. +- Transport boundary: + `ADR-2026-07-19-reject-mcp-actions-on-raw-websocket`. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report action/response shapes, +authorization cases, files changed, tests run, blockers, and risks. + +## Completion + +- Added `mcp.stop_task`, a narrow orchestrator stopper dependency, explicit + production wiring, and typed payload/lookup/authorization/error handling. +- Enforced direct-parent plus same-workspace authorization before mutation; + reason, force, session, and sender spoof fields cannot reach lifecycle code. +- Raw `/ws` now rejects every `mcp.*` action with `FORBIDDEN` before shared + dispatch, while direct trusted dispatcher tests preserve agent-stream use. +- Handler registration, relationship matrix, lookup failures, idempotent + statuses, invalid dependency/status behavior, and forged raw transport are + covered; gateway tests pass under `-race`. +- Production composition dispatches the action successfully through the real + backend route graph and orchestrator stopper. +- Remaining trust assumption: mode-scoped MCP adapters and trusted in-process + callers are the identity boundary described by the accepted ADR. diff --git a/docs/plans/parent-child-task-stop/task-04-task-mcp-tool.md b/docs/plans/parent-child-task-stop/task-04-task-mcp-tool.md new file mode 100644 index 0000000000..45ef413883 --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-04-task-mcp-tool.md @@ -0,0 +1,68 @@ +--- +id: "04-task-mcp-tool" +title: "Task MCP stop tool" +status: done +wave: 4 +depends_on: ["03-mcp-stop-handler"] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 04: Task MCP stop tool + +## Acceptance + +- `stop_task_kandev` is registered only in task mode with required `task_id` + and accurate halt-only versus stop-and-steer guidance. +- Server forwarding injects immutable current task identity and exposes no + sender, reason, or force-stop field to the agent. +- The `message_task_kandev` description presents the reciprocal three-way + choice: queue ordinary information, interrupt urgent replacement work, stop + halt-only work. +- Injected Kandev context advertises optional `delivery_mode` and the + queue/interrupt/stop choice. `registerKanbanTools` count becomes 15 and total + task-mode count becomes 27; Config, Office, and External still omit stop. + +## Verification + +```bash +cd apps/backend && go test ./internal/mcp/server +``` + +## Files Likely Touched + +- `apps/backend/internal/mcp/server/server.go` +- `apps/backend/internal/mcp/server/handlers.go` +- `apps/backend/internal/mcp/server/server_test.go` +- `apps/backend/internal/mcp/server/handlers_test.go` +- `apps/backend/internal/mcp/server/sysprompt_sync_test.go` +- `apps/backend/config/prompts/kandev-context.md` + +## Dependencies + +- Task 03 provides `mcp.stop_task` and its backend response contract. + +## Inputs + +- Spec: `What`, `Choosing the control`, `API surface`, and `Permissions`. +- Tool/forwarder precedent: `message_task_kandev`. +- Mode registration/count tests in `internal/mcp/server/server_test.go`. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report tool schema, modes, +identity injection, files changed, tests run, blockers, and risks. + +## Completion + +- Added task-only `stop_task_kandev` with a single required `task_id` and a + fresh trusted `{task_id, sender_task_id}` backend payload. +- Updated task counts to 15 Kanban / 27 total and proved Config, Office, and + External omission. +- Rewrote message and stop descriptions plus injected context around the + queued / interrupt-and-steer / halt-only decision; prompt-schema sync now + protects `delivery_mode` discoverability. +- Server and WebSocket packages pass focused tests. +- Existing agentctl instances retain their old per-instance tool registry and + need restart/recreation to see the new tool; old first-turn prompt text is + likewise not retroactive. diff --git a/docs/plans/parent-child-task-stop/task-05-integration-regression.md b/docs/plans/parent-child-task-stop/task-05-integration-regression.md new file mode 100644 index 0000000000..e77622d3d7 --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-05-integration-regression.md @@ -0,0 +1,62 @@ +--- +id: "05-integration-regression" +title: "Integrated stop regression" +status: done +wave: 5 +depends_on: ["01-execution-stop-semantics", "02-coordinator-stop-operation", "03-mcp-stop-handler", "04-task-mcp-tool"] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 05: Integrated stop regression + +## Acceptance + +- Real orchestrator, dispatcher, and MCP backend-handler wiring lets a direct + parent stop a long-running child without waiting for turn completion or + dispatching a replacement prompt. +- Action response returns promptly; child session becomes `CANCELLED`, eligible + child task becomes `REVIEW`, and the simulated lifecycle reports the + execution is no longer running. The test does not require simulated-manager + map removal. +- A repeat call returns `not_running` and changes no additional state. +- The test does not claim to exercise the local task-mode MCP transport adapter; + its exact action/payload forwarding is covered in Task 04. + +## Verification + +```bash +cd apps/backend && go test ./internal/integration -run TestMCPStopTask +``` + +## Files Likely Touched + +- `apps/backend/internal/integration/mcp_stop_task_test.go` + +## Dependencies + +- Tasks 01-04 complete the lifecycle, coordinator operation, backend action, and + tool adapter. + +## Inputs + +- Spec scenarios for direct-parent success, no replacement prompt, and + idempotent retry. +- Existing orchestrator integration harness and mock-agent long-turn patterns. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report observed timing/state, +files changed, tests run, blockers, and any unverified process-cleanup risks. + +## Completion + +- Added a real orchestrator, MCP-handler, and shared-dispatcher regression for a + long-running direct child. +- Deliberately blocked simulated runtime teardown and proved the response lands + after session `CANCELLED` and task `REVIEW`, but before process stop finishes. +- Verified no replacement prompt or queued message is created, runtime stop + completes after release, and a repeat returns `not_running` without mutation. +- Passed the focused test, 20 repeated runs, and the focused race-detector run. +- Remaining lifecycle boundary: the test proves the simulated runtime reaches + stopped status, not operating-system cleanup for every production provider. diff --git a/docs/plans/parent-child-task-stop/task-06-public-docs.md b/docs/plans/parent-child-task-stop/task-06-public-docs.md new file mode 100644 index 0000000000..518863a32f --- /dev/null +++ b/docs/plans/parent-child-task-stop/task-06-public-docs.md @@ -0,0 +1,67 @@ +--- +id: "06-public-docs" +title: "Public stop documentation" +status: done +wave: 5 +depends_on: ["03-mcp-stop-handler", "04-task-mcp-tool"] +plan: "plan.md" +spec: "../../specs/tasks/parent-child-task-stop.md" +--- + +# Task 06: Public stop documentation + +## Acceptance + +- Public task MCP and coordination docs recommend interrupt for stop-and-steer, + reserve stop for halt-only intent, and explain direct-child authority, + all-session scope, eligible active-Kanban `REVIEW`, and async teardown. +- Internal `mcp.stop_task` appears in the WebSocket transport catalog, which + clearly says raw `/ws` rejects every `mcp.*` action and supported access is + through MCP adapters. `stop_task_kandev` is assigned exactly once in public + coverage metadata, with its backend handler and gateway guard source/tests + mapped to the same area. +- Public docs validators pass; External MCP feature-status text explicitly says + stop and other live-session tools are omitted. + +## Verification + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` + +## Files Likely Touched + +- `docs/public/automation-and-mcp.md` +- `docs/public/coordination.md` +- `docs/public/sessions-and-review.md` +- `docs/public/feature-status.md` +- `docs/public/websocket-api.md` +- `docs/public/coverage.json` + +## Dependencies + +- Tasks 03-04 fix the final action, tool, permission, and response contracts. + +## Inputs + +- Spec: complete public behavior contract. +- Existing task MCP sections and `task-lifecycle` coverage area. +- Docs-maintainer validation workflow. + +## Output Contract + +Update this task to `done`, update `plan.md`, and report docs changed, +validation commands, blockers, and any intentionally undocumented internals. + +## Completion + +- Documented the queued / interrupt-and-steer / halt-only choice, direct-child + authority, all-session scope, guarded `REVIEW`, idempotency, async teardown, + and preserved resources across the public MCP, coordination, and session docs. +- Documented raw `/ws` rejection for every `mcp.*` action and External MCP's + omission of `stop_task_kandev`. +- Assigned `stop_task_kandev` exactly once to `task-lifecycle` coverage with the + handler and gateway sources/tests. +- Both public-doc validators pass; no internal implementation detail beyond the + durable behavior and transport boundary was added to public guidance. diff --git a/docs/public/automation-and-mcp.md b/docs/public/automation-and-mcp.md index c275f77216..867e6a6f91 100644 --- a/docs/public/automation-and-mcp.md +++ b/docs/public/automation-and-mcp.md @@ -135,7 +135,7 @@ Task mode currently registers these tool groups: | Group | Available operations | |---|---| -| Board lookups and task lifecycle | List workspaces, workflows, workflow steps, tasks, agents, and executor profiles; create, update, move, archive, or delete tasks. This mode does not mutate workflows, profiles, or executors. | +| Board lookups and task lifecycle | List workspaces, workflows, workflow steps, tasks, agents, and executor profiles; create, update, move, archive, or delete tasks; halt all live work on a direct child. This mode does not mutate workflows, profiles, or executors. | | Coordination | Message a task or targeted session, spawn a named session on the current or another same-workspace task, and read task conversation. | | User interaction | Ask a structured question when the current agent/session supports it. | | Plans | Create, get, update, and delete the current task plan. | @@ -147,7 +147,11 @@ Task identity is injected for operations that require it. Workspace, parent/subt `spawn_session_kandev` creates a named sibling session on the current task by default and can target another task in the same workspace. `message_task_kandev` can address a task's primary session or an explicit session ID: a running agent receives queued input, an idle/created session can be started, and a failed or cancelled session rejects the message. -A same-task message requires the sibling's session ID. Normal messages can cross workspaces when the sender knows the full task ID. Delivery to a running session is queued by default; `delivery_mode: "interrupt"` is allowed only from a direct parent task to its child, and another sender receives a hard error rather than a silent downgrade. +A same-task message requires the sibling's session ID. Normal messages can cross workspaces when the sender knows the full task ID. Delivery to a running session is queued by default. When a direct child must abandon its current approach and receive replacement work now, its parent should use `message_task_kandev` with `delivery_mode: "interrupt"`; another sender receives a hard error rather than a silent downgrade, and a request that cannot dispatch safely remains queued. + +Use `stop_task_kandev` only when the direct child should halt without a replacement prompt. It has no session selector: one call gracefully stops every execution Kandev still observes as live across the child's active sessions, including non-primary sessions. Accepted sessions become `CANCELLED` before runtime teardown is scheduled. `status: "stopped"` confirms logical cancellation and asynchronous teardown, not operating-system process exit; a child with no live execution returns the idempotent `status: "not_running"` without changing task or session state. + +After an accepted stop, Kandev attempts to move an unarchived, non-Office task from `IN_PROGRESS` or `SCHEDULING` to `REVIEW`; other task states are preserved. Worktrees, task environments, commits, task records, descendants, and queued messages remain available, and the task can be started again later. `add_branch_to_task_kandev` works only with the Worktree executor. It can add another branch of the same repository or a second repository entirely. Select at most one of `repository_id`, `repository_url`, or `local_path`; `repository_url` accepts a GitHub repository URL, while the URL/path forms find or create that repository in the task's workspace. A locator is optional for a single-repository task and required to disambiguate a multi-repository task. The new repository/branch receives its own worktree. `update_repository_base_branch_kandev` changes the base used for Kandev's diff, not a pull request's target branch. @@ -184,7 +188,7 @@ The settings page's static **Available tools** preview currently counts 29 and o In external mode, `create_task_kandev` has no current task and does not accept the `parent_id: "self"` shorthand. Its registered top-level contract asks for a repository ID, GitHub URL, or local path; workspace and workflow resolve automatically only when unambiguous. The current handler can nevertheless accept an omitted repository and create repo-less work, which is a contract/implementation mismatch rather than a supported equivalent of the regular UI's **None** option. Supply an explicit repository locator for portable clients. A resolvable agent profile is required even with `start_agent: false`; otherwise `start_agent` defaults to true. To create a subtask, pass the full ID of an existing parent. -External mode has no live Kandev session, so it does not expose task-scoped questions, plans, walkthroughs, sibling-session spawning, targeted session messages, branch operations, or step-completion signals. Some external tools can delete or materially reconfigure data; review the client's tool approvals. +External mode has no live Kandev session, so it does not expose `stop_task_kandev` or other task-scoped questions, plans, walkthroughs, sibling-session spawning, targeted session messages, branch operations, or step-completion signals. Some external tools can delete or materially reconfigure data; review the client's tool approvals. ### External MCP security boundary diff --git a/docs/public/coordination.md b/docs/public/coordination.md index 8c9cc417c0..55264bc176 100644 --- a/docs/public/coordination.md +++ b/docs/public/coordination.md @@ -116,9 +116,27 @@ For predictable top-level creation, pass `repository_url`, `repository_id`, or ` The default delivery mode is queued. Each session accepts 10 queued messages by default; operators can change it with `KANDEV_QUEUE_MAX_PER_SESSION`, and a value of `0` or less disables the limit. Only one queued message drains per agent turn. When the cap is reached, the sender receives a structured `queue_full` error and should retry after a target turn completes. -`delivery_mode: "interrupt"` cancels current work and prioritizes the message, but only a direct parent task may interrupt its child. Other senders receive an error. If immediate dispatch cannot proceed safely, the response can still report the message as queued. +Choose the control by intent: -Additional boundaries: +| Intent | Operation | Result | +|---|---|---| +| Send information that can wait | `message_task_kandev` with queued delivery or no `delivery_mode` | The current turn continues and the message waits FIFO. | +| Stop the current approach and give replacement work now | `message_task_kandev` with `delivery_mode: "interrupt"` | The direct parent requests immediate cancellation and redispatch. If that cannot proceed safely, the response reports the message as queued. | +| Halt all current work with no replacement prompt | `stop_task_kandev` | The direct parent requests logical cancellation and graceful teardown without creating or dispatching a message. | + +Only a direct parent may interrupt its child. Halt-only stop is stricter: it accepts only a same-workspace direct child, while self, siblings, ancestors other than the parent, deeper descendants, unrelated tasks, and cross-workspace callers are rejected. Use interrupt for stop-and-steer work. Reserve stop for halt-only intent. + +### Stop a direct child's work + +`stop_task_kandev` accepts the full ID of one direct child and has no session-specific option. Kandev inspects that child's active-session candidates and requests a graceful stop for every execution still observed as live, including non-primary sibling sessions. It does not recurse into descendants. + +For each accepted execution, Kandev persists the session as `CANCELLED` before scheduling runtime teardown. A `status: "stopped"` response confirms that logical state and scheduled teardown; cleanup continues asynchronously and the process may not have exited yet. When no live execution is accepted, the call succeeds idempotently with `status: "not_running"` and changes no task or session state. + +After at least one accepted stop, Kandev attempts to move a regular, unarchived, non-Office task from `IN_PROGRESS` or `SCHEDULING` to `REVIEW`, provided no session remains working. Office, archived, and already non-active tasks keep their state, and a failed secondary `REVIEW` reconciliation does not undo accepted session stops. + +Stopping preserves the task record, worktrees, environments, commits, descendants, and existing queued messages. It sends no prompt, creates no replacement turn, and does not create a durable pause: a later user or workflow action can start the task again. + +Additional messaging boundaries: - A task cannot message its own primary session through the default route, and a session cannot message itself. - Normal targeted messages can cross workspaces when the sender has the exact task ID. Session spawning cannot. @@ -217,6 +235,8 @@ When Office is enabled, it prototypes **Blocked by** and **Blocking** properties - **Inherited subtask sees unexpected changes:** it intentionally shares the parent's materialized files and branch. - **Message remains queued:** the target is busy and only one queued message drains per turn. Check for `queue_full` before retrying. - **Interrupt is rejected:** only the target's direct parent may use interrupt delivery. +- **Stop is rejected:** `stop_task_kandev` is task-mode only and accepts only a same-workspace direct child of the caller. +- **Stop reports `stopped` but a process is still visible:** the response confirms logical cancellation and scheduled graceful teardown, not process exit. - **Agent cannot spawn on another task:** `spawn_session_kandev` is same-workspace only; create a task or use a normal targeted message instead. - **Parent does not advance after children finish:** the parent needs an active session in `CREATED`, `STARTING`, `RUNNING`, or `WAITING_FOR_INPUT` in addition to terminal direct children. - **A multi-repository executor is disabled:** multi-repository tasks require **git-worktree**. diff --git a/docs/public/coverage.json b/docs/public/coverage.json index 4958b5a279..1e483cb32d 100644 --- a/docs/public/coverage.json +++ b/docs/public/coverage.json @@ -52,8 +52,26 @@ "audiences": ["user"], "stability": "stable", "docs": ["tasks-and-workflows", "coordination", "git-operations"], - "sources": ["apps/backend/internal/orchestrator/task_operations.go", "apps/backend/internal/mcp/handlers/config_task_handlers.go"], - "tests": ["apps/backend/internal/orchestrator/task_operations_test.go", "apps/backend/internal/mcp/handlers/handlers_test.go"], + "sources": [ + "apps/backend/internal/orchestrator/task_operations.go", + "apps/backend/internal/mcp/handlers/config_task_handlers.go", + "apps/backend/internal/mcp/handlers/stop_task.go", + "apps/backend/internal/mcp/server/handlers.go", + "apps/backend/internal/mcp/server/server.go", + "apps/backend/internal/gateway/websocket/client.go" + ], + "tests": [ + "apps/backend/internal/orchestrator/task_operations_test.go", + "apps/backend/internal/orchestrator/coordinator_stop_test.go", + "apps/backend/internal/mcp/handlers/handlers_test.go", + "apps/backend/internal/mcp/handlers/stop_task_test.go", + "apps/backend/internal/mcp/server/handlers_test.go", + "apps/backend/internal/mcp/server/server_test.go", + "apps/backend/internal/mcp/server/sysprompt_sync_test.go", + "apps/backend/internal/backendapp/mcp_stop_composition_test.go", + "apps/backend/internal/integration/mcp_stop_task_test.go", + "apps/backend/internal/gateway/websocket/client_mcp_actions_test.go" + ], "mcpTools": [ "add_branch_to_task_kandev", "archive_task_kandev", @@ -62,6 +80,7 @@ "list_tasks_kandev", "move_task_kandev", "step_complete_kandev", + "stop_task_kandev", "update_repository_base_branch_kandev", "update_task_kandev", "update_task_state_kandev" diff --git a/docs/public/feature-status.md b/docs/public/feature-status.md index fff205a1a2..40560b68ea 100644 --- a/docs/public/feature-status.md +++ b/docs/public/feature-status.md @@ -32,7 +32,7 @@ Released builds can lag this page. Check **Settings > System > About** or `kande | Multiple named task documents | In progress | Storage, revisions, Office UI, and Office MCP tools exist. Regular Kanban has no document panel and its task MCP does not register the document tools. | | Task labels and label filters | In progress | The shared task row stores a JSON label field, but label catalogs, pickers, filters, and maintained E2E coverage are on the feature-flagged Office surface, not regular Kanban. | | Named parallel sessions | Supported | Users can add, name, make primary, stop, resume, delete, and hand off sessions. Blank, copied, or summarized context is available when starting another agent; resume still depends on retained agent/executor state. | -| Agent-spawned sessions and targeted task messages | Dependency-bound | `spawn_session_kandev` and `message_task_kandev` are task-MCP tools. Queue limits, session IDs, parent/child interrupt rules, and agent MCP capability apply. | +| Agent-spawned sessions, targeted task messages, and direct-child stop | Dependency-bound | `spawn_session_kandev`, `message_task_kandev`, and `stop_task_kandev` are task-MCP tools. Queue limits, session IDs, direct-parent authority, and agent MCP capability apply. Stop is halt-only, covers all observed live sessions on the child, and schedules runtime teardown asynchronously. | | One-level subtasks | Supported | Regular Kanban supports a parent plus direct children through UI and MCP. A child may inherit the parent's materialized workspace or request a new one; inherited/shared files can conflict. Arbitrary-depth Office trees remain in progress. | | Blocker/dependency editing | In progress | Same-workspace storage and cycle checks exist, and related-task MCP can report stored relations. The picker and blocker properties are currently Office UI; regular Kanban has no complete create/edit/filter path. | | Multi-repository tasks | Limited | New Task supports multiple local or remote GitHub rows, and Changes/Review/PR surfaces group by repository. Two or more repositories currently require the Worktree executor. | @@ -89,7 +89,7 @@ Released builds can lag this page. Check **Settings > System > About** or `kande |---|---|---| | Task-scoped Kandev MCP | Dependency-bound | Injected into compatible task sessions with scoped task/session, plan, walkthrough, branch, workflow-step, and coordination tools. Available tools vary by MCP mode and task/session context. | | Profile-defined external MCP servers | Limited | Stdio, HTTP, SSE, and streamable HTTP are implemented, but current launch resolution starts from an allow-all baseline and overlays only explicit executor policy. Blank remote policies therefore allow transports instead of applying the separate deny-all remote default. A stdio process is per-session and cannot be shared like a network server. | -| External Kandev MCP endpoint | Supported | **Settings > External MCP** exposes streamable HTTP at `/mcp` and SSE at `/mcp/sse` for configuration and task management. It has no Kandev user authentication; keep it on loopback/VPN or place it behind an authenticated proxy. It intentionally lacks live-session question, plan, walkthrough, spawn, and message tools. | +| External Kandev MCP endpoint | Supported | **Settings > External MCP** exposes streamable HTTP at `/mcp` and SSE at `/mcp/sse` for configuration and task management. It has no Kandev user authentication; keep it on loopback/VPN or place it behind an authenticated proxy. It intentionally omits `stop_task_kandev` along with live-session question, plan, walkthrough, spawn, and message tools. | | External MCP tool preview | Limited | The endpoint registration is authoritative. The current settings preview omits several registered tools and should not be used as a complete allow-list. | | Office MCP and autonomous coordination | In progress | Office-only document, team, routine, and coordinator behavior is feature-gated and is not part of the supported external or regular task MCP contract. | | CLI/browser runtime, Linux/macOS service, Docker deployment, and remote hosting | Dependency-bound | Platform, filesystem, browser, systemd/launchd, container, reverse-proxy, and network configuration determine availability. The web app, API, WebSocket, and external MCP add no Kandev user-login boundary; protect the whole remotely reachable origin. | diff --git a/docs/public/sessions-and-review.md b/docs/public/sessions-and-review.md index 8684a047ff..8460559fdb 100644 --- a/docs/public/sessions-and-review.md +++ b/docs/public/sessions-and-review.md @@ -48,7 +48,7 @@ Right-click an agent tab on desktop to manage it. Available actions depend on it |---|---| | **Rename** | Changes the session's display name | | **Set as Primary** | Makes a stoppable session the task's primary target | -| **Stop** | Cancels the active agent turn | +| **Stop** | Cancels the active agent turn for this session | | **Resume** | Attempts to continue a completed, failed, or cancelled session | | **Delete** | Permanently removes the conversation; if it was primary, another session is promoted when possible | | **Share** | Opens the publishing preview for an eligible session | @@ -63,10 +63,11 @@ A CLI-passthrough profile displays the agent's native terminal interface in a PT ## Let agents coordinate sessions -Task MCP gives an agent two coordination operations: +Task MCP gives an agent three session-coordination operations: - `spawn_session_kandev` starts another session on the current task by default. It can select a profile and name, and can target another task in the same workspace. The new session shares the target task's environment; its supplied prompt is its initial context. - `message_task_kandev` sends work to a task's primary session or to an explicit session ID. A same-task sibling must be addressed by session ID, and a session cannot message itself. +- `stop_task_kandev` asks the current task to halt all live sessions on one same-workspace direct child. It sends no prompt and has no session-specific option. Delivery follows the target state: @@ -77,6 +78,8 @@ Delivery follows the target state: The default pending-message limit is 10 per session. Operators can change it with `KANDEV_QUEUE_MAX_PER_SESSION`; a value of `0` or less removes the cap, so the queue can grow without that bound. Interrupt delivery is restricted to a direct parent task messaging its child. Other senders must queue. +For urgent replacement work, the parent should use `message_task_kandev` with `delivery_mode: "interrupt"`; this cancels the current approach and immediately tries to dispatch the new prompt, with a safe queued fallback. Use `stop_task_kandev` only for halt-only intent. A successful stop marks every accepted live child session `CANCELLED` and schedules graceful teardown asynchronously. Kandev then attempts to move an eligible unarchived, non-Office task from `IN_PROGRESS` or `SCHEDULING` to `REVIEW`; other task states remain unchanged. A child with no live execution returns idempotent `not_running`, and its worktrees, environment, commits, task record, descendants, and queued messages are preserved. See [Coordination](coordination.md) for the complete authority and lifecycle contract. + Messages show peer attribution, and Kandev gives the receiving agent hidden reply instructions. The receiver can still decline the request. A full task UUID is sufficient for cross-workspace messaging, so treat task IDs as sensitive routing identifiers when untrusted agents share one deployment. See [Coordination](coordination.md) and [Automation and MCP](automation-and-mcp.md). ## Use the workbench diff --git a/docs/public/websocket-api.md b/docs/public/websocket-api.md index 075e51078b..9d03abdb89 100644 --- a/docs/public/websocket-api.md +++ b/docs/public/websocket-api.md @@ -29,7 +29,9 @@ An action constant alone is not evidence that an action is registered or emitted The current `/ws` upgrade handler does **not authenticate clients**. It reads `?token=` or the `Authorization` header but does not validate or use the value; JWT validation is still a code TODO. The default backend host is `0.0.0.0`, so a default process can listen on every interface even though examples use `localhost`. -Treat every client that can reach the backend as fully trusted. WebSocket actions can create and delete data, start agents and shells, read and change files, run Git operations, reveal stored secrets, and invoke configured integrations. Do not expose port `38429` directly to an untrusted LAN or the internet. Bind to loopback, firewall the port, or put Kandev behind an authenticated reverse proxy that terminates TLS and restricts access. See [Configuration](configuration.md) and [Run as a Service](run-as-a-service.md). +The raw gateway rejects every action whose name starts with `mcp.` using a `FORBIDDEN` error before the shared dispatcher runs. This prevents raw clients from forging task or session identity that trusted MCP adapters inject; it is not user authentication. Internal MCP actions remain available through their mode-scoped MCP adapters and trusted in-process dispatch paths. + +Treat every client that can reach the backend as fully trusted for the remaining WebSocket surface. Actions can create and delete data, start agents and shells, read and change files, run Git operations, reveal stored secrets, and invoke configured integrations. Do not expose port `38429` directly to an untrusted LAN or the internet. Bind to loopback, firewall the port, or put Kandev behind an authenticated reverse proxy that terminates TLS and restricts access. See [Configuration](configuration.md) and [Run as a Service](run-as-a-service.md). The upgrade does enforce an origin policy for browser clients: @@ -236,7 +238,7 @@ websocat ws://127.0.0.1:38429/ws ## Registered request action catalog -The following 309 unique action names have concrete dispatcher registrations in the current backend. The 12 subscription/focus actions in the previous table are additional gateway-handled requests. Availability can still depend on a configured integration, handler mode, or service; registration does not supply credentials, provider installation, a running executor, or permission to external systems. +The following 310 unique action names have concrete dispatcher registrations in the current backend. The 12 subscription/focus actions in the previous table are additional gateway-handled requests. Availability can still depend on a configured integration, handler mode, or service; registration does not supply credentials, provider installation, a running executor, or permission to external systems. Payloads are not uniform. Read the corresponding handler request struct before building a non-first-party client. Names below are exact, including `vscode.openFile` and underscore-separated `user_shell.*` actions. @@ -605,6 +607,7 @@ mcp.reorder_workflow_steps mcp.show_walkthrough mcp.spawn_session mcp.step_complete +mcp.stop_task mcp.update_agent mcp.update_agent_profile mcp.update_executor_profile @@ -618,7 +621,7 @@ mcp.update_workflow_step mcp.write_task_document ``` -These registrations back Kandev's agent/MCP bridge. The subset registered in a process depends on its MCP handler mode and enabled capabilities. They are internal transport shims, not a reason to tunnel MCP calls manually over `/ws`; use the MCP tools exposed to the agent so tool schemas, task/session scoping, and compatibility handling remain intact. +These registrations back Kandev's agent/MCP bridge. The subset registered in a process depends on its MCP handler mode and enabled capabilities. They are internal transport shims: raw `/ws` rejects every one of them before handler dispatch. Use the MCP tools exposed to the agent so tool schemas, task/session scoping, and compatibility handling remain intact. In particular, `mcp.stop_task` is the internal action behind task-mode `stop_task_kandev`; External MCP does not register that tool. ## Emitted notifications and recipients diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index 6caae01932..cb10a93936 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -63,6 +63,7 @@ Kandev's task model: documents, execution stages, labels, blocker escalation, su | [quick-chat-sessions](tasks/quick-chat-expiration.md) | shipped | | [quick-chat-repository-context](tasks/quick-chat-repository-context.md) | shipped | | [parent-child-message-interrupt](tasks/parent-child-message-interrupt.md) | shipped | +| [parent-child-task-stop](tasks/parent-child-task-stop.md) | shipped | ## agents/ — agent governance diff --git a/docs/specs/tasks/parent-child-message-interrupt.md b/docs/specs/tasks/parent-child-message-interrupt.md index f5d7a45be1..b113fab8d7 100644 --- a/docs/specs/tasks/parent-child-message-interrupt.md +++ b/docs/specs/tasks/parent-child-message-interrupt.md @@ -239,4 +239,8 @@ itself via the ordinary FIFO drain — the message is not left stranded. whether to resend, and a resend queues a distinct message (queuing is not idempotent). - A `delivery_mode` value that queues *without* the possibility of a later natural interrupt, or that interrupts without also queuing — the two are always coupled (interrupt implies - queue-then-cancel-and-take as one atomic step). + queue-then-cancel-and-take as one atomic step). This interrupt mode remains the + preferred control when a direct parent has replacement instructions. A parent + that instead needs to halt child work without sending a replacement prompt + uses the separate [Parent-Child Task Stop](parent-child-task-stop.md) + capability. diff --git a/docs/specs/tasks/parent-child-task-stop.md b/docs/specs/tasks/parent-child-task-stop.md new file mode 100644 index 0000000000..6b718702d1 --- /dev/null +++ b/docs/specs/tasks/parent-child-task-stop.md @@ -0,0 +1,240 @@ +--- +status: shipped +created: 2026-07-19 +owner: cfl +--- + +# Parent-Child Task Stop + +## Why + +A coordinating task can already stop and steer a busy direct child with +`message_task_kandev(delivery_mode="interrupt")`. That is the preferred control +when the child should abandon its current approach and immediately act on a new +instruction. Agents rarely choose it today because the short injected MCP context +does not mention `delivery_mode`, while queued delivery is the default. + +A separate case remains: the coordinator no longer needs any work from the child. +Sending an interrupting message would cancel the current turn but immediately +start a replacement turn just to process that message. Coordinators need a +halt-only operation that terminates unnecessary child execution without waiting +for a turn boundary, queuing a message, or starting another agent turn. + +## What + +- Make the existing interrupt choice prominent in both the + `message_task_kandev` tool description and injected Kandev MCP context. The + guidance says to prefer `delivery_mode="interrupt"` for urgent stop-and-steer + instructions to a running direct child. +- Task-mode MCP exposes `stop_task_kandev` for stopping a current task's direct + child. +- Stopping is distinct from + `message_task_kandev(delivery_mode="interrupt")`: it sends no prompt and starts + no replacement turn. +- One call requests a graceful stop for every execution Kandev observes as live + while inspecting the target task's active-session candidates, including + non-primary sibling sessions. There is no session-specific option. +- Kandev marks each accepted session stop `CANCELLED`, initiates graceful runtime + teardown, and attempts the existing guarded task-level transition to `REVIEW` + for an active, unarchived, non-Office task. +- Runtime teardown starts immediately but remains asynchronous. A successful tool + response confirms Kandev accepted the stop and changed logical session state; + it does not claim the operating-system process has already exited. +- Worktrees, task environments, commits, task records, descendants, and existing + queued messages are preserved. The stopped task can be started again later. +- Stopping a child with no live execution is an idempotent success and does not + change task or session state. + +### Choosing the control + +| Coordinator intent | Operation | Result | +|---|---|---| +| Send information that can wait | `message_task_kandev` with `delivery_mode="queued"` or omitted | Current turn continues; message waits FIFO. | +| Stop the current approach and give replacement work now | `message_task_kandev` with `delivery_mode="interrupt"` | Requests immediate cancel-and-redispatch. If immediate dispatch cannot land safely, the message remains queued. | +| Stop all current work because nothing should replace it | `stop_task_kandev` | Logical cancellation is accepted and runtime teardown begins; no message or replacement turn is created. | + +## API surface + +Decision: +[ADR-2026-07-19-reject-mcp-actions-on-raw-websocket](../../decisions/2026-07-19-reject-mcp-actions-on-raw-websocket.md) + +Task-mode MCP tool request: + +```json +{ + "task_id": "" +} +``` + +The MCP server injects the calling task identity. Callers cannot supply or +override it. The backend uses a fixed, non-destructive semantic stop reason for +this operation; caller-controlled text is never passed into executor cleanup +logic. The tool does not expose the runtime `force` option and uses graceful +stop. + +The existing message API does not change. Its description and injected context +now explicitly advertise optional +`delivery_mode: "queued" | "interrupt"` and the stop-and-steer selection rule. + +Internal `mcp.stop_task` action payload: + +```json +{ + "task_id": "", + "sender_task_id": "" +} +``` + +This action is an internal task-MCP transport surface, not a public raw WebSocket +authorization API. The raw gateway rejects every `mcp.*` action before shared +dispatcher invocation; agent-stream and mode-scoped MCP adapters bypass that raw +path. The task-mode MCP server supplies `sender_task_id`; unknown payload fields +are ignored and cannot alter the fixed reason or graceful-stop policy. External +MCP does not register the stop tool. + +Accepted stop response: + +```json +{ + "task_id": "", + "status": "stopped" +} +``` + +Idempotent no-work response: + +```json +{ + "task_id": "", + "status": "not_running" +} +``` + +The backend action is `mcp.stop_task`. + +`status="stopped"` means Kandev persisted logical cancellation for at least one +execution and scheduled graceful teardown for every execution still live in the +candidate set when inspected. It does not confirm process exit. + +### Errors + +| Condition | Code | Side effects | +|---|---|---| +| Malformed JSON | `BAD_REQUEST` | None | +| Raw `/ws` caller attempts any `mcp.*` action | `FORBIDDEN` | Handler is not invoked | +| Missing `task_id` or trusted caller identity | `VALIDATION_ERROR` | None | +| Caller task or target task does not exist | `NOT_FOUND` | None | +| Target is not the caller's direct child or is in another workspace | `FORBIDDEN` | None | +| Repository/lifecycle lookup, session `CANCELLED` write, or one or more synchronous stop attempts fails | `INTERNAL_ERROR` | Sessions already cancelled remain cancelled; untouched or failed candidates can be retried | + +## Permissions + +Only the target task's direct parent may call `stop_task_kandev`. Self, sibling, +child-to-parent, grandparent, unrelated, and cross-workspace callers are rejected +before any session or runtime mutation. The check uses the same direct-parent +predicate as parent interrupt messages, plus a defense-in-depth same-workspace +check. See +[Parent-Child Message Interrupt](parent-child-message-interrupt.md). + +The tool is absent from Config, Office, and External MCP modes. Those modes either +lack trusted current-task identity or use a different mutation surface. +The raw browser WebSocket rejects the entire `mcp.*` action namespace before +dispatcher routing, so it cannot forge the task-mode server's injected identity. + +## State and persistence + +- Each target execution found live when its candidate session is inspected + transitions its session to `CANCELLED` before teardown is scheduled for that + session. If persistence + fails, that session is not torn down by this attempt and the call returns an + error, so a retry can make progress without leaving a live-looking orphan row. +- Stop uses the existing per-session `cancelInFlightGuard` while rechecking and + accepting logical cancellation. This serializes it with interrupt, ready-event, + pending-move, and queued-message drain decisions; the guard is released before + asynchronous process teardown. +- After accepted stops, Kandev attempts to transition a non-Office, unarchived + task from `IN_PROGRESS` or `SCHEDULING` to `REVIEW` through the normal + guarded task update and event path. Office-managed, archived, and + already-terminal/non-active task states remain unchanged. A failure in this + secondary board-state reconciliation is logged but does not negate accepted + execution stops. +- Runtime process and executor cleanup continue asynchronously after the response. +- Existing message-queue entries are not removed or dispatched by this tool. They + retain their normal behavior if the task is later restarted. +- Worktrees and task environments remain available for review or resume. +- No durable pause or cancellation gate is created; later user or workflow actions + may start the task again. +- Candidate sessions are inspected sequentially. A launch that registers its + lifecycle execution after its candidate was inspected can escape the call, + whether that launch began before or after stopping started. The first version + does not add a task-wide launch fence. + +## Failure modes + +- If every candidate completes naturally before stop reaches it, the orchestrator + returns `status="not_running"` after each candidate is found absent. +- Repeating a successful stop returns `status="not_running"` without another + state transition. +- Only `lifecycle.ErrNoExecutionForSession` means a candidate naturally has no + live execution. Repository errors, other lifecycle lookup errors, and an empty + execution ID without that sentinel remain internal errors; they are not + collapsed into `not_running`. +- Authorization and existence checks complete before stop side effects. +- A synchronous partial failure is reported as an error rather than a false full + success. Sessions whose stop already succeeded remain cancelled, so retrying + targets only work that is still live. +- If zero stops are accepted and every candidate has naturally + vanished, the result is `not_running`. If one or more stops are accepted and + the rest have naturally vanished, the result is `stopped`. Any genuine lookup + or persistence failure returns `INTERNAL_ERROR`. +- Failure during asynchronous process teardown is logged after the tool response. + The first version does not add teardown retry/reconciliation, so a logical + `CANCELLED` state may disagree with a lingering process until later manual or + system cleanup. + +## Scenarios + +- **GIVEN** a direct child is pursuing the wrong approach but still has replacement + work, **WHEN** its parent needs to steer it now, **THEN** the tool guidance tells + the parent to use + `message_task_kandev(delivery_mode="interrupt")`, which requests immediate + cancel-and-redispatch and truthfully reports `sent` or safe `queued` + fallback. +- **GIVEN** a direct child has one running session and no replacement work, + **WHEN** its parent calls + `stop_task_kandev`, **THEN** the session becomes `CANCELLED`, runtime teardown + begins, an eligible active Kanban task moves to `REVIEW`, no prompt is + created, and the response is `status="stopped"`. +- **GIVEN** a direct child has multiple live sessions, **WHEN** its parent stops + the task, **THEN** every still-live session accepts a stop before the call + reports `status="stopped"`; a session that completes naturally during the + operation counts as no longer live. +- **GIVEN** a sibling, grandparent, child, self, unrelated task, or task in another + workspace, **WHEN** it calls `stop_task_kandev`, **THEN** the call returns + `FORBIDDEN` and no target state changes. +- **GIVEN** a direct child has no live execution, **WHEN** its parent calls the tool + once or repeatedly, **THEN** each call returns `status="not_running"` and leaves + task and session state unchanged. +- **GIVEN** a direct child has descendants, **WHEN** its parent stops that child, + **THEN** descendant sessions and task states are unchanged. +- **GIVEN** a target has existing queued messages, **WHEN** its parent stops the + task, **THEN** the stop tool creates and dispatches no message and the existing + queue entries remain stored for normal later handling. + +## Out of scope + +- Cancelling only one session or only the current turn. +- Sending a replacement prompt; use + `message_task_kandev(delivery_mode="interrupt")` for stop-and-steer behavior. +- Changing queued delivery to implicit interruption. Interrupt remains an explicit + per-call choice. +- Recursively pausing or cancelling a subtree. +- Setting the task state to `CANCELLED`, archiving, deleting, or cleaning up its + worktree or task environment. +- Stopping self, siblings, ancestors, grandchildren, or unrelated tasks. +- A force/SIGKILL option or waiting synchronously for process exit. +- A durable pause flag, task-wide launch fence, or guarantee against a launch + that registers its lifecycle execution after its candidate session was + inspected. +- Clearing or delivering existing queued messages. +- External/admin MCP stop access.