diff --git a/apps/backend/internal/sysprompt/sysprompt.go b/apps/backend/internal/sysprompt/sysprompt.go index c42bd2d64a..c7570ebb49 100644 --- a/apps/backend/internal/sysprompt/sysprompt.go +++ b/apps/backend/internal/sysprompt/sysprompt.go @@ -132,7 +132,7 @@ func KandevContext() string { }) } -const userQuestionSection = `- ask_user_question_kandev: Ask the user 1-4 related questions and wait for all answers. Use this whenever you need user input to proceed. +const userQuestionSection = `- ask_user_question_kandev: Ask the user 1-4 related questions and wait for all answers. Use this whenever you need user input to proceed. Treat this call as a hard user-input barrier: do not call another tool, do not continue working, and do not provide a final response until the tool returns completed user answers or a structured rejection. If the tool reports a validation error before creating a question, correct the request and retry. If an accepted question returns without completed answers or a structured rejection, end your turn immediately. This includes a timeout, disconnect, or pending wait. For an incomplete result, do not infer an answer or continue the task. ` const parentQuestionSection = `- ask_parent_question_kandev: Ask the direct parent task one or more critical questions. Use this only when you cannot continue safely without a parent decision. The question is sent to the parent task, and this call MUST end your turn; do not use an operator-question tool. diff --git a/apps/backend/internal/sysprompt/sysprompt_test.go b/apps/backend/internal/sysprompt/sysprompt_test.go index 5d0bafd3b5..f8317e4234 100644 --- a/apps/backend/internal/sysprompt/sysprompt_test.go +++ b/apps/backend/internal/sysprompt/sysprompt_test.go @@ -129,6 +129,21 @@ func TestKandevContext_PrefersNativeSubagentsForOrdinaryDelegation(t *testing.T) assert.Contains(t, context, "do not silently create a Kandev task or session") } +func TestFormatKandevContext_UserQuestionIsHardInputBarrier(t *testing.T) { + context := FormatKandevContextWithOptions("task-abc", "session-xyz", KandevContextOptions{ + IncludeUserQuestionTool: true, + IncludeCoordinatorTaskControls: true, + }) + + assert.Contains(t, context, "hard user-input barrier") + assert.Contains(t, context, "do not call another tool") + assert.Contains(t, context, "do not continue working") + assert.Contains(t, context, "do not provide a final response") + assert.Contains(t, context, "until the tool returns completed user answers or a structured rejection") + assert.Contains(t, context, "If the tool reports a validation error before creating a question, correct the request and retry") + assert.Contains(t, context, "If an accepted question returns without completed answers or a structured rejection, end your turn immediately") +} + func TestFormatKandevContext_TitleToolFollowsCapability(t *testing.T) { withoutTitle := FormatKandevContextWithOptions("task-abc", "session-xyz", KandevContextOptions{}) assert.NotContains(t, withoutTitle, "set_task_title_kandev") diff --git a/docs/plans/user-question-turn-boundary/plan.md b/docs/plans/user-question-turn-boundary/plan.md new file mode 100644 index 0000000000..d823a54260 --- /dev/null +++ b/docs/plans/user-question-turn-boundary/plan.md @@ -0,0 +1,138 @@ +--- +spec: docs/specs/tasks/user-question-turn-boundary.md +created: 2026-08-22 +status: shipped +--- + +# Fix Plan: Enforce the User Question Turn Boundary + +## Overview + +Strengthen the capability-aware `ask_user_question_kandev` entry in the +first-turn task system context and bind the hard-wait wording to a focused +regression test. The transport and clarification lifecycle already block and +recover unanswered calls, so the repair stays within prompt composition. + +## Confirmed root cause + +`apps/backend/internal/sysprompt/sysprompt.go` currently describes +`ask_user_question_kandev` as waiting for all answers. It does not define the +agent's actions after the call. It also gives no instruction for a result that +contains no answers. By contrast, the neighboring parent-question and +autopilot guidance says that the question ends the turn. That guidance also +forbids further tool calls or work. + +The handler in `apps/backend/internal/mcp/server/handlers.go` already waits on +the backend and streams progress keepalives while the user considers the +question. Its existing test proves the final result is withheld until the +backend releases an answer. The remaining failure occurs when an agent or MCP +client treats an early non-answer result as permission to continue. The normal +task prompt does not define a hard input barrier. + +Smallest source-level reproduction: format a normal task context with +`sysprompt.FormatKandevContextWithOptions` and inspect the +`ask_user_question_kandev` bullet. It contains "wait for all answers." It has +no instruction to avoid another tool call, further work, or a final response. +It has no fail-closed action for timeout, disconnect, or pending results. + +## Backend + +### Capability-aware prompt guidance + +- Update `userQuestionSection` in + `apps/backend/internal/sysprompt/sysprompt.go` so the tool entry remains a + concise overview while defining a hard user-input barrier. +- State that the agent must not call another tool, continue working, or produce + a final response until the tool returns completed user answers or a + structured rejection. +- State that a validation error before question creation is recoverable. The + agent corrects the request and retries. +- State that an accepted return without completed answers or a structured + rejection requires the agent to end the turn immediately. Preserve the + ability to continue when either usable result is returned. +- Keep the text inside the capability-controlled section. Do not add it to the + static `kandev-context.md` template. That location exposes the instruction to + autopilot roots that have no user-question tool. + +## Frontend + +No frontend change. The system context is hidden from the rendered chat, and +the clarification overlay and waiting-state behavior remain unchanged. + +## Tests + +- **What:** A normal task context describes `ask_user_question_kandev` as a hard + barrier: no later tool call, continued work, or final response before answers, + and immediate turn end for a non-answer result. + **File:** `apps/backend/internal/sysprompt/sysprompt_test.go`. + **How:** Add a focused formatting test against a normal task context. Write it + first and confirm it fails because the current bullet has only the weaker + "wait for all answers" wording. +- **What:** Capability boundaries and the live question schema remain aligned. + **Files:** `apps/backend/internal/sysprompt/sysprompt_test.go` and + `apps/backend/internal/mcp/server/sysprompt_sync_test.go`. + **How:** Run the existing autopilot question-capability tests and schema-sync + test with the new regression. +- **What:** The actual MCP handler continues withholding its final result while + the backend waits for a user answer. + **File:** `apps/backend/internal/mcp/server/ask_user_question_test.go`. + **How:** Rerun the existing streamable HTTP keepalive regression; do not + change transport code. + +No browser E2E is needed because this changes hidden agent instructions, not a +rendered user interaction. + +## Verification Results + +- RED: `cd apps/backend && go test ./internal/sysprompt -run + '^TestFormatKandevContext_UserQuestionIsHardInputBarrier$' -count=1` — failed + as expected because the normal prompt lacked the barrier wording. +- GREEN: `cd apps/backend && go test ./internal/sysprompt -run + '^TestFormatKandevContext_UserQuestionIsHardInputBarrier$' -count=1` — passed, + 1 test. +- Review RED: the same regression failed after adding the pre-acceptance retry + assertions because the first prompt did not distinguish validation errors. +- Review GREEN: the focused regression passed after the prompt distinguished + validation errors from accepted-question waits. +- Review RED: the focused regression failed after adding the structured- + rejection assertion because the turn-boundary wording did not make that + usable result explicit. +- Review GREEN: the focused regression passed after the prompt limited the + fail-closed rule to accepted results with neither answers nor rejection. +- Task checks: `cd apps/backend && go test ./internal/sysprompt ./internal/mcp/server + -run 'TestFormatKandevContext_(UserQuestionIsHardInputBarrier|AutopilotChildUsesParentQuestionOnly|AutopilotRootHasNoQuestionTool)|TestAskUserQuestionDocs_MatchSchema|TestAskUserQuestion_StreamsKeepAliveDuringWait' -count=1` + — passed, 5 tests across 2 packages. +- `git diff --check` and checks for all new design files — passed. +- Baseline evidence from planning remains valid: the existing sysprompt package + suite passed 60 tests, and the two pre-existing MCP tests passed. + +## Implementation Waves And Parallel Candidates + +Wave 1: + +- [x] [Task 01: Strengthen question wait guidance](task-01-strengthen-question-wait-guidance.md) — done + +Execution is sequential in the primary conversation. No subagents are +authorized. + +## Risks + +- Prompt text cannot mechanically prevent every model from disobeying, so the + regression can prove only that every normal first-turn context contains the + unambiguous barrier. +- Wording that says every question call always ends the turn can incorrectly + forbid the healthy path where the blocking call returns completed answers or + a structured rejection. The repair must distinguish usable results from + early incomplete returns. +- Validation errors occur before a question enters the clarification lifecycle. + The prompt must keep those errors retryable. +- Putting the rule in the static template can expose instructions for a tool + absent from autopilot roots. Keep it in `userQuestionSection`. + +## Out of Scope + +- MCP handler, keepalive, timeout, clarification persistence, and resume changes. +- Tool schema or output changes. +- Config, Office, external, or autopilot prompt changes. +- Public documentation changes. The externally observable MCP contract is + unchanged. diff --git a/docs/plans/user-question-turn-boundary/task-01-strengthen-question-wait-guidance.md b/docs/plans/user-question-turn-boundary/task-01-strengthen-question-wait-guidance.md new file mode 100644 index 0000000000..36d2fa0358 --- /dev/null +++ b/docs/plans/user-question-turn-boundary/task-01-strengthen-question-wait-guidance.md @@ -0,0 +1,105 @@ +--- +id: "01-strengthen-question-wait-guidance" +title: "Strengthen question wait guidance" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/tasks/user-question-turn-boundary.md" +--- + +# Task 01: Strengthen Question Wait Guidance + +## Acceptance + +1. Every normal task first-turn context that exposes + `ask_user_question_kandev` explicitly forbids another tool call, continued + work, or a final response until completed user answers or a structured + rejection return. +2. The same guidance requires an immediate turn end when an accepted call + returns without completed answers or a structured rejection, while either + usable result remains actionable. +3. A validation error before question creation remains retryable, and the prompt + does not tell the agent to end the turn for that error. +4. Autopilot child/root capability guidance and the question tool schema, + transport, timeout, and persistence behavior remain unchanged. + +## Verification + +Follow strict TDD, then run: + +```bash +cd apps/backend +go test ./internal/sysprompt ./internal/mcp/server -run \ + 'TestFormatKandevContext_(UserQuestionIsHardInputBarrier|AutopilotChildUsesParentQuestionOnly|AutopilotRootHasNoQuestionTool)|TestAskUserQuestionDocs_MatchSchema|TestAskUserQuestion_StreamsKeepAliveDuringWait' \ + -count=1 +cd ../.. +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/sysprompt/sysprompt.go` +- `apps/backend/internal/sysprompt/sysprompt_test.go` +- `docs/specs/tasks/user-question-turn-boundary.md` +- `docs/specs/INDEX.md` +- `docs/plans/user-question-turn-boundary/plan.md` +- `docs/plans/user-question-turn-boundary/task-01-strengthen-question-wait-guidance.md` + +## Dependencies + +None. + +## Parallelism + +`sequential` — the prompt wording and its regression test are one localized +behavioral repair. + +## Inputs + +- [User Question Turn Boundary spec](../../specs/tasks/user-question-turn-boundary.md) +- [Fix plan](plan.md) +- `userQuestionSection`, `parentQuestionSection`, and `autopilotSection` in + `apps/backend/internal/sysprompt/sysprompt.go` +- Existing question capability tests in + `apps/backend/internal/sysprompt/sysprompt_test.go` +- Existing blocking/keepalive regression in + `apps/backend/internal/mcp/server/ask_user_question_test.go` +- [ADR 0015](../../decisions/0015-explicit-completion-signal-for-auto-advance.md) + +## Risks + +- Keep the distinction between an answered call and an early non-answer return + explicit. Otherwise, the prompt can stop useful work after the user replies. +- Keep pre-acceptance validation errors retryable because no clarification wait + exists for those calls. +- Keep the new text capability-scoped so an autopilot root never receives + instructions for an unavailable tool. + +## Output contract + +Report red/green evidence, the final prompt contract, files changed, exact test +results, residual risks, and synchronized task/plan status. + +## Results + +- RED: `cd apps/backend && go test ./internal/sysprompt -run + '^TestFormatKandevContext_UserQuestionIsHardInputBarrier$' -count=1` failed + because `userQuestionSection` did not state a hard input barrier. +- GREEN: The same focused test passed after the prompt update. +- Review RED: the focused test failed after adding the validation-retry + assertion because the prompt did not distinguish pre-acceptance errors. +- Review GREEN: the focused test passed after the distinction was added. +- Review RED: the focused test failed after adding the structured-rejection + assertion because the prompt did not define that result as usable. +- Review GREEN: the focused test passed after the fail-closed rule was limited + to accepted results with neither completed answers nor structured rejection. +- `cd apps/backend && go test ./internal/sysprompt ./internal/mcp/server -run + 'TestFormatKandevContext_(UserQuestionIsHardInputBarrier|AutopilotChildUsesParentQuestionOnly|AutopilotRootHasNoQuestionTool)|TestAskUserQuestionDocs_MatchSchema|TestAskUserQuestion_StreamsKeepAliveDuringWait' -count=1` + — passed, 5 tests across 2 packages. +- `gofmt -w internal/sysprompt/sysprompt.go internal/sysprompt/sysprompt_test.go` completed. +- `git diff --check` and checks for the three new design files — passed. +- Changed files match the task scope: the sysprompt constant, its regression + test, the spec index, the repair spec, the plan, and this task file. +- No browser E2E ran because the change affects hidden agent instructions only. +- No security, external side effects, or cleanup work applies. diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index 71fa6b64d8..4ed772d459 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -114,6 +114,7 @@ Kandev's task model: documents, execution stages, labels, blocker escalation, su | [prompt-attachments](tasks/prompt-attachments.md) | draft | | [sidebar-task-edit](tasks/sidebar-task-edit.md) | approved | | [autopilot-mode](tasks/autopilot-mode.md) | draft | +| [user-question-turn-boundary](tasks/user-question-turn-boundary.md) | shipped | | [explicit-completion-signal](workflow/explicit-completion-signal/spec.md) | shipped | | [cancelled-turn-completion](workflow/cancelled-turn-completion/spec.md) | building | | [task-step-transition-ledger](workflow/task-step-transition-ledger/spec.md) | draft | diff --git a/docs/specs/tasks/user-question-turn-boundary.md b/docs/specs/tasks/user-question-turn-boundary.md new file mode 100644 index 0000000000..0397077996 --- /dev/null +++ b/docs/specs/tasks/user-question-turn-boundary.md @@ -0,0 +1,72 @@ +--- +status: shipped +created: 2026-08-22 +owner: kandev +--- + +# User Question Turn Boundary + +## Why + +An agent can ask for required user input and then continue the same turn before +the user answers. That behavior lets the agent make decisions without the input +it declared necessary, especially when an MCP client returns early because the +question wait timed out or disconnected. + +## What + +- The first-turn Kandev MCP overview for a normal task identifies + `ask_user_question_kandev` as a hard user-input barrier. +- After calling `ask_user_question_kandev`, the agent does not call another tool, + continue working, or produce a final response until the tool returns the + user's answers. +- If the tool reports a validation error before it creates a question, the agent + corrects the request and retries it. +- If the accepted call returns without completed user answers or a structured + rejection, including a timeout, disconnect, or pending result, the agent ends + the turn immediately. It does not infer an answer or continue the task for + that incomplete result. +- When the call returns completed user answers, the agent can continue using + those answers. +- The guidance appears only when the task MCP profile exposes + `ask_user_question_kandev`. Existing autopilot parent-question and no-question + profiles keep their current guidance and capability boundaries. + +## Failure modes + +- If the MCP wait ends without completed answers or a structured rejection, the + prompt requires a fail-closed turn boundary. The existing clarification + lifecycle remains responsible for preserving or resuming the pending + question. +- If validation fails before a question is accepted, the agent can correct the + request and retry. No clarification wait exists in this path. +- If the user rejects the question bundle, the structured rejection is a + completed user response. The agent can handle that response without inventing + a selected option. + +## Scenarios + +- **GIVEN** a normal task whose MCP profile exposes + `ask_user_question_kandev`, **WHEN** Kandev builds the first-turn system + context, **THEN** the tool overview says not to make another tool call, + continue work, or provide a final response until user answers return. +- **GIVEN** the agent calls `ask_user_question_kandev`, **WHEN** the tool returns + completed answers, **THEN** the agent can continue the task using those + answers. +- **GIVEN** the agent calls `ask_user_question_kandev`, **WHEN** the accepted + call returns without completed answers or a structured rejection, **THEN** + the agent ends the turn immediately and performs no further task work. +- **GIVEN** the agent submits an invalid question request, **WHEN** MCP returns a + validation error before creating a question, **THEN** the agent corrects the + request and retries instead of ending the turn. +- **GIVEN** an autopilot child or root task, **WHEN** Kandev builds its first-turn + system context, **THEN** the existing parent-question or no-question guidance + remains unchanged and the user-question guidance is absent. + +## Out of scope + +- Changing the `ask_user_question_kandev` schema, response envelope, timeout, + keepalive, persistence, or resume behavior. +- Changing configuration-mode, Office-mode, or external MCP prompt surfaces. +- Adding model-specific prompt variants or runtime enforcement that cancels an + agent immediately after the tool call.