10073 lines
390 KiB
Go
10073 lines
390 KiB
Go
// Package orchestrator provides the main orchestrator service that ties all components together.
|
|
package orchestrator
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
|
client "github.com/kandev/kandev/internal/agent/runtime/agentctl"
|
|
dynamicruntime "github.com/kandev/kandev/internal/agent/runtime/dynamic"
|
|
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
|
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
|
|
"github.com/kandev/kandev/internal/common/constants"
|
|
"github.com/kandev/kandev/internal/editors/capabilities"
|
|
"github.com/kandev/kandev/internal/events"
|
|
"github.com/kandev/kandev/internal/events/bus"
|
|
"github.com/kandev/kandev/internal/orchestrator/dto"
|
|
"github.com/kandev/kandev/internal/orchestrator/executor"
|
|
"github.com/kandev/kandev/internal/orchestrator/messagequeue"
|
|
"github.com/kandev/kandev/internal/orchestrator/queue"
|
|
"github.com/kandev/kandev/internal/sysprompt"
|
|
"github.com/kandev/kandev/internal/task/models"
|
|
wfmodels "github.com/kandev/kandev/internal/workflow/models"
|
|
workflowmove "github.com/kandev/kandev/internal/workflow/move"
|
|
v1 "github.com/kandev/kandev/pkg/api/v1"
|
|
)
|
|
|
|
// PromptResult contains the result of a prompt operation
|
|
type PromptResult struct {
|
|
StopReason string // The reason the agent stopped (e.g., "end_turn")
|
|
AgentMessage string // The agent's accumulated response message
|
|
TurnID string // The exact turn accepted for this prompt, when known.
|
|
}
|
|
|
|
// CoordinatorTaskStopStatus is the idempotent product result returned to a
|
|
// parent task that asks to halt a direct child.
|
|
type CoordinatorTaskStopStatus string
|
|
|
|
const (
|
|
CoordinatorTaskStopStatusStopped CoordinatorTaskStopStatus = "stopped"
|
|
CoordinatorTaskStopStatusNotRunning CoordinatorTaskStopStatus = "not_running"
|
|
coordinatorMCPStopReason = "stopped by parent task via MCP"
|
|
)
|
|
|
|
// CoordinatorTaskStopResult reports whether at least one live child session
|
|
// accepted logical cancellation. Runtime teardown continues asynchronously.
|
|
type CoordinatorTaskStopResult struct {
|
|
Status CoordinatorTaskStopStatus `json:"status"`
|
|
}
|
|
|
|
// resumeReasonErrorRecovery is the resume reason returned when a session is in
|
|
// error-recovery state (WAITING_FOR_INPUT with a non-empty ErrorMessage).
|
|
const resumeReasonErrorRecovery = "error_recovery"
|
|
|
|
// resumeReasonFailedSessionResumable is the resume reason returned when a
|
|
// FAILED session is auto-resumed because its runtime is Resumable. Distinct
|
|
// from "agent_not_running" so log filtering can isolate FAILED auto-resumes.
|
|
const resumeReasonFailedSessionResumable = "failed_session_resumable"
|
|
|
|
// resumeReasonArchiveCancelledResumable is the resume reason returned when a
|
|
// session cancelled by archiving (see models.IsArchiveCancelReason) is
|
|
// resumable once its task is unarchived — the cancellation was a system side
|
|
// effect of Service.ArchiveTask / HandoffService's cascade archive, not an
|
|
// explicit user/coordinator stop, so it must recover like a FAILED session
|
|
// instead of staying stuck on read-only workspace restore.
|
|
const resumeReasonArchiveCancelledResumable = "archive_cancelled_resumable"
|
|
|
|
// resumeReasonOrphanCancelledResumable is the resume reason returned when the
|
|
// session reconciliation sweep cancelled an execution-less session. Existing
|
|
// rows with this exact system reason remain eligible for same-session recovery;
|
|
// explicit user/coordinator stops continue through the terminal-state path.
|
|
const resumeReasonOrphanCancelledResumable = "orphan_cancelled_resumable"
|
|
|
|
// resumeReasonInterruptedSessionResumable is returned for a session that the
|
|
// restart/stall sweep settled to WAITING_FOR_INPUT after execution loss. The
|
|
// durable session marker keeps this shape resumable even when the executor row
|
|
// was removed before the sweep ran.
|
|
const resumeReasonInterruptedSessionResumable = "interrupted_session_resumable"
|
|
|
|
// resumeReasonTaskArchived tells clients that recovery is unavailable until
|
|
// the owning task is unarchived. It is intentionally distinct from a runtime
|
|
// failure so archived history never enters recovery UI.
|
|
const resumeReasonTaskArchived = "task_archived"
|
|
|
|
var ErrAgentPromptInProgress = errors.New("agent is currently processing a prompt")
|
|
var ErrAgentNotReadyForPrompt = errors.New("agent not ready for prompt")
|
|
var ErrSessionResetInProgress = errors.New("session reset in progress")
|
|
|
|
// ErrSessionRuntimeUnavailable is returned by promptTask when
|
|
// ensureSessionRunning fails for the single reason that is safe to treat as
|
|
// "not yet launched" (see errSessionAwaitingRuntimeLaunch) — e.g. a session
|
|
// promoted by a workflow step move whose runtime has not finished launching
|
|
// yet. Callers can safely queue the prompt for delivery once the runtime
|
|
// comes up instead of reporting it as failed, because nothing was
|
|
// dispatched. Every other ensureSessionRunning failure (a real launch
|
|
// error, an office-scheduler refusal, an exhausted resume retry) keeps
|
|
// surfacing as a plain, visible error instead.
|
|
var ErrSessionRuntimeUnavailable = errors.New("session runtime unavailable")
|
|
|
|
// errSessionAwaitingRuntimeLaunch marks attemptColdResume's "prepared but
|
|
// not launched yet" outcome: the session has no executors_running row
|
|
// because the launch that creates one has not run yet, not because a launch
|
|
// attempt failed. This is the only ensureSessionRunning failure narrow
|
|
// enough to be classified ErrSessionRuntimeUnavailable — a session that
|
|
// tried and failed to come up (bad SSH handshake, exhausted cold-resume
|
|
// retries, an office-scheduler refusal) must not be silently queued with no
|
|
// visible error and no log, since there is no future launch that will drain
|
|
// the queue for it.
|
|
var errSessionAwaitingRuntimeLaunch = errors.New("session is not resumable: no executor record")
|
|
|
|
type primarySessionTaskStateUpdater interface {
|
|
UpdateTaskStateIfPrimarySessionState(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
expectedSessionState models.TaskSessionState,
|
|
state v1.TaskState,
|
|
) (bool, error)
|
|
}
|
|
|
|
// dynamicRouteStateLoader exposes the durable route row without coupling the
|
|
// orchestrator to the concrete task repository. The route row is authoritative
|
|
// when a generation claim committed before task-session attribution could be
|
|
// written; launch reconciliation repairs the session from that row.
|
|
type dynamicRouteStateLoader interface {
|
|
LoadRouteState(context.Context, string) (*dynamicruntime.RouteState, error)
|
|
}
|
|
|
|
// ErrSessionNotPromptable is returned when a session cannot accept a prompt
|
|
// because of its lifecycle state (STARTING, CREATED, FAILED, CANCELLED).
|
|
// Distinct from ErrAgentPromptInProgress, which is RUNNING-only — confusing
|
|
// the two misleads the UI and any caller doing errors.Is checks.
|
|
var ErrSessionNotPromptable = errors.New("session not promptable")
|
|
|
|
// errQueuedDispatchSuperseded is returned by promptTask when a queued
|
|
// message's dispatch loses its claim (see isCurrentQueuedDispatch) to a
|
|
// newer dispatch for the same session — e.g. a second parent interrupt
|
|
// cancelling and re-taking while the first dispatch's own goroutine was
|
|
// still settling. Not surfaced to callers of the public PromptTask; only
|
|
// executeQueuedMessage's internal call passes a claim token that can
|
|
// trigger this.
|
|
var errQueuedDispatchSuperseded = errors.New("queued dispatch superseded by a newer one for this session")
|
|
|
|
var (
|
|
// Backend restart recovery can restore the session state before the ACP
|
|
// stream is promptable again. Keep this above CI's slow-start tail so a
|
|
// valid resume waits instead of surfacing "Failed to send message to agent".
|
|
agentPromptReadyTimeout = 30 * time.Second
|
|
agentPromptReadyInterval = 100 * time.Millisecond
|
|
)
|
|
|
|
const promptFailureCleanupTimeout = 5 * time.Second
|
|
|
|
const promptReadinessRecoveryStopReason = "prompt readiness recovery"
|
|
|
|
type agentPromptStreamRecoverer interface {
|
|
RecoverAgentPromptStream(ctx context.Context, sessionID string) error
|
|
}
|
|
|
|
type resumeAttemptBinder interface {
|
|
BindResumeAttempt(ctx context.Context, sessionID, attemptID string) error
|
|
}
|
|
|
|
func isAgentPromptInProgressError(err error) bool {
|
|
return err != nil && errors.Is(err, ErrAgentPromptInProgress)
|
|
}
|
|
|
|
// isSessionBusyError reports whether the session is in a state where a queued
|
|
// or auto-started prompt should be retried later rather than dropped. Covers
|
|
// both "the agent is mid-turn" (ErrAgentPromptInProgress, RUNNING) and "the
|
|
// session isn't yet ready to accept input" (ErrSessionNotPromptable —
|
|
// STARTING, CREATED, FAILED, CANCELLED). The pre-PR code path collapsed both
|
|
// into ErrAgentPromptInProgress; this helper preserves the requeue behaviour
|
|
// after the error split so queued messages targeting a session that is
|
|
// briefly STARTING/CREATED don't get silently dropped (see TODO about the
|
|
// missing dead-letter queue in executeQueuedMessage).
|
|
func isSessionBusyError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
return errors.Is(err, ErrAgentPromptInProgress) || errors.Is(err, ErrSessionNotPromptable)
|
|
}
|
|
|
|
func isSessionResetInProgressError(err error) bool {
|
|
return err != nil && errors.Is(err, ErrSessionResetInProgress)
|
|
}
|
|
|
|
// isTransientPromptError reports whether a prompt error is worth retrying via
|
|
// the queue. ErrExecutionNotFound is intentionally NOT included here:
|
|
// callers that can recover (autoStartStepPrompt → fallbackFreshLaunchOnMissingExecution)
|
|
// detect it explicitly via errors.Is and route differently; callers that
|
|
// can't (executeQueuedMessage) should not infinite-requeue on it. Treating
|
|
// "execution not found" as transient blanket-applies a retry that loops
|
|
// forever when the execution is genuinely gone.
|
|
func isTransientPromptError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
var pendingCompletionTimeout *lifecycle.PendingDispatchedPromptTimeoutError
|
|
if errors.As(err, &pendingCompletionTimeout) {
|
|
return true
|
|
}
|
|
msg := strings.ToLower(err.Error())
|
|
return strings.Contains(msg, "agent stream disconnected") ||
|
|
strings.Contains(msg, "use of closed network connection")
|
|
}
|
|
|
|
// isManualRecoveryPromptError identifies a provider error that should retain
|
|
// the queued prompt until the user explicitly resumes the session. It is
|
|
// intentionally separate from isTransientPromptError: exhausted usage does
|
|
// not become available on a short automatic retry, and repeatedly attempting
|
|
// it would consume the queue's retry budget without making progress.
|
|
func isManualRecoveryPromptError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
message := strings.ToLower(err.Error())
|
|
return strings.Contains(message, "usagelimitexceeded") ||
|
|
strings.Contains(message, "you've hit your usage limit")
|
|
}
|
|
|
|
func isAgentAlreadyRunningError(err error) bool {
|
|
return err != nil && errors.Is(err, lifecycle.ErrAgentAlreadyRunning)
|
|
}
|
|
|
|
// missingSessionWorktreePaths returns the session's recorded worktree
|
|
// directories that are no longer on disk.
|
|
func missingSessionWorktreePaths(session *models.TaskSession) []string {
|
|
var missing []string
|
|
for _, wt := range session.Worktrees {
|
|
if wt == nil || wt.WorktreePath == "" {
|
|
continue
|
|
}
|
|
if _, err := os.Stat(wt.WorktreePath); err != nil {
|
|
missing = append(missing, wt.WorktreePath)
|
|
}
|
|
}
|
|
return missing
|
|
}
|
|
|
|
// noteMissingWorktreesBeforeResume records that a resume is about to go
|
|
// through the worktree manager's recreate path.
|
|
//
|
|
// A missing directory is NOT a resume blocker. Archive cleanup deletes the
|
|
// worktree directory while keeping the environment repository row and the git
|
|
// branch (DestroyWorktree passes removeBranch=false), so *every* resume after
|
|
// an unarchive observes exactly this shape. worktree.Manager.Create reuses the
|
|
// stored record, restores the branch — locally, or by fetching it back from
|
|
// origin — and rebuilds the directory at the same path, which is what the
|
|
// recreate path was written for. Rejecting the resume here instead made
|
|
// unarchive a dead end: the session could never be resumed and the workspace
|
|
// was never restored, even though the work itself was still intact.
|
|
func (s *Service) noteMissingWorktreesBeforeResume(sessionID string, session *models.TaskSession) {
|
|
missing := missingSessionWorktreePaths(session)
|
|
if len(missing) == 0 {
|
|
return
|
|
}
|
|
s.logger.Info("worktree directory missing on resume; it will be recreated from the stored branch",
|
|
zap.String("session_id", sessionID),
|
|
zap.Strings("paths", missing))
|
|
}
|
|
|
|
// EnqueueTask manually adds a task to the queue
|
|
func (s *Service) EnqueueTask(ctx context.Context, task *v1.Task) error {
|
|
s.logger.Debug("manually enqueueing task",
|
|
zap.String("task_id", task.ID),
|
|
zap.String("title", task.Title))
|
|
return s.scheduler.EnqueueTask(task)
|
|
}
|
|
|
|
// PrepareTaskSession creates a session entry without launching the agent.
|
|
// This allows the WS handler to return the session ID immediately while workspace setup
|
|
// continues in the background. Use StartCreatedSession to continue with agent launch.
|
|
// When launchWorkspace is true, workspace infrastructure (agentctl) is launched asynchronously;
|
|
// the frontend receives preparation progress via executor.prepare.progress WS events.
|
|
func (s *Service) PrepareTaskSession(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, workflowStepID string, launchWorkspace bool) (string, error) {
|
|
ctx = executor.WithTaskRunnerProfileExplicit(ctx, strings.TrimSpace(executorProfileID) != "")
|
|
s.logger.Debug("preparing task session",
|
|
zap.String("task_id", taskID),
|
|
zap.String("agent_profile_id", agentProfileID),
|
|
zap.String("executor_id", executorID),
|
|
zap.String("executor_profile_id", executorProfileID),
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.Bool("launch_workspace", launchWorkspace))
|
|
|
|
// Fetch the task to get workspace info
|
|
task, err := s.scheduler.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
s.logger.Error("failed to fetch task for session preparation",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
return "", err
|
|
}
|
|
// Resolve agent/executor profile from task metadata if not explicitly provided
|
|
if agentProfileID == "" {
|
|
if v, ok := task.Metadata[models.MetaKeyAgentProfileID].(string); ok && v != "" {
|
|
agentProfileID = v
|
|
}
|
|
}
|
|
if executorID == "" {
|
|
if v, ok := task.Metadata[models.MetaKeyExecutorID].(string); ok && v != "" {
|
|
executorID = v
|
|
}
|
|
}
|
|
if executorProfileID == "" {
|
|
if v, ok := task.Metadata[models.MetaKeyExecutorProfileID].(string); ok && v != "" {
|
|
executorProfileID = v
|
|
}
|
|
}
|
|
|
|
// Inherit agent/executor profile from parent task's primary session when not
|
|
// explicitly provided. This covers subtasks created with start_agent=false that
|
|
// are later opened manually from the UI. We check each field independently so
|
|
// that a caller providing only some fields still gets the rest filled in.
|
|
if (agentProfileID == "" || executorProfileID == "" || executorID == "") && task.ParentID != "" {
|
|
agentProfileID, executorProfileID, executorID = s.inheritFromParentSession(
|
|
ctx, task.ParentID, agentProfileID, executorProfileID, executorID,
|
|
)
|
|
|
|
// Fall back to workspace defaults for agent profile (subtasks only —
|
|
// regular tasks resolve defaults downstream in the executor layer).
|
|
if agentProfileID == "" {
|
|
workspace, err := s.repo.GetWorkspace(ctx, task.WorkspaceID)
|
|
if err == nil && workspace != nil && workspace.DefaultAgentProfileID != nil && *workspace.DefaultAgentProfileID != "" {
|
|
agentProfileID = *workspace.DefaultAgentProfileID
|
|
}
|
|
}
|
|
// An inherit_parent child must keep the parent's effective executor. An
|
|
// empty executor here is meaningful: it selects the local executor when
|
|
// the parent also used the workspace default. For other subtasks, keep
|
|
// the historical worktree default so they receive an isolated checkout.
|
|
if executorID == "" && executorProfileID == "" && !isInheritParentWorkspace(task) {
|
|
executorID = models.ExecutorIDWorktree
|
|
}
|
|
}
|
|
|
|
// Fall back to the task's current workflow step when the caller didn't provide one.
|
|
// This ensures sessions created via the kanban card (which doesn't send workflow_step_id)
|
|
// inherit the task's step and participate in workflow events.
|
|
if workflowStepID == "" {
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to fetch task for workflow step fallback",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
} else if dbTask.WorkflowStepID != "" {
|
|
workflowStepID = dbTask.WorkflowStepID
|
|
}
|
|
}
|
|
if s.profileExecutionResolver != nil {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, agentProfileID); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
// Create session entry in database. Office tasks route through
|
|
// EnsureSessionForAgent so runs + advanced-mode reuse one row.
|
|
// prepareSessionForStart also propagates any inherited workspace
|
|
// environment (inherit_parent / shared_group) onto the new session.
|
|
// A manual prepare call has no scheduler-owned Office run identity. Keep the
|
|
// participant slot empty so createStartSession uses the task assignee, while
|
|
// agentProfileID remains the concrete execution profile.
|
|
sessionID, sessionCreated, err := s.prepareSessionForStart(ctx, task, agentProfileID, "", executorID, executorProfileID, workflowStepID)
|
|
if err != nil {
|
|
s.logger.Error("failed to prepare session",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
return "", err
|
|
}
|
|
|
|
// Notify the frontend that a new CREATED session exists. The start path
|
|
// transitions through updateTaskSessionState which broadcasts; the prepare
|
|
// path writes the row directly, so without this the per-task session list
|
|
// stays empty until a manual reload.
|
|
if sessionCreated {
|
|
s.publishSessionCreatedEvent(ctx, taskID, sessionID, workflowStepID)
|
|
}
|
|
|
|
if launchWorkspace {
|
|
// Launch workspace infrastructure (agentctl) in the background so the WS response
|
|
// returns the session ID immediately. The frontend navigates to the session page
|
|
// and shows preparation progress via executor.prepare.progress WS events.
|
|
go func() {
|
|
bgCtx := context.Background()
|
|
launchProfileID := agentProfileID
|
|
// This workspace-only launch (StartAgent unset) never starts the
|
|
// agent, so it never reaches "active" itself — the eventual
|
|
// StartCreatedSession call does that. But if a route was claimed
|
|
// below and this launch then fails, nothing else will ever move
|
|
// it off "starting" or "retrying"; the deferred guard covers both
|
|
// failure points uniformly. It only fires while the route is
|
|
// still "starting" or "retrying" (see Engine.MarkActionRequired),
|
|
// so it is a no-op if a concurrent real launch already marked it
|
|
// active.
|
|
var claimedRouteGeneration int64
|
|
launchOwned := false
|
|
defer func() {
|
|
if claimedRouteGeneration > 0 && !launchOwned {
|
|
s.markDynamicRouteActionRequired(bgCtx, sessionID, claimedRouteGeneration, "prepared_session_launch_failed")
|
|
}
|
|
}()
|
|
if s.profileExecutionResolver != nil {
|
|
launchSession, loadErr := s.repo.GetTaskSession(bgCtx, sessionID)
|
|
if loadErr != nil {
|
|
s.logger.Warn("failed to reload prepared session for dynamic route resolution",
|
|
zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(loadErr))
|
|
_ = s.handleSessionLaunchFailure(bgCtx, taskID, sessionID, loadErr)
|
|
return
|
|
}
|
|
resolved, routeClaimed, resolveErr := s.resolveExecutionForLaunchSession(bgCtx, launchSession)
|
|
if resolveErr != nil {
|
|
s.recordDynamicRouteResolutionFailure(bgCtx, launchSession, resolveErr)
|
|
s.logger.Warn("failed to resolve dynamic route for prepared session",
|
|
zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(resolveErr))
|
|
_ = s.handleSessionLaunchFailure(bgCtx, taskID, sessionID, resolveErr, launchSession)
|
|
return
|
|
}
|
|
if resolved.ExecutionProfileID != "" {
|
|
launchProfileID = resolved.ExecutionProfileID
|
|
if routeClaimed {
|
|
claimedRouteGeneration = resolved.Generation
|
|
applyResolvedExecution(launchSession, resolved)
|
|
if updateErr := s.repo.UpdateTaskSession(bgCtx, launchSession); updateErr != nil {
|
|
s.logger.Warn("failed to persist dynamic route for prepared session",
|
|
zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(updateErr))
|
|
_ = s.handleSessionLaunchFailure(bgCtx, taskID, sessionID, updateErr, launchSession)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
prepExec, launchErr := s.executor.LaunchPreparedSession(bgCtx, task, sessionID, executor.LaunchOptions{AgentProfileID: launchProfileID, ExecutorID: executorID, WorkflowStepID: workflowStepID})
|
|
if launchErr != nil {
|
|
// LaunchAgent failures persist FAILED in the executor. Earlier
|
|
// workspace failures return here and are recorded through the same
|
|
// session-level recovery claim.
|
|
launchErr = s.handleSessionLaunchFailure(bgCtx, taskID, sessionID, launchErr)
|
|
s.logger.Warn("failed to launch workspace for prepared session (file browsing may be unavailable)",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(launchErr))
|
|
return
|
|
}
|
|
launchOwned = true
|
|
if prepExec != nil {
|
|
s.ensureSessionPRWatch(bgCtx, taskID, prepExec.SessionID, prepExec.WorktreeBranch)
|
|
}
|
|
}()
|
|
}
|
|
|
|
s.logger.Info("task session prepared",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
|
|
return sessionID, nil
|
|
}
|
|
|
|
type initialPromptPreviewContextKey struct{}
|
|
|
|
func withInitialPromptPreview(ctx context.Context, preview *models.InitialPromptPreview) context.Context {
|
|
if preview == nil {
|
|
return ctx
|
|
}
|
|
return context.WithValue(ctx, initialPromptPreviewContextKey{}, preview)
|
|
}
|
|
|
|
func (s *Service) persistInitialPromptPreviewFromContext(ctx context.Context, taskID, sessionID string) error {
|
|
preview, ok := ctx.Value(initialPromptPreviewContextKey{}).(*models.InitialPromptPreview)
|
|
if !ok || preview == nil {
|
|
return nil
|
|
}
|
|
if err := s.repo.SetSessionMetadataKey(ctx, sessionID, models.SessionMetaKeyInitialPromptPreview, preview); err != nil {
|
|
return s.handleSessionLaunchFailure(ctx, taskID, sessionID, fmt.Errorf("persist initial prompt preview: %w", err))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isInheritParentWorkspace(task *v1.Task) bool {
|
|
if task == nil {
|
|
return false
|
|
}
|
|
mode, _ := workspacePolicyMode(task.Metadata)
|
|
return mode == "inherit_parent"
|
|
}
|
|
|
|
// StartCreatedSession starts agent execution for a task using a session that is in CREATED state.
|
|
// This is used when a session was prepared (via PrepareSession) but the agent was not launched,
|
|
// and the user now wants to start the agent with a prompt (e.g., from the plan panel or chat).
|
|
// When skipMessageRecord is true, only the session state is updated (the caller already stored the user message).
|
|
// When planMode is true, plan mode instructions are injected into the prompt and session metadata is set.
|
|
// autoStart marks the launch as having been triggered by an automated path
|
|
// (only consumed when skipMessageRecord is false — callers that store their
|
|
// own message control its metadata directly).
|
|
// references contains validated entity references whose exact server-generated
|
|
// context block may survive first-turn canonicalization.
|
|
//
|
|
//nolint:cyclop,funlen,gocognit // Existing complexity inherited from session-lifecycle handling.
|
|
func (s *Service) StartCreatedSession(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
skipMessageRecord, planMode, autoStart bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startCreatedSession(
|
|
ctx, taskID, sessionID, agentProfileID, prompt,
|
|
skipMessageRecord, planMode, autoStart, attachments, references, "", startCreatedSessionOptions{},
|
|
)
|
|
}
|
|
|
|
// StartCreatedSessionWithPromptContext starts a prepared session while
|
|
// preserving the exact server-generated saved-prompt context prepared by the
|
|
// message handler. Direct first turns need this narrow seam because the
|
|
// session-start path canonicalizes the prompt again before dispatch.
|
|
func (s *Service) StartCreatedSessionWithPromptContext(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
skipMessageRecord, planMode, autoStart bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
promptReferenceContext string,
|
|
promptReferencesPrepared bool,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startCreatedSession(
|
|
ctx, taskID, sessionID, agentProfileID, prompt,
|
|
skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext,
|
|
startCreatedSessionOptions{promptReferencesPrepared: promptReferencesPrepared},
|
|
)
|
|
}
|
|
|
|
// StartCreatedSessionWithPromptContextAndCanvasGuidance starts a prepared
|
|
// direct-message session with the exact server-side canvas capability decision
|
|
// made while the user message was admitted. This prevents the launch-time
|
|
// canonicalizer from producing a prompt that differs from the persisted row.
|
|
func (s *Service) StartCreatedSessionWithPromptContextAndCanvasGuidance(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
skipMessageRecord, planMode, autoStart bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
promptReferenceContext string,
|
|
promptReferencesPrepared bool,
|
|
canvasGuidanceResolved, includeCanvasGuidance bool,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startCreatedSession(
|
|
ctx, taskID, sessionID, agentProfileID, prompt,
|
|
skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext,
|
|
startCreatedSessionOptions{
|
|
canvasGuidanceResolved: canvasGuidanceResolved,
|
|
includeCanvasGuidance: includeCanvasGuidance,
|
|
promptReferencesPrepared: promptReferencesPrepared,
|
|
},
|
|
)
|
|
}
|
|
|
|
// StartCreatedSessionWithPromptContextAndCanvasGuidancePreservingDirectPrompt
|
|
// starts a prepared direct-message session with a prompt whose visible task
|
|
// brief and instruction were already admitted and persisted together.
|
|
func (s *Service) StartCreatedSessionWithPromptContextAndCanvasGuidancePreservingDirectPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
skipMessageRecord, planMode, autoStart bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
promptReferenceContext string,
|
|
promptReferencesPrepared bool,
|
|
canvasGuidanceResolved, includeCanvasGuidance bool,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startCreatedSession(
|
|
ctx, taskID, sessionID, agentProfileID, prompt,
|
|
skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext,
|
|
startCreatedSessionOptions{
|
|
canvasGuidanceResolved: canvasGuidanceResolved,
|
|
includeCanvasGuidance: includeCanvasGuidance,
|
|
promptReferencesPrepared: promptReferencesPrepared,
|
|
preserveDirectPrompt: true,
|
|
},
|
|
)
|
|
}
|
|
|
|
// startCreatedSessionWithComposedPrompt launches a prepared session from an
|
|
// auto-start path whose prompt was already composed and recorded by the
|
|
// orchestrator. The ordinary public entry point intentionally applies the
|
|
// workflow/config/plan transforms itself; reapplying them here would duplicate
|
|
// wrappers and would also fall back to the task description for an empty,
|
|
// already-prompted workflow step. Keep this seam private to the auto-start
|
|
// admission path so user-initiated starts retain their existing contract.
|
|
func (s *Service) startCreatedSessionWithComposedPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
retryPrompt string,
|
|
promptReferenceContext string,
|
|
skipMessageRecord, planMode, autoStart, initialCreatePrompt bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startCreatedSession(
|
|
ctx, taskID, sessionID, agentProfileID, prompt,
|
|
skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext, startCreatedSessionOptions{
|
|
initialCreatePrompt: initialCreatePrompt,
|
|
skipTaskDescriptionFallback: true,
|
|
promptAlreadyComposed: true,
|
|
retryPrompt: retryPrompt,
|
|
},
|
|
)
|
|
}
|
|
|
|
type startCreatedSessionOptions struct {
|
|
initialCreatePrompt bool
|
|
skipTaskDescriptionFallback bool
|
|
promptAlreadyComposed bool
|
|
retryPrompt string
|
|
// canvasGuidanceResolved carries the server-side capability projection
|
|
// from message admission. When false, this launch resolves the capability
|
|
// locally because no earlier producer has made a trusted decision.
|
|
canvasGuidanceResolved bool
|
|
includeCanvasGuidance bool
|
|
// promptReferencesPrepared records that the caller accepted the exact
|
|
// server-owned expansion snapshot, including an empty snapshot.
|
|
promptReferencesPrepared bool
|
|
preserveDirectPrompt bool
|
|
// ceilingEntryBinding is carried by a replay and checked at both the
|
|
// admission boundary and immediately before runtime dispatch.
|
|
ceilingEntryBinding *models.CeilingWorkflowEntryBinding
|
|
}
|
|
|
|
//nolint:cyclop,funlen,gocognit // Existing complexity inherited from session-lifecycle handling.
|
|
func (s *Service) startCreatedSession(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentProfileID, prompt string,
|
|
skipMessageRecord, planMode, autoStart bool,
|
|
attachments []v1.MessageAttachment,
|
|
references []v1.EntityReference,
|
|
promptReferenceContext string,
|
|
options startCreatedSessionOptions,
|
|
) (*executor.TaskExecution, error) {
|
|
releaseLifecycleLock := s.acquireSessionLifecycleLock(sessionID)
|
|
defer releaseLifecycleLock()
|
|
if options.ceilingEntryBinding == nil {
|
|
options.ceilingEntryBinding = ceilingEntryBindingFromContext(ctx)
|
|
}
|
|
if options.ceilingEntryBinding != nil {
|
|
ctx = withCeilingEntryBinding(ctx, options.ceilingEntryBinding)
|
|
}
|
|
|
|
// One GetWorkflowMeta read shared by profile resolution and prompt build.
|
|
ctx = withWorkflowMetaCache(ctx)
|
|
|
|
s.logger.Debug("starting created session",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("agent_profile_id", agentProfileID))
|
|
|
|
// Load and verify session
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if session.TaskID != taskID {
|
|
return nil, fmt.Errorf("session does not belong to task")
|
|
}
|
|
// Accept CREATED (normal) or WAITING_FOR_INPUT (after on_turn_start step transition).
|
|
// When the user sends the first message to a prepared session, on_turn_start may fire
|
|
// and move the step, which sets the session to WAITING_FOR_INPUT before we get here.
|
|
if session.State != models.TaskSessionStateCreated && session.State != models.TaskSessionStateWaitingForInput {
|
|
return nil, fmt.Errorf("session is not in CREATED or WAITING_FOR_INPUT state (current: %s)", session.State)
|
|
}
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, options.ceilingEntryBinding); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Office-owned sessions must be started by the scheduler. Reject before
|
|
// resolving profiles or persisting any caller-derived session metadata.
|
|
isOfficeTask, err := s.lookupOfficeTask(ctx, taskID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to determine office task status: %w", err)
|
|
}
|
|
if isOfficeTask {
|
|
return nil, errOfficeTaskStartRequiresScheduler
|
|
}
|
|
|
|
startPayload := seam2StartCreatedPayload(sessionID, agentProfileID, prompt, skipMessageRecord, planMode, autoStart, attachments, references, promptReferenceContext, options)
|
|
startPayload = s.enrichCeilingLaunchPayload(ctx, taskID, sessionID, startPayload)
|
|
seam2Res, deferred, err := s.admitOrDeferSeam2(ctx, taskID, sessionID, originFromAutoStart(autoStart), startPayload)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if deferred {
|
|
return nil, nil
|
|
}
|
|
defer seam2Res.releaseIfNotConsumed()
|
|
|
|
// Reserve any pending "start it later" intent for the rest of this start.
|
|
// Taken here rather than just before the launch: everything below persists
|
|
// session state this start owns, and a gate that claims the intent during
|
|
// that work launches its own session alongside it. Deferred release covers
|
|
// every failure path; see deferredLaunchClaim.
|
|
launchClaim := s.claimDeferredLaunchForStart(ctx, taskID)
|
|
defer launchClaim.releaseIfHeld(ctx)
|
|
|
|
// Use agent profile from request, fall back to session's stored value.
|
|
effectiveProfileID := agentProfileID
|
|
if effectiveProfileID == "" {
|
|
effectiveProfileID = session.AgentProfileID
|
|
}
|
|
|
|
// Resolve the workflow step override / workflow default before the
|
|
// required-profile guard, so a session without its own agent_profile_id
|
|
// inherits the workflow's default agent. resolveEffectiveAgentProfile keeps
|
|
// the caller profile only when neither a step override nor a workflow
|
|
// default applies; either of those overrides a non-empty caller.
|
|
effectiveProfileID, err = s.resolveEffectiveAgentProfile(ctx, taskID, "", effectiveProfileID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if effectiveProfileID == "" {
|
|
return nil, fmt.Errorf("agent_profile_id is required")
|
|
}
|
|
if s.profileExecutionResolver != nil {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, effectiveProfileID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// If the workflow step overrode the profile, update the session record in DB
|
|
// so the frontend tab displays the correct agent (it reads session.agent_profile_id).
|
|
if effectiveProfileID != session.AgentProfileID {
|
|
observedState := session.State
|
|
s.logger.Info("updating session agent profile for workflow step override",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("old_profile", session.AgentProfileID),
|
|
zap.String("new_profile", effectiveProfileID))
|
|
session.AgentProfileID = effectiveProfileID
|
|
// Re-resolve the agent profile snapshot so the tab shows the correct agent logo/name.
|
|
// Set a minimal snapshot first so stale data is never persisted if resolution fails.
|
|
session.AgentProfileSnapshot = map[string]interface{}{"id": effectiveProfileID}
|
|
if profileInfo, err := s.agentManager.ResolveAgentProfile(ctx, effectiveProfileID); err != nil {
|
|
s.logger.Warn("failed to resolve agent profile snapshot for workflow step override",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("profile_id", effectiveProfileID),
|
|
zap.Error(err))
|
|
} else if profileInfo != nil {
|
|
session.AgentProfileSnapshot = map[string]interface{}{
|
|
"id": profileInfo.ProfileID,
|
|
"name": profileInfo.ProfileName,
|
|
"agent_id": profileInfo.AgentID,
|
|
"agent_name": profileInfo.AgentName,
|
|
"model": profileInfo.Model,
|
|
"mode": profileInfo.Mode,
|
|
"config_options": maps.Clone(profileInfo.ConfigOptions),
|
|
}
|
|
}
|
|
if err := s.persistFullTaskSessionIfCurrent(ctx, session, observedState); err != nil {
|
|
return nil, fmt.Errorf("persist workflow session profile: %w", err)
|
|
}
|
|
// Tag as workflow-spawned provenance only after the guarded profile
|
|
// write succeeds; a concurrent stop owns a rejected session.
|
|
s.tagSessionAsWorkflowSwitchedForSnapshot(ctx, session)
|
|
s.promoteSessionIfTaskHasNoPrimary(ctx, taskID, session)
|
|
}
|
|
|
|
// Transition task state: CREATED → SCHEDULING → (IN_PROGRESS via executor).
|
|
if err := s.scheduleTaskForSession(ctx, taskID, sessionID); err != nil {
|
|
s.logger.Warn("failed to update task state to SCHEDULING",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
task, err := s.scheduler.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
effectivePrompt := prompt
|
|
if effectivePrompt == "" && !options.skipTaskDescriptionFallback {
|
|
effectivePrompt = task.Description
|
|
}
|
|
|
|
// NOTE: on_turn_start is intentionally NOT processed here.
|
|
// - User-initiated path: dispatchPromptAsync (message_handlers.go) already
|
|
// calls ProcessOnTurnStart before invoking StartCreatedSession via
|
|
// forwardMessageAsPrompt, so on_turn_start has already fired.
|
|
// - Workflow auto-start path: autoStartStepPrompt calls us because the
|
|
// workflow just transitioned us into this step (via on_turn_complete or
|
|
// on_enter). Firing on_turn_start again here cascades the workflow back
|
|
// out before the step's auto-start prompt can be delivered to its agent.
|
|
//
|
|
// However, for the user-initiated path, on_turn_start may have switched the
|
|
// session profile already, in which case the session ID we were called with
|
|
// is now COMPLETED and we need to redirect to the new active session.
|
|
session, err = s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to reload session: %w", err)
|
|
}
|
|
if session.State == models.TaskSessionStateCompleted {
|
|
activeSession, activeErr := s.repo.GetActiveTaskSessionByTaskID(ctx, taskID)
|
|
if activeErr != nil || activeSession == nil {
|
|
return nil, fmt.Errorf("session was switched but no active session found: %w", activeErr)
|
|
}
|
|
session = activeSession
|
|
seam2Res.rekeyToSession(ctx, activeSession.ID)
|
|
sessionID = activeSession.ID
|
|
effectiveProfileID = activeSession.AgentProfileID
|
|
}
|
|
s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam2Res.manualOverride, seam2Res.population, seam2Res.populationKnown, seam2Res.ceiling)
|
|
|
|
if effectiveProfileID, err = s.resolveDynamicLaunchExecution(ctx, session, effectiveProfileID, true); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Apply workflow step prompt wrapping and plan mode injection.
|
|
// Called unconditionally so workflow-step prompt composition (prefix/suffix)
|
|
// applies even when plan mode is not requested.
|
|
// Re-read the task after on_turn_start may have changed the workflow step.
|
|
// Ephemeral tasks skip workflow step processing since they have no workflow.
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to reload task after on_turn_start: %w", err)
|
|
}
|
|
configMode := false
|
|
if cm, ok := session.Metadata["config_mode"].(bool); ok && cm {
|
|
configMode = true
|
|
}
|
|
titleOwner := false
|
|
if !configMode {
|
|
titleOwner, err = s.ClaimTaskTitleSession(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to claim first-turn task title: %w", err)
|
|
}
|
|
}
|
|
planModeActive := planMode
|
|
if !options.promptAlreadyComposed {
|
|
var generatedPromptReferenceContext string
|
|
if options.preserveDirectPrompt {
|
|
effectivePrompt, planModeActive, generatedPromptReferenceContext = s.applyWorkflowAndPlanModeWithPromptContextOptions(
|
|
ctx, effectivePrompt, taskID, sessionID, dbTask.WorkflowStepID,
|
|
planMode, task.IsEphemeral, session.IsPassthrough, false, promptReferenceContext,
|
|
options.promptReferencesPrepared, true,
|
|
)
|
|
} else {
|
|
effectivePrompt, planModeActive, generatedPromptReferenceContext = s.applyWorkflowAndPlanModeWithPromptContextOptions(
|
|
ctx, effectivePrompt, taskID, sessionID, dbTask.WorkflowStepID,
|
|
planMode, task.IsEphemeral, session.IsPassthrough, false, promptReferenceContext,
|
|
options.promptReferencesPrepared, false,
|
|
)
|
|
}
|
|
if generatedPromptReferenceContext != "" {
|
|
// The workflow helper preserves a direct message's acceptance-time
|
|
// context. Auto-started workflow prompts have no prior context, so their
|
|
// launch-time expansion becomes the trusted value here.
|
|
promptReferenceContext = generatedPromptReferenceContext
|
|
}
|
|
}
|
|
|
|
// Inject config context for config-mode sessions (dedicated settings chat)
|
|
if configMode {
|
|
if !options.promptAlreadyComposed {
|
|
effectivePrompt = sysprompt.InjectConfigContext(sessionID, effectivePrompt)
|
|
}
|
|
}
|
|
|
|
// Wrap the first prompt with the Kandev MCP system block. See the
|
|
// matching block in startTask for the rationale (DB stores wrapped form;
|
|
// Message.ToAPI strips for display). The injectors canonicalize any upstream
|
|
// wrap from current server state before launch.
|
|
// Passthrough profiles skip the wrap: the prompt is typed straight into the
|
|
// agent CLI's TTY and the user sees it verbatim — they don't want a wall of
|
|
// MCP-tool boilerplate prepended to "hello".
|
|
includeCanvasGuidance := false
|
|
if (effectivePrompt != "" || len(attachments) > 0) && !isOfficeTask && !session.IsPassthrough && !configMode {
|
|
if options.canvasGuidanceResolved {
|
|
includeCanvasGuidance = options.includeCanvasGuidance
|
|
} else {
|
|
includeCanvasGuidance, err = s.taskSessionCanvasGuidanceEnabled(ctx, taskID, session, true)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to resolve canvas prompt capability: %w", err)
|
|
}
|
|
}
|
|
}
|
|
if effectivePrompt != "" || len(attachments) > 0 {
|
|
effectivePrompt = s.wrapCreatedSessionPrompt(
|
|
ctx, effectivePrompt, taskID, sessionID, session, dbTask,
|
|
isOfficeTask, configMode, titleOwner, includeCanvasGuidance, references, promptReferenceContext,
|
|
)
|
|
}
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, options.ceilingEntryBinding); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
executorID := session.ExecutorID
|
|
|
|
// Cache the raw prompt so a transient-provider-error (529) retry can
|
|
// re-drive this first turn — initial launches bypass PromptTask.
|
|
retryPrompt := prompt
|
|
if options.promptAlreadyComposed {
|
|
retryPrompt = options.retryPrompt
|
|
}
|
|
s.rememberTurnPrompt(sessionID, retryPrompt, "", planMode, attachments)
|
|
|
|
mcpMode := ""
|
|
if isOfficeTask {
|
|
mcpMode = executor.McpModeOffice
|
|
}
|
|
initialTurnID, initialTurnCreated := s.startTurnForSessionWithOwnership(ctx, sessionID)
|
|
ctx = bindWorkflowStartPromptAttemptTurn(ctx, initialTurnID)
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, options.ceilingEntryBinding); err != nil {
|
|
if initialTurnCreated {
|
|
s.completeTurnIfCurrent(ctx, sessionID, initialTurnID)
|
|
}
|
|
return nil, err
|
|
}
|
|
if options.initialCreatePrompt && session.IsPassthrough {
|
|
s.armInitialCreatePromptPassthroughForLaunch(ctx, session, initialTurnID)
|
|
defer func() {
|
|
if err != nil {
|
|
s.retireInitialCreatePromptPassthroughForQueue(
|
|
session.ID,
|
|
session.QueueIncarnationID,
|
|
initialTurnID,
|
|
s.promptGenerationForSession(ctx, session.ID),
|
|
)
|
|
}
|
|
}()
|
|
}
|
|
launchOptions := executor.LaunchOptions{
|
|
AgentProfileID: effectiveProfileID,
|
|
ExecutorID: executorID,
|
|
Prompt: effectivePrompt,
|
|
StartAgent: true,
|
|
McpMode: mcpMode,
|
|
Attachments: attachments,
|
|
TurnID: initialTurnID,
|
|
}
|
|
if options.initialCreatePrompt && session.IsPassthrough {
|
|
launchOptions.OnExecutionAdmitted = func(executionID string) {
|
|
s.bindInitialCreatePromptPassthroughExecution(ctx, sessionID, initialTurnID, executionID)
|
|
}
|
|
}
|
|
execution, err := s.launchPreparedSessionWithDynamicFallback(ctx, task, sessionID, launchOptions)
|
|
if err != nil {
|
|
// The executor persists LaunchAgent failures. Cover earlier prepared-session
|
|
// failures here; the session-level claim makes either completion order safe.
|
|
if initialTurnCreated {
|
|
s.completeTurnIfCurrent(ctx, sessionID, initialTurnID)
|
|
}
|
|
return nil, s.handleSessionLaunchFailure(ctx, taskID, sessionID, err)
|
|
}
|
|
|
|
// Record the initial user message and set plan mode metadata after launch.
|
|
// Note: we do NOT set session state here — the executor sets it to STARTING,
|
|
// and event handlers (handleAgentReady) transition it to WAITING_FOR_INPUT.
|
|
s.postLaunchCreated(ctx, taskID, sessionID, effectivePrompt, skipMessageRecord, planModeActive, autoStart, attachments)
|
|
|
|
// The agent is running, so the reservation becomes a consumption.
|
|
launchClaim.consume(ctx)
|
|
seam2Res.consume()
|
|
|
|
// Ensure a PR watch exists so the poller can detect PRs created by the agent.
|
|
// PrepareTaskSession may have already created one, but if that goroutine failed
|
|
// or hadn't completed, this guarantees coverage.
|
|
go s.ensureSessionPRWatch(context.Background(), taskID, execution.SessionID, execution.WorktreeBranch)
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
// wrapCreatedSessionPrompt adds the first-turn context for a prepared session.
|
|
// Passthrough sessions intentionally receive only the short pending-title
|
|
// instruction; structured sessions receive the normal task or Office block.
|
|
func (s *Service) wrapCreatedSessionPrompt(
|
|
ctx context.Context,
|
|
prompt, taskID, sessionID string,
|
|
session *models.TaskSession,
|
|
dbTask *models.Task,
|
|
isOfficeTask, configMode, titleOwner, includeCanvasGuidance bool,
|
|
references []v1.EntityReference,
|
|
promptReferenceContext string,
|
|
) string {
|
|
prompt, pullRequestTargetContext := s.addTaskPullRequestTargetContext(
|
|
ctx, taskID, prompt, session.IsPassthrough,
|
|
)
|
|
referenceContext := EntityReferenceContext(references)
|
|
switch {
|
|
case session.IsPassthrough:
|
|
if !configMode && titleOwner {
|
|
return sysprompt.PendingTaskTitlePassthroughInstruction() + "\n\n" + prompt
|
|
}
|
|
return prompt
|
|
case isOfficeTask:
|
|
return sysprompt.InjectOfficeContextWithOptions(
|
|
taskID, sessionID, prompt,
|
|
s.WorkflowStepRequiresCompletionSignal(ctx, dbTask.WorkflowStepID),
|
|
referenceContext, promptReferenceContext, pullRequestTargetContext,
|
|
)
|
|
default:
|
|
return sysprompt.InjectKandevContextWithOptions(taskID, sessionID, prompt, sysprompt.KandevContextOptions{
|
|
RequiresCompletionSignal: s.WorkflowStepRequiresCompletionSignal(ctx, dbTask.WorkflowStepID),
|
|
IncludeCoordinatorTaskControls: !configMode,
|
|
IncludeTaskTitleTool: !configMode && titleOwner,
|
|
IncludeCanvasGuidance: includeCanvasGuidance,
|
|
Autopilot: dbTask.Autopilot,
|
|
IncludeUserQuestionTool: !dbTask.Autopilot && !session.IsPassthrough,
|
|
IncludeParentQuestionTool: dbTask.Autopilot && dbTask.ParentID != "",
|
|
}, referenceContext, promptReferenceContext, pullRequestTargetContext)
|
|
}
|
|
}
|
|
|
|
// handleSessionLaunchFailure covers launch and resume failures that have not
|
|
// already won terminal bookkeeping. The state CAS makes it safe as an
|
|
// idempotent fallback when executor bookkeeping did run.
|
|
func (s *Service) handleSessionLaunchFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
launchErr error,
|
|
preloadedSession ...*models.TaskSession,
|
|
) error {
|
|
failureCtx := context.WithoutCancel(ctx)
|
|
safeErr := routingerr.SanitizeError(launchErr)
|
|
// A launch failure cannot produce a retryable prompt lifecycle. Release the
|
|
// replay payload here so an early admission or startup failure does not keep
|
|
// attachment data alive until a later session event.
|
|
s.clearTransientRetryState(sessionID)
|
|
_ = s.recordSessionLaunchFailure(
|
|
failureCtx, taskID, sessionID, safeErr, preloadedSession...,
|
|
)
|
|
return safeErr
|
|
}
|
|
|
|
// recordSessionLaunchFailure transitions a still-active session to FAILED and
|
|
// updates the task only while the same failed session still owns it. Typed
|
|
// launch-error persistence belongs to the executor's session transition.
|
|
// Returns whether the session itself was durably transitioned to FAILED — a
|
|
// caller that is about to force-stop the process on the strength of that
|
|
// transition (stopNeverStartedExecution) must not do so when this reports
|
|
// false, or it kills the process while the session row still claims RUNNING.
|
|
func (s *Service) recordSessionLaunchFailure(ctx context.Context, taskID, sessionID string, launchErr error, preloadedSession ...*models.TaskSession) bool {
|
|
_, changed := s.updateTaskSessionStateWithHook(
|
|
ctx, taskID, sessionID, models.TaskSessionStateFailed, launchErr.Error(), false,
|
|
nil, preloadedSession...,
|
|
)
|
|
if !changed {
|
|
return false
|
|
}
|
|
updated, err := s.updateTaskStateForEarlyLaunchFailure(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to update task state to FAILED after early launch error",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return true
|
|
}
|
|
if !updated {
|
|
return true
|
|
}
|
|
s.processParentChildrenCompletedForTaskState(ctx, taskID, v1.TaskStateFailed)
|
|
return true
|
|
}
|
|
|
|
func (s *Service) updateTaskStateForEarlyLaunchFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
) (bool, error) {
|
|
if updater, ok := s.taskRepo.(primarySessionTaskStateUpdater); ok {
|
|
return updater.UpdateTaskStateIfPrimarySessionState(
|
|
ctx, taskID, sessionID, models.TaskSessionStateFailed, v1.TaskStateFailed,
|
|
)
|
|
}
|
|
// Lightweight test repositories predate the primary-aware extension.
|
|
return s.taskRepo.UpdateTaskStateIfSessionState(
|
|
ctx, taskID, sessionID, models.TaskSessionStateFailed, v1.TaskStateFailed,
|
|
)
|
|
}
|
|
|
|
func (s *Service) reconcileTaskStateForEarlyLaunchFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
state v1.TaskState,
|
|
) error {
|
|
if state != v1.TaskStateFailed {
|
|
return fmt.Errorf("unsupported early launch task state %q", state)
|
|
}
|
|
updated, err := s.updateTaskStateForEarlyLaunchFailure(ctx, taskID, sessionID)
|
|
if err != nil || !updated {
|
|
return err
|
|
}
|
|
s.processParentChildrenCompletedForTaskState(ctx, taskID, state)
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) scheduleTaskForSession(ctx context.Context, taskID, sessionID string) error {
|
|
s.taskRuntimeStateMu.Lock()
|
|
defer s.taskRuntimeStateMu.Unlock()
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if session == nil {
|
|
return fmt.Errorf("%w: agent session not found: %s", models.ErrTaskSessionNotFound, sessionID)
|
|
}
|
|
if session.State != models.TaskSessionStateCreated &&
|
|
session.State != models.TaskSessionStateWaitingForInput {
|
|
if isTerminalSessionState(session.State) {
|
|
return &executor.SessionStateSupersededError{SessionID: session.ID, State: session.State}
|
|
}
|
|
return fmt.Errorf("session %s is %s; cannot schedule task", session.ID, session.State)
|
|
}
|
|
return s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateScheduling)
|
|
}
|
|
|
|
func (s *Service) promoteSessionIfTaskHasNoPrimary(ctx context.Context, taskID string, session *models.TaskSession) {
|
|
if session == nil || session.IsPrimary {
|
|
return
|
|
}
|
|
sessions, err := s.repo.ListTaskSessions(ctx, taskID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to inspect task sessions for missing primary",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", session.ID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
for _, existing := range sessions {
|
|
if existing.IsPrimary {
|
|
return
|
|
}
|
|
}
|
|
if err := s.SetPrimarySession(ctx, session.ID); err != nil {
|
|
s.logger.Warn("failed to promote workflow session as primary",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", session.ID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
session.IsPrimary = true
|
|
}
|
|
|
|
// postLaunchCreated handles post-launch bookkeeping for a created session:
|
|
// records the initial user message (unless skipped) and sets plan mode metadata.
|
|
// It does NOT modify session state — the executor sets STARTING, and event handlers
|
|
// (handleAgentReady) handle the transition to WAITING_FOR_INPUT.
|
|
// autoStart marks the message as having been created by an automated trigger
|
|
// (preserved through to recordInitialMessage's auto_start metadata tag); the
|
|
// flag is only consumed when skipMessage is false (callers that store their
|
|
// own message control its metadata directly).
|
|
func (s *Service) postLaunchCreated(ctx context.Context, taskID, sessionID, prompt string, skipMessage, planModeActive, autoStart bool, attachments []v1.MessageAttachment) {
|
|
if !skipMessage {
|
|
s.recordInitialMessage(ctx, taskID, sessionID, prompt, planModeActive, autoStart, attachments)
|
|
}
|
|
|
|
if planModeActive {
|
|
sess, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err == nil {
|
|
s.setSessionPlanMode(ctx, sess, true)
|
|
}
|
|
}
|
|
}
|
|
|
|
// StartTask manually starts agent execution for a task.
|
|
// If workflowStepID is provided and workflowStepGetter is set, the prompt will be built
|
|
// using the step's prompt_prefix + base prompt + prompt_suffix, and plan mode will be
|
|
// applied if the step has plan_mode enabled.
|
|
// If planMode is true and the workflow step doesn't already apply plan mode,
|
|
// default plan mode instructions are injected into the prompt.
|
|
// autoStart marks the launch as having been triggered by an automated path
|
|
// (PR/issue/Jira/Linear watch, workflow auto-start) rather than direct user
|
|
// input — the seed prompt is tagged so the github cleanup loop can tell
|
|
// "agent ran on its own" from "user actually engaged".
|
|
func (s *Service) StartTask(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, priority string, prompt string, workflowStepID string, planMode, autoStart bool, attachments []v1.MessageAttachment) (*executor.TaskExecution, error) {
|
|
return s.startTask(ctx, taskID, agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, startTaskOptions{})
|
|
}
|
|
|
|
// StartTaskWithEnv starts a task and carries launch-scoped environment variables
|
|
// through to the agent runtime. Existing StartTask callers keep the old behavior.
|
|
func (s *Service) StartTaskWithEnv(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, priority string, prompt string, workflowStepID string, planMode, autoStart bool, attachments []v1.MessageAttachment, env map[string]string) (*executor.TaskExecution, error) {
|
|
return s.startTask(ctx, taskID, agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, startTaskOptions{Env: env})
|
|
}
|
|
|
|
// StartTaskWithEnvAndSkills starts a task with launch-scoped environment and
|
|
// skills selected by the Office run rather than by the durable profile.
|
|
//
|
|
// Its only production caller is the Office scheduler's task starter
|
|
// (backendapp's newOfficeTaskStarter), which always passes autoStart=false —
|
|
// the user kicked off the task; Office only chose the agent and skills. AC-13d
|
|
// requires that traffic be classified automatic regardless, so the origin is
|
|
// forced here rather than trusted from autoStart.
|
|
func (s *Service) StartTaskWithEnvAndSkills(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, priority string, prompt string, workflowStepID string, planMode, autoStart bool, attachments []v1.MessageAttachment, env map[string]string, additionalSkillSlugs []string) (*executor.TaskExecution, error) {
|
|
return s.startTask(ctx, taskID, agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, startTaskOptions{
|
|
Env: env,
|
|
AdditionalSkillSlugs: append([]string(nil), additionalSkillSlugs...),
|
|
Origin: launchOriginAutomatic,
|
|
})
|
|
}
|
|
|
|
// startTaskOptions carries the optional, server-only launch inputs that only
|
|
// some callers supply. Keeping them in one struct avoids growing startTask's
|
|
// already long positional parameter list for every new orthogonal concern.
|
|
type startTaskOptions struct {
|
|
// ProfileExplicit marks a non-empty profile selected through an explicit
|
|
// selector-backed choice. It bypasses workflow-step profile resolution for
|
|
// this new session.
|
|
ProfileExplicit bool
|
|
// Env holds launch-scoped environment variables for the agent runtime.
|
|
Env map[string]string
|
|
// AdditionalSkillSlugs are materialized for this launch in addition to the
|
|
// profile's durable skill selection.
|
|
AdditionalSkillSlugs []string
|
|
// Route pins a concrete execution profile chosen by Office provider routing.
|
|
Route *executor.RouteOverride
|
|
// SpawnOrigin is set when another agent session spawned this launch; it
|
|
// produces the spawner-attribution system block on the first turn.
|
|
SpawnOrigin *SpawnOrigin
|
|
// EntryOptions carries one-shot workflow-move overrides for a fresh
|
|
// auto-started session. The profile override is applied via the profile
|
|
// argument (with ProfileExplicit); the instructions are appended once to the
|
|
// composed workflow prompt below because the prompt is rebuilt from the
|
|
// durable step by ID and cannot see a transient step overlay.
|
|
EntryOptions *workflowmove.EntryOptions
|
|
// WorkflowEntryID pins source bindings and explicit route retries to the
|
|
// immutable step-entry ledger row that initiated an automatic launch.
|
|
WorkflowEntryID int64
|
|
// Origin overrides the automatic/manual classification the session ceiling
|
|
// would otherwise derive from autoStart (AC-13a). It exists for callers
|
|
// AC-13d names, whose autoStart=false does not mean a human asked for this
|
|
// process to start now: an Office scheduler launch only chose a provider.
|
|
// Zero value means "derive from autoStart".
|
|
Origin launchOrigin
|
|
// ceilingEntryBinding is set only by a replay that owns a persisted
|
|
// workflow-entry record. The start path rechecks it immediately before
|
|
// runtime admission so a stale route cannot dispatch the old payload.
|
|
ceilingEntryBinding *models.CeilingWorkflowEntryBinding
|
|
}
|
|
|
|
// StartTaskWithRoute launches a stable Office identity through a complete
|
|
// concrete execution profile selected by the routing dispatcher.
|
|
//
|
|
// launch carries the Office-built launch context (prompt, env, workflow
|
|
// step, attachments, plan-mode flag) so routed launches behave
|
|
// identically to the legacy launch path for everything except provider
|
|
// selection. Without launch, routed runs would fall back to
|
|
// task.Description and drop role framing / AGENTS.md / wake context.
|
|
//
|
|
// Office-routed launches use autoStart=false: the user kicked off the
|
|
// task; the office layer only chose the provider.
|
|
func (s *Service) StartTaskWithRoute(
|
|
ctx context.Context, taskID, agentProfileID string,
|
|
launch executor.LaunchContext, route executor.RouteOverride,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.startTask(ctx, taskID, agentProfileID,
|
|
launch.ExecutorID, launch.ExecutorProfileID, launch.Priority,
|
|
launch.Prompt, launch.WorkflowStepID, launch.PlanMode, false,
|
|
launch.Attachments, startTaskOptions{
|
|
Env: launch.Env,
|
|
AdditionalSkillSlugs: append([]string(nil), launch.AdditionalSkillSlugs...),
|
|
Route: &route,
|
|
// AC-13d: a routed Office launch always passes autoStart=false
|
|
// (the user kicked off the task; Office only chose the provider),
|
|
// which is not the ceiling's manual/automatic question.
|
|
Origin: launchOriginAutomatic,
|
|
})
|
|
}
|
|
|
|
func (s *Service) prepareExplicitWorkflowStartRoute(
|
|
ctx context.Context,
|
|
taskID string,
|
|
workflowSessionConfigStepID string,
|
|
entryIDs ...int64,
|
|
) (*models.TaskSession, string, *models.WorkflowSessionRoute, error) {
|
|
if workflowSessionConfigStepID == "" || s.workflowStepGetter == nil {
|
|
return nil, "", nil, nil
|
|
}
|
|
|
|
candidateStep, err := s.workflowStepGetter.GetStep(ctx, workflowSessionConfigStepID)
|
|
if err != nil {
|
|
return nil, "", nil, fmt.Errorf("load workflow step for session target: %w", err)
|
|
}
|
|
if candidateStep == nil || candidateStep.SessionTarget == nil {
|
|
return nil, "", nil, nil
|
|
}
|
|
|
|
selectedSession, profileID, err := s.selectExplicitWorkflowStartSession(ctx, taskID, candidateStep)
|
|
if err != nil {
|
|
return nil, "", nil, err
|
|
}
|
|
if s.profileExecutionResolver != nil {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, profileID); err != nil {
|
|
return nil, "", nil, err
|
|
}
|
|
}
|
|
|
|
startPolicy := s.resolveStepProfileSessionStartPolicy(candidateStep)
|
|
route := &models.WorkflowSessionRoute{
|
|
OperationID: workflowSessionRouteID(taskID, candidateStep.ID, s.workflowEntryIdentity(ctx, taskID, entryIDs...), candidateStep.SessionTarget, startPolicy),
|
|
DestinationStepID: candidateStep.ID,
|
|
EntryIdentity: s.workflowEntryIdentity(ctx, taskID, entryIDs...),
|
|
TargetKind: string(candidateStep.SessionTarget.Kind),
|
|
TargetStepID: candidateStep.SessionTarget.StepID,
|
|
AgentProfileID: profileID,
|
|
Phase: workflowSessionRoutePrepared,
|
|
}
|
|
if recordedRoute, recordedSession, err := s.loadRecordedWorkflowSessionRoute(ctx, taskID, route.OperationID, candidateStep, profileID); err != nil {
|
|
return nil, "", nil, err
|
|
} else if recordedSession != nil {
|
|
selectedSession = recordedSession
|
|
route = recordedRoute
|
|
}
|
|
if selectedSession != nil || !s.supportsAtomicWorkflowSessionRouteCreation() {
|
|
if err := s.persistWorkflowSessionRoute(ctx, taskID, *route); err != nil {
|
|
return nil, "", nil, err
|
|
}
|
|
}
|
|
return selectedSession, profileID, route, nil
|
|
}
|
|
|
|
func (s *Service) promoteSelectedExplicitWorkflowSession(
|
|
ctx context.Context,
|
|
taskID string,
|
|
session *models.TaskSession,
|
|
route *models.WorkflowSessionRoute,
|
|
) (string, bool, error) {
|
|
if err := s.preflightWorkflowSessionTarget(ctx, taskID, session); err != nil {
|
|
return "", false, err
|
|
}
|
|
var promoted bool
|
|
var err error
|
|
if route != nil {
|
|
promoted, err = s.promoteWorkflowSessionRoute(ctx, taskID, session, route)
|
|
} else {
|
|
promoted, err = s.setNonterminalSessionPrimary(ctx, session.ID)
|
|
}
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("promote explicit workflow session: %w", err)
|
|
}
|
|
if !promoted {
|
|
return "", false, nil
|
|
}
|
|
s.tagSessionAsWorkflowSwitchedForSnapshot(ctx, session)
|
|
return session.ID, true, nil
|
|
}
|
|
|
|
//nolint:cyclop,funlen,gocognit,nestif // launch path threads many orthogonal concerns (workflow-step / agent-profile / office-task / config-mode / route / system-prompt wrapping); splitting it would require shared mutable state across helpers
|
|
func (s *Service) startTask(ctx context.Context, taskID string, agentProfileID string, executorID string, executorProfileID string, priority string, prompt string, workflowStepID string, planMode, autoStart bool, attachments []v1.MessageAttachment, opts startTaskOptions) (*executor.TaskExecution, error) {
|
|
ctx = executor.WithTaskRunnerProfileExplicit(ctx, strings.TrimSpace(executorProfileID) != "")
|
|
if opts.ceilingEntryBinding != nil {
|
|
ctx = withCeilingEntryBinding(ctx, opts.ceilingEntryBinding)
|
|
}
|
|
// One GetWorkflowMeta read shared by profile resolution and prompt build.
|
|
ctx = withWorkflowMetaCache(ctx)
|
|
// Fail before task-state or session mutations when the selected logical
|
|
// profile belongs to a disabled dynamic family. The workflow step may later
|
|
// override the caller profile, so repeat the check after that resolution —
|
|
// unless opts.ProfileExplicit says a non-empty caller profile must be used
|
|
// exactly as supplied, in which case the step's pin is never consulted and
|
|
// must not gate this preflight either.
|
|
preflightProfileID := agentProfileID
|
|
if !opts.ProfileExplicit || agentProfileID == "" {
|
|
var err error
|
|
preflightProfileID, err = s.resolveEffectiveAgentProfile(ctx, taskID, workflowStepID, agentProfileID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if s.profileExecutionResolver != nil {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, preflightProfileID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
env, route := opts.Env, opts.Route
|
|
s.logger.Debug("manually starting task",
|
|
zap.String("task_id", taskID),
|
|
zap.String("agent_profile_id", agentProfileID),
|
|
zap.String("executor_id", executorID),
|
|
zap.String("priority", priority),
|
|
zap.Int("prompt_length", len(prompt)),
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.Bool("plan_mode", planMode),
|
|
zap.Bool("auto_start", autoStart),
|
|
zap.Int("attachments", len(attachments)))
|
|
|
|
origin := opts.Origin
|
|
if origin == "" {
|
|
origin = originFromAutoStart(autoStart)
|
|
}
|
|
claimedCeilingBinding := opts.ceilingEntryBinding != nil
|
|
// Bind workflow-origin launches before seam 1 records a sessionless
|
|
// deferral. The destination session does not exist yet, so this uses the
|
|
// task's current route when available and otherwise the immutable step-entry
|
|
// identity. A later replay must compare this binding before it allocates or
|
|
// dispatches a destination session.
|
|
if opts.ceilingEntryBinding == nil && (workflowStepID != "" || opts.WorkflowEntryID > 0) {
|
|
if bindingTask, bindingErr := s.repo.GetTask(ctx, taskID); bindingErr == nil && bindingTask != nil {
|
|
bindingPayload := map[string]interface{}{
|
|
metaKeyWorkflowStepID: workflowStepID,
|
|
}
|
|
if opts.WorkflowEntryID > 0 {
|
|
bindingPayload["workflow_entry_id"] = opts.WorkflowEntryID
|
|
}
|
|
if binding, bound := s.deriveCeilingEntryBinding(ctx, bindingTask, bindingPayload, ""); bound {
|
|
opts.ceilingEntryBinding = &binding
|
|
}
|
|
}
|
|
}
|
|
if claimedCeilingBinding {
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, opts.ceilingEntryBinding); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
seam1Res, deferred, err := s.admitOrDeferSeam1(ctx, taskID, origin,
|
|
seam1StartPayload(agentProfileID, executorID, executorProfileID, priority, prompt, workflowStepID, planMode, autoStart, attachments, opts))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if deferred {
|
|
return nil, ErrCeilingLaunchDeferred
|
|
}
|
|
defer seam1Res.releaseIfNotConsumed()
|
|
|
|
// Reserve any pending "start it later" intent for the whole of this start,
|
|
// taken before the session is prepared rather than just before the launch:
|
|
// the gate only needs the intent to still be there, so any window in which
|
|
// this start has already created a session is a window that ends in two.
|
|
// The deferred release covers every failure path, including the early
|
|
// returns just below.
|
|
launchClaim := s.claimDeferredLaunchForStart(ctx, taskID)
|
|
defer launchClaim.releaseIfHeld(ctx)
|
|
|
|
isOfficeTask, err := s.lookupOfficeTask(ctx, taskID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to determine office task status: %w", err)
|
|
}
|
|
if isOfficeTask {
|
|
if err := validateOfficeLaunchEnv(taskID, env); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
// Automated launch callers perform this check before entering startTask,
|
|
// but several event paths can race after that check and before session
|
|
// preparation. Keep the terminal-PR guard at the session boundary too so
|
|
// no automated path can create a session for a closed or merged PR.
|
|
if autoStart {
|
|
if task, taskErr := s.repo.GetTask(ctx, taskID); taskErr != nil {
|
|
return nil, fmt.Errorf("failed to fetch task for automatic launch gate: %w", taskErr)
|
|
} else if taskRequiresManualForkPRStart(task) {
|
|
return nil, errForkPRManualStartRequired
|
|
} else if s.shouldSkipTerminalPRAutoStart(ctx, task) {
|
|
return nil, nil
|
|
}
|
|
}
|
|
workflowSessionConfigStepID := workflowStepID
|
|
// Manual starts often omit workflow_step_id because the task is already
|
|
// bound to its current step. Resolve that canonical step before profile
|
|
// selection and launch-layer session configuration so a start-step rule is
|
|
// applied before the first prompt as well.
|
|
if workflowSessionConfigStepID == "" {
|
|
if dbTask, taskErr := s.repo.GetTask(ctx, taskID); taskErr != nil {
|
|
s.logger.Warn("failed to fetch task for workflow step fallback",
|
|
zap.String("task_id", taskID), zap.Error(taskErr))
|
|
} else if dbTask != nil {
|
|
workflowSessionConfigStepID = dbTask.WorkflowStepID
|
|
}
|
|
}
|
|
|
|
// Office tasks do NOT transition through SCHEDULING / IN_PROGRESS on
|
|
// every run. Their lifecycle status (todo / in_review / done /
|
|
// blocked / cancelled) reflects the *user-meaningful workflow*, not
|
|
// the orchestrator's runtime cycle. Runs schedule the *agent*,
|
|
// not the task — the agent's runtime state is shown via the topbar
|
|
// Working spinner + inline session timeline entry. Suppressing the
|
|
// transition here avoids gratuitous flicker (REVIEW → SCHEDULING →
|
|
// IN_PROGRESS → REVIEW for a single comment-reply cycle) and matches
|
|
// the user's mental model.
|
|
if isOfficeTask {
|
|
s.logger.Debug("skipping SCHEDULING transition for office task",
|
|
zap.String("task_id", taskID))
|
|
} else if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateScheduling); err != nil {
|
|
s.logger.Warn("failed to update task state to SCHEDULING",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
|
|
s.moveTaskToWorkflowStep(ctx, taskID, workflowStepID)
|
|
|
|
officeAgentProfileID := ""
|
|
if isOfficeTask {
|
|
officeAgentProfileID = agentProfileID
|
|
}
|
|
|
|
// Resolve the workflow step's agent profile override.
|
|
// The frontend may pass the workspace default profile, but the step may
|
|
// require a different agent (e.g., Codex on "In Progress", Auggie on "Review").
|
|
callerProfileID := agentProfileID
|
|
if opts.ProfileExplicit && agentProfileID != "" {
|
|
s.logger.Info("explicit selector-backed agent profile selection takes precedence over workflow step",
|
|
zap.String("task_id", taskID),
|
|
zap.String("agent_profile_id", agentProfileID))
|
|
} else {
|
|
agentProfileID = preflightProfileID
|
|
}
|
|
if s.profileExecutionResolver != nil {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, agentProfileID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
overrideApplied := agentProfileID != callerProfileID
|
|
if route != nil && route.ExecutionProfileID != "" {
|
|
agentProfileID = route.ExecutionProfileID
|
|
}
|
|
|
|
// Fetch the task from the repository to get complete task info
|
|
task, err := s.scheduler.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
s.logger.Error("failed to fetch task for manual start",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
return nil, err
|
|
}
|
|
launchErrorStamp := ""
|
|
if launchError, found := models.LoadTaskLaunchError(task.Metadata); found {
|
|
launchErrorStamp = launchError.Stamp()
|
|
}
|
|
if executorID == "" {
|
|
if v, ok := task.Metadata[models.MetaKeyExecutorID].(string); ok && v != "" {
|
|
executorID = v
|
|
}
|
|
}
|
|
if executorProfileID == "" {
|
|
if v, ok := task.Metadata[models.MetaKeyExecutorProfileID].(string); ok && v != "" {
|
|
executorProfileID = v
|
|
}
|
|
}
|
|
|
|
// Override priority if provided in the request
|
|
if priority != "" {
|
|
task.Priority = priority
|
|
}
|
|
|
|
var explicitStartRoute *models.WorkflowSessionRoute
|
|
var selectedExplicitSession *models.TaskSession
|
|
var explicitProfileID string
|
|
// opts.ProfileExplicit means the caller's agent_profile_id must be used
|
|
// exactly as supplied, with no inheritance or defaulting from the
|
|
// destination workflow step (AC-PROFILES-001.4/D14a). SessionTarget
|
|
// routing is itself a form of that defaulting — it can resolve to a
|
|
// different profile (and reuse a different session) than the one the
|
|
// caller asked for — so it must not even be consulted here, not merely
|
|
// have its result discarded, or a step-targeted session/executor could
|
|
// still be picked underneath the overwritten profile.
|
|
if !opts.ProfileExplicit {
|
|
selectedExplicitSession, explicitProfileID, explicitStartRoute, err = s.prepareExplicitWorkflowStartRoute(
|
|
ctx,
|
|
task.ID,
|
|
workflowSessionConfigStepID,
|
|
opts.WorkflowEntryID,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if explicitStartRoute != nil && explicitProfileID != "" {
|
|
agentProfileID = explicitProfileID
|
|
overrideApplied = agentProfileID != callerProfileID
|
|
}
|
|
}
|
|
|
|
// Use provided prompt, fall back to task description
|
|
effectivePrompt := prompt
|
|
if effectivePrompt == "" {
|
|
effectivePrompt = task.Description
|
|
}
|
|
|
|
// Prepare session first so we have the sessionID for config context injection.
|
|
// For office tasks, replace the per-launch PrepareSession with the per-(task,
|
|
// agent) EnsureSessionForAgent so runs reuse one row across turns.
|
|
var sessionID string
|
|
var sessionCreated bool
|
|
if selectedExplicitSession != nil {
|
|
var promoted bool
|
|
var promoteErr error
|
|
sessionID, promoted, promoteErr = s.promoteSelectedExplicitWorkflowSession(ctx, task.ID, selectedExplicitSession, explicitStartRoute)
|
|
if promoteErr != nil {
|
|
return nil, promoteErr
|
|
}
|
|
if !promoted {
|
|
selectedExplicitSession = nil
|
|
}
|
|
}
|
|
if sessionID == "" {
|
|
if explicitStartRoute != nil {
|
|
sessionID, sessionCreated, err = s.prepareSessionForStartWithWorkflowRoute(ctx, task, agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID, explicitStartRoute)
|
|
} else {
|
|
sessionID, sessionCreated, err = s.prepareSessionForStart(ctx, task, agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if explicitStartRoute != nil && selectedExplicitSession == nil {
|
|
promoted, promoteErr := s.promoteWorkflowSessionRoute(ctx, task.ID, &models.TaskSession{ID: sessionID}, explicitStartRoute)
|
|
if promoteErr != nil {
|
|
return nil, fmt.Errorf("promote explicit workflow start session: %w", promoteErr)
|
|
}
|
|
if !promoted {
|
|
return nil, fmt.Errorf("explicit workflow start session became terminal before promotion")
|
|
}
|
|
}
|
|
seam1Res.rebindToSession(sessionID)
|
|
s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam1Res.manualOverride, seam1Res.population, seam1Res.populationKnown, seam1Res.ceiling)
|
|
|
|
// Seed a matching conditional session configuration before lifecycle
|
|
// startup. The ACP manager applies this durable runtime layer after the
|
|
// selected profile and before the first prompt, preserving the original
|
|
// session tab.
|
|
s.applyWorkflowSessionConfigBeforeLaunchForStep(ctx, taskID, sessionID, workflowSessionConfigStepID)
|
|
// Surface the newly created session before LaunchPreparedSession performs
|
|
// potentially slow environment setup (for example, a Docker health check).
|
|
// The frontend adopts this CREATED session and can render
|
|
// executor.prepare.progress events while the launch request is still pending.
|
|
if sessionCreated {
|
|
s.publishSessionCreatedEvent(ctx, taskID, sessionID, workflowStepID)
|
|
}
|
|
|
|
// When the workflow step overrode the caller's profile, tag the session
|
|
// for provenance: the profile came from workflow routing rather than
|
|
// direct user selection.
|
|
if overrideApplied {
|
|
s.tagSessionAsWorkflowSwitched(ctx, sessionID)
|
|
}
|
|
|
|
// Passthrough sessions skip both the workflow-prompt "@name" expansion below
|
|
// and the Kandev MCP wrap further down: the prompt is typed straight into
|
|
// the agent CLI's TTY and the user sees it verbatim — they don't want a
|
|
// wall of MCP-tool boilerplate or a hidden expansion block prepended to
|
|
// "hello". Use the session snapshot, not a live profile lookup, so a
|
|
// mid-run profile edit cannot change wrap behavior. Looked up once here and
|
|
// reused below so we don't hit GetTaskSession twice for the same fact.
|
|
// If the lookup fails, resolveIsPassthroughForLaunch fails safe by
|
|
// treating the session as passthrough: skipping the wrap/expansion is
|
|
// strictly less harmful than leaking hidden <kandev-system> content into
|
|
// a real passthrough session's PTY.
|
|
isPassthrough := s.resolveIsPassthroughForLaunch(ctx, sessionID)
|
|
launchSession, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to reload launch session: %w", err)
|
|
}
|
|
if explicitStartRoute == nil && workflowSessionConfigStepID != "" && s.workflowStepGetter != nil {
|
|
if sourceStep, stepErr := s.workflowStepGetter.GetStep(ctx, workflowSessionConfigStepID); stepErr != nil {
|
|
return nil, fmt.Errorf("load workflow source step for session binding: %w", stepErr)
|
|
} else if sourceStep != nil {
|
|
if bindErr := s.recordWorkflowSourceBinding(ctx, task.ID, sourceStep, launchSession, opts.WorkflowEntryID); bindErr != nil {
|
|
return nil, bindErr
|
|
}
|
|
}
|
|
}
|
|
|
|
if route == nil {
|
|
if agentProfileID, err = s.resolveDynamicLaunchExecution(ctx, launchSession, agentProfileID, false); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
// Dynamic resolution updates the session snapshot after the initial
|
|
// passthrough check. Use the concrete candidate's snapshot for prompt
|
|
// wrapping and launch behavior.
|
|
isPassthrough = launchSession.IsPassthrough
|
|
|
|
skipStepPrompt := opts.EntryOptions != nil && opts.EntryOptions.SkipStepPrompt
|
|
effectivePrompt, planModeActive, promptReferenceContext := s.applyWorkflowAndPlanMode(
|
|
ctx, effectivePrompt, task.ID, sessionID, workflowStepID,
|
|
planMode, task.IsEphemeral, isPassthrough, skipStepPrompt,
|
|
)
|
|
|
|
// Append one-shot workflow-move instructions to a fresh auto-started
|
|
// session. The prompt was rebuilt from the durable step by ID above, so a
|
|
// transient step overlay could not carry them; append the same sentinel
|
|
// block once here. The sentinel guards against a double append on retry.
|
|
if opts.EntryOptions != nil && opts.EntryOptions.Instructions != "" &&
|
|
!strings.Contains(effectivePrompt, workflowmove.InstructionsEnd) {
|
|
wrapped := workflowmove.WrapInstructions(opts.EntryOptions.Instructions)
|
|
if effectivePrompt == "" {
|
|
effectivePrompt = wrapped
|
|
} else {
|
|
effectivePrompt = effectivePrompt + "\n\n" + wrapped
|
|
}
|
|
}
|
|
|
|
// Inject config context for config-mode sessions (dedicated settings chat)
|
|
configMode := false
|
|
if cm, ok := launchSession.Metadata["config_mode"].(bool); ok && cm {
|
|
configMode = true
|
|
effectivePrompt = sysprompt.InjectConfigContext(sessionID, effectivePrompt)
|
|
}
|
|
titleOwner := false
|
|
if !configMode && !isOfficeTask {
|
|
titleOwner, err = s.ClaimTaskTitleSession(ctx, task.ID, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to claim first-turn task title: %w", err)
|
|
}
|
|
}
|
|
|
|
// Wrap the first prompt with the Kandev MCP system block (task/session IDs +
|
|
// tool list). Done at the orchestrator layer so recordInitialMessage persists
|
|
// the wrapped form to task_session_messages; Message.ToAPI strips the
|
|
// <kandev-system> block for the UI bubble and exposes it via raw_content.
|
|
// Only the first launch carries this wrap — follow-up prompts and resumes
|
|
// rely on the agent CLI's conversation history retaining it.
|
|
// Idempotent: upstream call sites (wsAddMessage on CREATED sessions,
|
|
// recordAutoStartMessage) wrap before recording the user message so the DB
|
|
// row carries the block; the mode-aware injector canonicalizes it instead of
|
|
// double-wrapping.
|
|
skipKandevMCPWrap := isPassthrough
|
|
// `task` here is *v1.Task from the scheduler, which does NOT carry the
|
|
// orchestrator's WorkflowStepID — go through the task-ID variant so the
|
|
// repo lookup pulls the canonical step. Using the workflowStepID parameter
|
|
// directly is wrong because it can be empty on manual user-initiated starts
|
|
// while the task is already bound to a signal-gated step in the DB.
|
|
if effectivePrompt != "" || len(attachments) > 0 {
|
|
includeCanvasGuidance := false
|
|
if !isOfficeTask && !skipKandevMCPWrap && !configMode {
|
|
includeCanvasGuidance, err = s.taskSessionCanvasGuidanceEnabled(ctx, task.ID, launchSession, true)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to resolve canvas prompt capability: %w", err)
|
|
}
|
|
}
|
|
effectivePrompt = s.applyLaunchPromptContext(ctx, launchPromptContext{
|
|
prompt: effectivePrompt,
|
|
taskID: task.ID,
|
|
sessionID: sessionID,
|
|
isOfficeTask: isOfficeTask,
|
|
isPassthrough: skipKandevMCPWrap,
|
|
configMode: configMode,
|
|
referenceContext: promptReferenceContext,
|
|
includeTaskTitleTool: !configMode && titleOwner,
|
|
includeCanvasGuidance: includeCanvasGuidance,
|
|
autopilot: task.Autopilot,
|
|
includeParentQuestionTool: task.Autopilot && task.ParentID != "",
|
|
spawnOrigin: opts.SpawnOrigin,
|
|
})
|
|
}
|
|
|
|
// Office tasks restrict the MCP toolset: kanban tools (move/update/list
|
|
// task, etc.) are excluded because office agents call those via the
|
|
// kandev CLI ($KANDEV_CLI). See docs/specs/office/system-design/agents-03.md.
|
|
mcpMode := ""
|
|
if isOfficeTask {
|
|
mcpMode = executor.McpModeOffice
|
|
}
|
|
if claimedCeilingBinding {
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, opts.ceilingEntryBinding); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
initialTurnID, initialTurnCreated := s.startTurnForSessionWithOwnership(ctx, sessionID)
|
|
if claimedCeilingBinding {
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, opts.ceilingEntryBinding); err != nil {
|
|
if initialTurnCreated {
|
|
s.completeTurnIfCurrent(ctx, sessionID, initialTurnID)
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Cache the raw prompt so a transient-provider-error (529) retry can
|
|
// re-drive this first turn — initial launches bypass PromptTask.
|
|
s.rememberTurnPrompt(sessionID, prompt, "", planMode, attachments)
|
|
|
|
execution, err := s.launchPreparedSessionWithDynamicFallback(ctx, task, sessionID, executor.LaunchOptions{
|
|
AgentProfileID: agentProfileID,
|
|
OfficeAgentProfileID: officeAgentProfileID,
|
|
ExecutorID: executorID,
|
|
TurnID: initialTurnID,
|
|
Prompt: effectivePrompt,
|
|
WorkflowStepID: workflowStepID,
|
|
StartAgent: true,
|
|
McpMode: mcpMode,
|
|
Attachments: attachments,
|
|
Env: env,
|
|
AdditionalSkillSlugs: append([]string(nil), opts.AdditionalSkillSlugs...),
|
|
RouteOverride: route,
|
|
})
|
|
if err != nil {
|
|
if initialTurnCreated {
|
|
s.completeTurnIfCurrent(ctx, sessionID, initialTurnID)
|
|
}
|
|
return nil, s.handleSessionLaunchFailure(ctx, taskID, sessionID, err)
|
|
}
|
|
|
|
s.postLaunchStart(ctx, taskID, execution, effectivePrompt, planModeActive || configMode, planModeActive, autoStart, attachments)
|
|
execution.TurnID = initialTurnID
|
|
s.clearTaskLaunchErrorIfStamp(ctx, taskID, launchErrorStamp)
|
|
|
|
// The agent is running, so the reservation becomes a consumption.
|
|
launchClaim.consume(ctx)
|
|
seam1Res.consume()
|
|
|
|
// Note: Task stays in SCHEDULING state until the agent is fully initialized.
|
|
// The executor will transition to IN_PROGRESS after StartAgentProcess() succeeds.
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
func (s *Service) applyWorkflowSessionConfigBeforeLaunchForStep(
|
|
ctx context.Context,
|
|
taskID string,
|
|
sessionID string,
|
|
workflowStepID string,
|
|
) {
|
|
if s.workflowStepGetter == nil || workflowStepID == "" {
|
|
return
|
|
}
|
|
workflowStep, err := s.workflowStepGetter.GetStep(ctx, workflowStepID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to load workflow step for session configuration",
|
|
zap.String("task_id", taskID), zap.String("step_id", workflowStepID), zap.Error(err))
|
|
return
|
|
}
|
|
if workflowStep == nil {
|
|
return
|
|
}
|
|
preparedSession, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to reload session for session configuration",
|
|
zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Error(err))
|
|
return
|
|
}
|
|
s.applyWorkflowSessionConfigBeforeLaunch(ctx, taskID, preparedSession, workflowStep)
|
|
}
|
|
|
|
// launchPromptContext carries what the first turn of a launch needs in order to
|
|
// compose its system context.
|
|
type launchPromptContext struct {
|
|
prompt string
|
|
taskID string
|
|
sessionID string
|
|
isOfficeTask bool
|
|
isPassthrough bool
|
|
configMode bool
|
|
includeTaskTitleTool bool
|
|
includeCanvasGuidance bool
|
|
autopilot bool
|
|
includeParentQuestionTool bool
|
|
referenceContext string
|
|
spawnOrigin *SpawnOrigin
|
|
}
|
|
|
|
// applyLaunchPromptContext prepends the first-turn system context to a launch
|
|
// prompt: the Kandev (or Office) MCP block, plus spawner attribution when the
|
|
// launch came from spawn_session_kandev.
|
|
//
|
|
// Passthrough profiles get attribution only, as plain text — see
|
|
// applySpawnOriginText for why they skip the MCP block entirely.
|
|
func (s *Service) applyLaunchPromptContext(ctx context.Context, p launchPromptContext) string {
|
|
var pullRequestTargetContext string
|
|
p.prompt, pullRequestTargetContext = s.addTaskPullRequestTargetContext(
|
|
ctx, p.taskID, p.prompt, p.isPassthrough,
|
|
)
|
|
if p.isPassthrough {
|
|
prompt := applySpawnOriginText(p.prompt, p.spawnOrigin)
|
|
if p.includeTaskTitleTool {
|
|
return sysprompt.PendingTaskTitlePassthroughInstruction() + "\n\n" + prompt
|
|
}
|
|
return prompt
|
|
}
|
|
// Spawner attribution is built here, not by the MCP handler that filled
|
|
// SpawnOrigin: the injectors below strip every <kandev-system> block they do
|
|
// not recognize, so the block has to be generated from the same server state
|
|
// that whitelists it as trusted content.
|
|
prompt, spawnContext := applySpawnOriginContext(p.prompt, p.spawnOrigin)
|
|
if p.isOfficeTask {
|
|
return sysprompt.InjectOfficeContextWithOptions(
|
|
p.taskID, p.sessionID, prompt,
|
|
s.StepRequiresCompletionSignal(ctx, p.taskID),
|
|
p.referenceContext, spawnContext, pullRequestTargetContext,
|
|
)
|
|
}
|
|
return sysprompt.InjectKandevContextWithOptions(p.taskID, p.sessionID, prompt, sysprompt.KandevContextOptions{
|
|
RequiresCompletionSignal: s.StepRequiresCompletionSignal(ctx, p.taskID),
|
|
IncludeCoordinatorTaskControls: !p.configMode,
|
|
IncludeTaskTitleTool: p.includeTaskTitleTool,
|
|
IncludeCanvasGuidance: p.includeCanvasGuidance,
|
|
Autopilot: p.autopilot,
|
|
IncludeUserQuestionTool: !p.autopilot && !p.isPassthrough,
|
|
IncludeParentQuestionTool: p.autopilot && p.includeParentQuestionTool,
|
|
}, p.referenceContext, spawnContext, pullRequestTargetContext)
|
|
}
|
|
|
|
// spawnOriginContent renders the spawner-attribution text for a launch requested
|
|
// through spawn_session_kandev, or "" for ordinary launches.
|
|
func spawnOriginContent(origin *SpawnOrigin) string {
|
|
if origin == nil {
|
|
return ""
|
|
}
|
|
return sysprompt.SpawnedSessionContext(origin.TaskID, origin.SessionID, origin.SessionName)
|
|
}
|
|
|
|
// applySpawnOriginContext prepends the spawner-attribution system block for a
|
|
// launch requested through spawn_session_kandev and returns the content that
|
|
// the first-turn injector must whitelist so the block is not stripped again as
|
|
// untrusted. Ordinary launches pass through unchanged with empty content.
|
|
func applySpawnOriginContext(prompt string, origin *SpawnOrigin) (string, string) {
|
|
content := spawnOriginContent(origin)
|
|
if content == "" {
|
|
return prompt, ""
|
|
}
|
|
return sysprompt.Wrap(content) + "\n\n" + prompt, content
|
|
}
|
|
|
|
// applySpawnOriginText prepends the same attribution as plain prompt text, for
|
|
// passthrough profiles whose prompt reaches the agent through a TTY the user is
|
|
// watching. There is no injector on that path to whitelist a system block, and
|
|
// wrapping it would only put raw <kandev-system> markup on the terminal.
|
|
func applySpawnOriginText(prompt string, origin *SpawnOrigin) string {
|
|
content := spawnOriginContent(origin)
|
|
if content == "" {
|
|
return prompt
|
|
}
|
|
return content + "\n\n" + prompt
|
|
}
|
|
|
|
// isOfficeTask returns the repository's canonical Office ownership projection.
|
|
func (s *Service) isOfficeTask(ctx context.Context, taskID string) bool {
|
|
isOfficeTask, err := s.lookupOfficeTask(ctx, taskID)
|
|
return err == nil && isOfficeTask
|
|
}
|
|
|
|
func (s *Service) lookupOfficeTask(ctx context.Context, taskID string) (bool, error) {
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return dbTask != nil && dbTask.IsFromOffice, nil
|
|
}
|
|
|
|
func validateOfficeRuntimeEnv(env map[string]string) error {
|
|
required := []string{
|
|
"KANDEV_CLI",
|
|
"KANDEV_API_URL",
|
|
"KANDEV_API_KEY",
|
|
"KANDEV_AGENT_ID",
|
|
"KANDEV_WORKSPACE_ID",
|
|
"KANDEV_RUN_ID",
|
|
"KANDEV_TASK_ID",
|
|
}
|
|
missing := make([]string, 0, len(required))
|
|
for _, key := range required {
|
|
if strings.TrimSpace(env[key]) == "" {
|
|
missing = append(missing, key)
|
|
}
|
|
}
|
|
if len(missing) == 0 {
|
|
return nil
|
|
}
|
|
return fmt.Errorf(
|
|
"office runtime context is incomplete (missing %s); start or wake the task through Office",
|
|
strings.Join(missing, ", "),
|
|
)
|
|
}
|
|
|
|
var (
|
|
errForkPRManualStartRequired = errors.New("fork pull request review requires a manual start")
|
|
errOfficeTaskStartRequiresScheduler = errors.New(
|
|
"office tasks must be started through Office; use StartTaskWithEnv with scheduler-injected credentials",
|
|
)
|
|
errOfficeTaskResumeRequiresScheduler = errors.New(
|
|
"office tasks must be resumed through Office; use the Office scheduler to start a fresh run",
|
|
)
|
|
errOfficePreparedResumeRequiresScheduler = errors.New(
|
|
"office tasks must be restarted through Office; prepared-workspace resume is not supported for Office-owned tasks",
|
|
)
|
|
)
|
|
|
|
// validateOfficeLaunchEnv requires the scheduler context to be bound to the
|
|
// task being launched. StartTaskWithEnv is only wired to the internal Office
|
|
// scheduler adapter; the task binding still prevents a complete context map
|
|
// from being reused for a different task.
|
|
func validateOfficeLaunchEnv(taskID string, env map[string]string) error {
|
|
if err := validateOfficeRuntimeEnv(env); err != nil {
|
|
return err
|
|
}
|
|
if env["KANDEV_TASK_ID"] != taskID {
|
|
return fmt.Errorf(
|
|
"office runtime context is bound to task %q, not %q; start or wake the task through Office",
|
|
env["KANDEV_TASK_ID"], taskID,
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// prepareSessionForStart creates the session for a launch and propagates any
|
|
// inherited workspace environment onto it.
|
|
//
|
|
// The propagation (inherit_parent / shared_group) lives here rather than only
|
|
// in PrepareTaskSession so every launch entry point inherits consistently —
|
|
// including the direct start path (startTask), which MCP-created subtasks reach
|
|
// via auto-start. Without this, an inherit_parent subtask launched through
|
|
// startTask would provision a fresh worktree instead of reusing the parent's.
|
|
// propagateInheritedEnvironment is a no-op for tasks without a workspace policy.
|
|
func (s *Service) prepareSessionForStart(
|
|
ctx context.Context, task *v1.Task,
|
|
agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID string,
|
|
) (string, bool, error) {
|
|
return s.prepareSessionForStartWithWorkflowRoute(ctx, task, agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID, nil)
|
|
}
|
|
|
|
func (s *Service) prepareSessionForStartWithWorkflowRoute(
|
|
ctx context.Context, task *v1.Task,
|
|
agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID string,
|
|
workflowRoute *models.WorkflowSessionRoute,
|
|
) (string, bool, error) {
|
|
sessionID, created, err := s.createStartSessionWithWorkflowRoute(ctx, task, agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID, workflowRoute)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if err := s.propagateInheritedEnvironment(ctx, task, sessionID); err != nil {
|
|
// This legacy creation path cannot bind an inherited ID until it has a
|
|
// session row. Compensate before returning so callers never observe a
|
|
// partial sibling session when the required parent/group workspace is
|
|
// unavailable.
|
|
if deleteErr := s.deleteSessionAndPublishRemoval(ctx, task.ID, sessionID); deleteErr != nil {
|
|
s.logger.Warn("failed to compensate inherited workspace session",
|
|
zap.String("session_id", sessionID), zap.Error(deleteErr))
|
|
}
|
|
return "", false, err
|
|
}
|
|
if created {
|
|
if err := s.persistInitialPromptPreviewFromContext(ctx, task.ID, sessionID); err != nil {
|
|
return "", false, err
|
|
}
|
|
}
|
|
return sessionID, created, nil
|
|
}
|
|
|
|
// resolveIsPassthroughForLaunch reloads the session snapshot to decide
|
|
// whether this launch is a passthrough session (skip the Kandev MCP wrap and
|
|
// the "@name" prompt-reference expansion). If the reload fails, this fails
|
|
// safe by treating the session as passthrough: skipping the wrap/expansion is
|
|
// strictly less harmful than leaking hidden <kandev-system> content into a
|
|
// real passthrough session's PTY.
|
|
func (s *Service) resolveIsPassthroughForLaunch(ctx context.Context, sessionID string) bool {
|
|
launchSession, sessErr := s.repo.GetTaskSession(ctx, sessionID)
|
|
if sessErr != nil {
|
|
s.logger.Warn("failed to reload session for passthrough check; defaulting to passthrough (skip hidden-content injection) as the safer failure mode",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(sessErr))
|
|
return true
|
|
}
|
|
return launchSession.IsPassthrough
|
|
}
|
|
|
|
// resolveExecutionForLaunchSession keeps the logical profile on the session
|
|
// while selecting a concrete profile only once for a new dynamic route. A
|
|
// persisted route is authoritative during resume and must not advance merely
|
|
// because a caller is launching the same logical session again.
|
|
func (s *Service) resolveExecutionForLaunchSession(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
) (agentruntime.ProfileExecution, bool, error) {
|
|
if session == nil {
|
|
return agentruntime.ProfileExecution{}, false, errors.New("launch session is required for profile resolution")
|
|
}
|
|
if resolved, routeClaimed, handled, err := s.resolveDurableExecutionForLaunch(ctx, session); handled {
|
|
return resolved, routeClaimed, err
|
|
}
|
|
if session.RouteGeneration > 0 && session.ExecutionProfileID != "" {
|
|
resolved, err := s.profileExecutionResolver.ResolveExisting(
|
|
ctx, session.ID, session.AgentProfileID, session.ExecutionProfileID,
|
|
session.RouteGeneration, 0, session.RouteReason,
|
|
)
|
|
return resolved, false, err
|
|
}
|
|
resolved, err := s.profileExecutionResolver.Resolve(
|
|
ctx, session.ID, session.AgentProfileID, session.RouteGeneration, "",
|
|
)
|
|
if err != nil {
|
|
return agentruntime.ProfileExecution{}, false, err
|
|
}
|
|
return resolved, resolved.Generation > 0, nil
|
|
}
|
|
|
|
func (s *Service) resolveDurableExecutionForLaunch(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
) (agentruntime.ProfileExecution, bool, bool, error) {
|
|
loader, ok := s.repo.(dynamicRouteStateLoader)
|
|
if !ok {
|
|
return agentruntime.ProfileExecution{}, false, false, nil
|
|
}
|
|
state, err := loader.LoadRouteState(ctx, session.ID)
|
|
if err != nil {
|
|
return agentruntime.ProfileExecution{}, false, true, fmt.Errorf("load durable dynamic route state: %w", err)
|
|
}
|
|
if state == nil || state.LogicalProfileID != session.AgentProfileID || state.Generation <= 0 {
|
|
return agentruntime.ProfileExecution{}, false, false, nil
|
|
}
|
|
if state.ExecutionProfileID == "" {
|
|
if state.Status != dynamicRouteStatusWaiting {
|
|
return agentruntime.ProfileExecution{}, false, true, &dynamicruntime.NoEligibleCandidateError{
|
|
SessionID: session.ID, LogicalProfile: session.AgentProfileID,
|
|
Generation: state.Generation,
|
|
}
|
|
}
|
|
resolved, resolveErr := s.profileExecutionResolver.Resolve(
|
|
ctx, session.ID, session.AgentProfileID, state.Generation, "",
|
|
)
|
|
return resolved, resolved.Generation > 0, true, resolveErr
|
|
}
|
|
resolved, err := s.profileExecutionResolver.ResolveExisting(
|
|
ctx, session.ID, session.AgentProfileID, state.ExecutionProfileID,
|
|
state.Generation, state.ProfileVersion, "durable_route_state",
|
|
)
|
|
return resolved, true, true, err
|
|
}
|
|
|
|
func (s *Service) resolveDynamicLaunchExecution(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
profileID string,
|
|
validate bool,
|
|
) (string, error) {
|
|
if s.profileExecutionResolver == nil {
|
|
return profileID, nil
|
|
}
|
|
if validate {
|
|
if err := s.profileExecutionResolver.ValidateProfile(ctx, session.AgentProfileID); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
resolved, routeClaimed, err := s.resolveExecutionForLaunchSession(ctx, session)
|
|
if err != nil {
|
|
s.recordDynamicRouteResolutionFailure(ctx, session, err)
|
|
return "", err
|
|
}
|
|
if !routeClaimed {
|
|
if resolved.ExecutionProfileID == "" {
|
|
return profileID, nil
|
|
}
|
|
return resolved.ExecutionProfileID, nil
|
|
}
|
|
applyResolvedExecution(session, resolved)
|
|
if err := s.repo.UpdateTaskSession(ctx, session); err != nil {
|
|
return "", fmt.Errorf("persist dynamic route attribution: %w", err)
|
|
}
|
|
return resolved.ExecutionProfileID, nil
|
|
}
|
|
|
|
func applyResolvedExecution(session *models.TaskSession, resolved agentruntime.ProfileExecution) {
|
|
if session == nil {
|
|
return
|
|
}
|
|
previousExecutionProfileID := session.ExecutionProfileID
|
|
session.ExecutionProfileID = resolved.ExecutionProfileID
|
|
session.RouteGeneration = resolved.Generation
|
|
session.RouteState = "starting"
|
|
session.RouteReason = resolved.Decision.Reason
|
|
if session.RouteReason == "" {
|
|
session.RouteReason = "candidate_order"
|
|
}
|
|
applyDynamicRouteDecisionProjection(session, resolved.Decision)
|
|
// A new concrete candidate never inherits a provider-native ACP identity.
|
|
// Reapplying the persisted route during restart must keep the identity so
|
|
// native conversation resume remains possible.
|
|
if previousExecutionProfileID != resolved.ExecutionProfileID {
|
|
session.DownstreamACPSessionID = ""
|
|
}
|
|
if resolved.Profile != nil {
|
|
session.AgentProfileSnapshot = map[string]interface{}{
|
|
"id": resolved.Profile.ID,
|
|
"name": resolved.Profile.Name,
|
|
"agent_id": resolved.Profile.AgentID,
|
|
"model": resolved.Profile.Model,
|
|
"mode": resolved.Profile.Mode,
|
|
"config_options": resolved.Profile.ConfigOptions,
|
|
"auto_approve": resolved.Profile.AutoApprove,
|
|
"dangerously_skip_permissions": resolved.Profile.DangerouslySkipPermissions,
|
|
"cli_passthrough": resolved.Profile.CLIPassthrough,
|
|
}
|
|
session.IsPassthrough = resolved.Profile.CLIPassthrough
|
|
}
|
|
}
|
|
|
|
func (s *Service) recordDynamicRouteResolutionFailure(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
err error,
|
|
) {
|
|
if session == nil || err == nil {
|
|
return
|
|
}
|
|
var noCandidate *dynamicruntime.NoEligibleCandidateError
|
|
if !errors.As(err, &noCandidate) {
|
|
return
|
|
}
|
|
session.RouteGeneration = noCandidate.Generation
|
|
session.RouteState = dynamicRouteStatusWaiting
|
|
session.RouteReason = "no_eligible_candidate"
|
|
session.UpdatedAt = time.Now().UTC()
|
|
if updateErr := s.repo.UpdateTaskSession(ctx, session); updateErr != nil {
|
|
s.logger.Warn("failed to persist dynamic route waiting state",
|
|
zap.String("session_id", session.ID), zap.Error(updateErr))
|
|
}
|
|
}
|
|
|
|
// createStartSession picks the right session-creation path for the task:
|
|
// Office tasks use the per-(task, agent) EnsureSessionForAgent path (so runs
|
|
// reuse one row across turns); kanban / quick-chat fall through to the
|
|
// per-launch PrepareSession used since day one. An Office task can be created
|
|
// before its runner seat is projected, so the run's captured identity or its
|
|
// resolved execution profile supplies the owner until the seat is assigned.
|
|
//
|
|
// The session-owner identity passed to EnsureSessionForAgentWithCreation is,
|
|
// by default, the task's runner seat (dbTask.AssigneeAgentProfileID) — even
|
|
// for a reviewer/approver run whose agent differs from the runner, which
|
|
// wrongly binds that run's session (and later its decisions) to the runner's
|
|
// identity. When features.officeSessionIdentity is on, officeAgentProfileID
|
|
// (the run's own agent, captured by the caller before step/routing overrides
|
|
// mutate agentProfileID) is used instead so each participant agent gets its
|
|
// own session per task.
|
|
func (s *Service) createStartSession(
|
|
ctx context.Context, task *v1.Task,
|
|
agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID string,
|
|
) (string, bool, error) {
|
|
return s.createStartSessionWithWorkflowRoute(ctx, task, agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID, nil)
|
|
}
|
|
|
|
func (s *Service) createStartSessionWithWorkflowRoute(
|
|
ctx context.Context, task *v1.Task,
|
|
agentProfileID, officeAgentProfileID, executorID, executorProfileID, workflowStepID string,
|
|
workflowRoute *models.WorkflowSessionRoute,
|
|
) (string, bool, error) {
|
|
if workflowRoute != nil {
|
|
if err := s.workflowRouteMutationAllowed(ctx, task.ID); err != nil {
|
|
return "", false, err
|
|
}
|
|
}
|
|
dbTask, err := s.repo.GetTask(ctx, task.ID)
|
|
if err != nil || dbTask == nil || !dbTask.IsFromOffice {
|
|
var sessionID string
|
|
var prepareErr error
|
|
if workflowRoute != nil {
|
|
sessionID, prepareErr = s.executor.PrepareSessionWithWorkflowRoute(ctx, task, agentProfileID, executorID, executorProfileID, workflowStepID, workflowRoute)
|
|
} else {
|
|
sessionID, prepareErr = s.executor.PrepareSession(ctx, task, agentProfileID, executorID, executorProfileID, workflowStepID)
|
|
}
|
|
return sessionID, prepareErr == nil, prepareErr
|
|
}
|
|
|
|
sessionOwnerID := s.officeSessionOwnerID(dbTask, agentProfileID, officeAgentProfileID)
|
|
if sessionOwnerID == "" {
|
|
var sessionID string
|
|
var prepareErr error
|
|
if workflowRoute != nil {
|
|
sessionID, prepareErr = s.executor.PrepareSessionWithWorkflowRoute(ctx, task, agentProfileID, executorID, executorProfileID, workflowStepID, workflowRoute)
|
|
} else {
|
|
sessionID, prepareErr = s.executor.PrepareSession(ctx, task, agentProfileID, executorID, executorProfileID, workflowStepID)
|
|
}
|
|
return sessionID, prepareErr == nil, prepareErr
|
|
}
|
|
if workflowRoute != nil {
|
|
// Office sessions are identity-owned and may already exist. The route
|
|
// promotion path handles an exact selected row; a fresh Office session
|
|
// is not created through the generic PrepareSession transaction.
|
|
return "", false, fmt.Errorf("workflow session routes are not supported for new office sessions")
|
|
}
|
|
session, created, ensureErr := s.executor.EnsureSessionForAgentWithCreation(
|
|
ctx, task, sessionOwnerID, agentProfileID, executorID, executorProfileID,
|
|
)
|
|
if ensureErr != nil {
|
|
return "", false, ensureErr
|
|
}
|
|
return session.ID, created, nil
|
|
}
|
|
|
|
// officeSessionOwnerID selects the stable identity for an Office session. A
|
|
// projected runner wins by default. When identity separation is enabled, a
|
|
// scheduler-captured participant identity wins for assigned runs. Before a
|
|
// runner seat is projected, the captured identity or execution profile keeps
|
|
// the session on the Office path instead of creating a generic workspace row.
|
|
func (s *Service) officeSessionOwnerID(task *models.Task, agentProfileID, officeAgentProfileID string) string {
|
|
if task.AssigneeAgentProfileID == "" {
|
|
if officeAgentProfileID != "" {
|
|
return officeAgentProfileID
|
|
}
|
|
return agentProfileID
|
|
}
|
|
if s.config.OfficeSessionIdentity && officeAgentProfileID != "" {
|
|
return officeAgentProfileID
|
|
}
|
|
return task.AssigneeAgentProfileID
|
|
}
|
|
|
|
// moveTaskToWorkflowStep moves a task to the target workflow step if provided and different from current.
|
|
func (s *Service) moveTaskToWorkflowStep(ctx context.Context, taskID, workflowStepID string) {
|
|
if workflowStepID == "" {
|
|
return
|
|
}
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil || dbTask.WorkflowStepID == workflowStepID {
|
|
return
|
|
}
|
|
dbTask.WorkflowStepID = workflowStepID
|
|
dbTask.UpdatedAt = time.Now().UTC()
|
|
if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, dbTask); err != nil {
|
|
s.logger.Warn("failed to move task to workflow step",
|
|
zap.String("task_id", taskID),
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
s.publishTaskUpdated(ctx, dbTask)
|
|
}
|
|
|
|
// resolveEffectiveAgentProfile checks whether the task's workflow step overrides
|
|
// the agent profile. If the step (or workflow default) specifies a different
|
|
// profile, that profile is returned instead of the caller-provided one.
|
|
// This ensures the initial task start uses the step's agent — not just the
|
|
// workspace default the frontend sends.
|
|
func (s *Service) resolveEffectiveAgentProfile(ctx context.Context, taskID, workflowStepID, callerProfileID string) (string, error) {
|
|
if s.workflowStepGetter == nil {
|
|
s.logger.Debug("resolveEffectiveAgentProfile: no workflowStepGetter, using caller profile",
|
|
zap.String("task_id", taskID),
|
|
zap.String("caller_profile", callerProfileID))
|
|
return callerProfileID, nil
|
|
}
|
|
|
|
// Determine the effective step ID: explicit param > task's current step.
|
|
effectiveStepID := workflowStepID
|
|
if effectiveStepID == "" {
|
|
if s.repo == nil {
|
|
return "", fmt.Errorf("task repository unavailable while resolving workflow profile for task %q", taskID)
|
|
}
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return "", fmt.Errorf("load task %q while resolving effective workflow profile: %w", taskID, err)
|
|
}
|
|
if dbTask == nil {
|
|
return "", fmt.Errorf("task %q not found while resolving effective workflow profile", taskID)
|
|
}
|
|
s.logger.Debug("resolveEffectiveAgentProfile: loaded task from DB",
|
|
zap.String("task_id", taskID),
|
|
zap.String("db_workflow_step_id", dbTask.WorkflowStepID))
|
|
if dbTask.WorkflowStepID == "" {
|
|
s.logger.Debug("resolveEffectiveAgentProfile: task has no workflow step, using caller profile",
|
|
zap.String("task_id", taskID))
|
|
return callerProfileID, nil
|
|
}
|
|
effectiveStepID = dbTask.WorkflowStepID
|
|
}
|
|
|
|
step, err := s.workflowStepGetter.GetStep(ctx, effectiveStepID)
|
|
if err != nil {
|
|
return "", fmt.Errorf("load workflow step %q while resolving effective workflow profile: %w", effectiveStepID, err)
|
|
}
|
|
if step == nil {
|
|
s.logger.Debug("resolveEffectiveAgentProfile: workflow step not found, using caller profile",
|
|
zap.String("task_id", taskID),
|
|
zap.String("step_id", effectiveStepID),
|
|
)
|
|
return callerProfileID, nil
|
|
}
|
|
|
|
s.logger.Debug("resolveEffectiveAgentProfile: loaded step",
|
|
zap.String("task_id", taskID),
|
|
zap.String("step_id", effectiveStepID),
|
|
zap.String("step_name", step.Name),
|
|
zap.String("step_agent_profile_id", step.AgentProfileID),
|
|
zap.String("step_workflow_id", step.WorkflowID))
|
|
|
|
stepProfile, err := s.resolveStepAgentProfileForTaskID(ctx, taskID, step)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
s.logger.Debug("resolveEffectiveAgentProfile: resolved step profile",
|
|
zap.String("task_id", taskID),
|
|
zap.String("step_profile", stepProfile),
|
|
zap.String("caller_profile", callerProfileID))
|
|
|
|
if stepProfile == "" || stepProfile == callerProfileID {
|
|
return callerProfileID, nil
|
|
}
|
|
|
|
s.logger.Info("overriding agent profile with workflow step profile",
|
|
zap.String("task_id", taskID),
|
|
zap.String("step_id", effectiveStepID),
|
|
zap.String("step_name", step.Name),
|
|
zap.String("caller_profile", callerProfileID),
|
|
zap.String("step_profile", stepProfile))
|
|
return stepProfile, nil
|
|
}
|
|
|
|
// postLaunchStart records the initial message and sets plan mode after a successful launch.
|
|
func (s *Service) postLaunchStart(ctx context.Context, taskID string, execution *executor.TaskExecution, prompt string, recordPlanMode, setPlanMode, autoStart bool, attachments []v1.MessageAttachment) {
|
|
if execution.SessionID != "" {
|
|
s.recordInitialMessage(ctx, taskID, execution.SessionID, prompt, recordPlanMode, autoStart, attachments)
|
|
|
|
if setPlanMode {
|
|
session, err := s.repo.GetTaskSession(ctx, execution.SessionID)
|
|
if err == nil {
|
|
s.setSessionPlanMode(ctx, session, true)
|
|
}
|
|
}
|
|
|
|
// Persist prepare_result using SetSessionMetadataKey (json_set) which
|
|
// atomically sets ONE key without touching others. This avoids the
|
|
// read-modify-write race where UpdateSessionMetadata clobbers plan_mode.
|
|
if execution.PrepareResult != nil && execution.PrepareResult.Success {
|
|
pr := lifecycle.SerializePrepareResult(execution.PrepareResult)
|
|
if err := s.repo.SetSessionMetadataKey(ctx, execution.SessionID, "prepare_result", pr); err != nil {
|
|
s.logger.Warn("failed to persist prepare_result",
|
|
zap.String("session_id", execution.SessionID), zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
go s.ensureSessionPRWatch(context.Background(), taskID, execution.SessionID, execution.WorktreeBranch)
|
|
}
|
|
|
|
// applyWorkflowAndPlanMode applies workflow step configuration and plan mode injection to a prompt.
|
|
// Returns the effective prompt and whether plan mode is active (from either the step or the caller).
|
|
// For ephemeral tasks (quick chat), workflow step processing is skipped since they have no workflow.
|
|
func (s *Service) applyWorkflowAndPlanMode(
|
|
ctx context.Context,
|
|
prompt string,
|
|
taskID string,
|
|
sessionID string,
|
|
workflowStepID string,
|
|
planMode bool,
|
|
isEphemeral bool,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
) (string, bool, string) {
|
|
return s.applyWorkflowAndPlanModeWithPromptContext(
|
|
ctx, prompt, taskID, sessionID, workflowStepID, planMode,
|
|
isEphemeral, isPassthrough, skipStepPrompt, "",
|
|
)
|
|
}
|
|
|
|
// applyWorkflowAndPlanModeWithPromptContext composes a workflow prompt when
|
|
// available and otherwise prepares the launch prompt directly. A direct
|
|
// message already has canonical reference content, so resolving it again
|
|
// would make persistence and ACP dispatch depend on mutable prompt records.
|
|
// Workflow-only prompts still use the normal expansion path.
|
|
func (s *Service) applyWorkflowAndPlanModeWithPromptContext(
|
|
ctx context.Context,
|
|
prompt string,
|
|
taskID string,
|
|
sessionID string,
|
|
workflowStepID string,
|
|
planMode bool,
|
|
isEphemeral bool,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
trustedPromptContext string,
|
|
) (string, bool, string) {
|
|
return s.applyWorkflowAndPlanModeWithPromptContextOptions(
|
|
ctx, prompt, taskID, sessionID, workflowStepID, planMode,
|
|
isEphemeral, isPassthrough, skipStepPrompt, trustedPromptContext, false, false,
|
|
)
|
|
}
|
|
|
|
func (s *Service) applyWorkflowAndPlanModeWithPromptContextOptions(
|
|
ctx context.Context,
|
|
prompt string,
|
|
taskID string,
|
|
sessionID string,
|
|
workflowStepID string,
|
|
planMode bool,
|
|
isEphemeral bool,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
trustedPromptContext string,
|
|
promptReferencesPrepared bool,
|
|
preserveDirectPrompt bool,
|
|
) (string, bool, string) {
|
|
effectivePrompt := prompt
|
|
promptReferenceContext := ""
|
|
if isPassthrough && trustedPromptContext != "" {
|
|
trustedBlock := sysprompt.Wrap(trustedPromptContext)
|
|
if strings.Contains(effectivePrompt, trustedBlock) {
|
|
effectivePrompt = strings.TrimSpace(strings.ReplaceAll(effectivePrompt, trustedBlock, ""))
|
|
}
|
|
// Dynamic routing can switch a prepared structured launch to a
|
|
// passthrough profile. Hidden saved-prompt context must not reach its PTY.
|
|
trustedPromptContext = ""
|
|
promptReferenceContext = ""
|
|
}
|
|
|
|
stepHasPlanMode := false
|
|
workflowPromptComposed := false
|
|
// Skip workflow step prompt injection for ephemeral tasks - they don't have workflows
|
|
if !isEphemeral && workflowStepID != "" && s.workflowStepGetter != nil {
|
|
step, err := s.workflowStepGetter.GetStep(ctx, workflowStepID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to get workflow step for prompt building",
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.Error(err))
|
|
} else if step != nil {
|
|
workflowPromptComposed = true
|
|
stepHasPlanMode = step.HasOnEnterAction(wfmodels.OnEnterEnablePlanMode)
|
|
effectivePrompt, promptReferenceContext = s.buildWorkflowStepPrompt(
|
|
ctx, effectivePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt,
|
|
trustedPromptContext, promptReferencesPrepared, preserveDirectPrompt,
|
|
)
|
|
}
|
|
}
|
|
if !workflowPromptComposed {
|
|
effectivePrompt, promptReferenceContext = s.applyDirectPromptReferenceContext(
|
|
ctx, effectivePrompt, isPassthrough, trustedPromptContext, promptReferencesPrepared,
|
|
)
|
|
}
|
|
|
|
if planMode && !stepHasPlanMode {
|
|
var parts []string
|
|
parts = append(parts, sysprompt.Wrap(sysprompt.DefaultPlanPrefix()))
|
|
parts = append(parts, effectivePrompt)
|
|
effectivePrompt = strings.Join(parts, "\n\n")
|
|
}
|
|
|
|
return effectivePrompt, planMode || stepHasPlanMode, promptReferenceContext
|
|
}
|
|
|
|
func (s *Service) applyDirectPromptReferenceContext(
|
|
ctx context.Context,
|
|
prompt string,
|
|
isPassthrough bool,
|
|
trustedPromptContext string,
|
|
promptReferencesPrepared bool,
|
|
) (string, string) {
|
|
if trustedPromptContext != "" {
|
|
trustedBlock := sysprompt.Wrap(trustedPromptContext)
|
|
if !strings.Contains(prompt, trustedBlock) {
|
|
prompt += "\n\n" + trustedBlock
|
|
}
|
|
return prompt, trustedPromptContext
|
|
}
|
|
if promptReferencesPrepared {
|
|
// An accepted empty snapshot is authoritative. Re-expanding here
|
|
// would observe saved-prompt definitions changed after admission.
|
|
return prompt, ""
|
|
}
|
|
return s.expandPromptReferencesWithContext(ctx, prompt, isPassthrough)
|
|
}
|
|
|
|
func (s *Service) buildWorkflowStepPrompt(
|
|
ctx context.Context,
|
|
basePrompt string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID string,
|
|
sessionID string,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
trustedPromptContext string,
|
|
promptReferencesPrepared bool,
|
|
preserveDirectPrompt bool,
|
|
) (string, string) {
|
|
if trustedPromptContext != "" {
|
|
return s.buildWorkflowPromptWithTrustedContextOptions(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt,
|
|
trustedPromptContext, preserveDirectPrompt, promptReferencesPrepared,
|
|
)
|
|
}
|
|
return s.buildWorkflowPromptWithContextOptions(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt, preserveDirectPrompt,
|
|
promptReferencesPrepared,
|
|
)
|
|
}
|
|
|
|
// backfillInitialUserMessageIfMissing records the task's description as the
|
|
// first user message when the session has no messages at all. This covers the
|
|
// edge case where the initial launch failed before recordInitialMessage was
|
|
// called (postLaunchStart only runs after a successful LaunchAgent), leaving
|
|
// the chat empty even though the task carries the prompt the user originally
|
|
// typed.
|
|
//
|
|
// The check requires *zero* messages, not just zero user messages: if any
|
|
// agent output already exists from a partial prior run, the backfilled
|
|
// message would land at the bottom of the chat (CreateMessage stamps
|
|
// CreatedAt=now), which is worse than leaving the chat alone.
|
|
func (s *Service) backfillInitialUserMessageIfMissing(ctx context.Context, taskID, sessionID, prompt string) {
|
|
if prompt == "" || s.messageCreator == nil {
|
|
return
|
|
}
|
|
msgs, err := s.repo.ListMessages(ctx, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("backfill initial message: list messages failed",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if len(msgs) > 0 {
|
|
return
|
|
}
|
|
s.recordInitialMessage(ctx, taskID, sessionID, prompt, false, false, nil)
|
|
}
|
|
|
|
// recordInitialMessage creates the initial user message and updates session state after launch.
|
|
// autoStart marks the message as having been created by an automated trigger
|
|
// (workflow auto-start, PR/issue watch, Jira/Linear integration) so cleanup
|
|
// logic can distinguish "agent ran on its own" from "user actually engaged".
|
|
func (s *Service) recordInitialMessage(ctx context.Context, taskID, sessionID, prompt string, planModeActive, autoStart bool, attachments []v1.MessageAttachment) {
|
|
if s.messageCreator != nil && (prompt != "" || len(attachments) > 0) {
|
|
meta := NewUserMessageMeta().WithPlanMode(planModeActive).WithAutoStart(autoStart).WithAttachments(attachments)
|
|
if err := s.messageCreator.CreateUserMessage(ctx, taskID, prompt, sessionID, s.getActiveTurnID(sessionID), meta.ToMap()); err != nil {
|
|
s.logger.Error("failed to create initial user message",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// buildWorkflowPrompt constructs the effective prompt using workflow step configuration.
|
|
// If step.Prompt contains {{task_prompt}}, it is replaced with the base prompt.
|
|
// Otherwise, step.Prompt fully replaces the base prompt.
|
|
// If the step has enable_plan_mode in on_enter events, plan mode prefix is also prepended.
|
|
// Only true internal instructions are wrapped in <kandev-system> tags so they can be stripped from the visible chat.
|
|
func (s *Service) buildWorkflowPrompt(ctx context.Context, basePrompt string, step *wfmodels.WorkflowStep, taskID string, sessionID string, isPassthrough bool) string {
|
|
prompt, _ := s.buildWorkflowPromptWithContext(ctx, basePrompt, step, taskID, sessionID, isPassthrough, false)
|
|
return prompt
|
|
}
|
|
|
|
// buildWorkflowEntryPrompt applies the task-description fallback for one
|
|
// workflow entry. Empty steps use the description only when this call
|
|
// atomically claims the session's first prompt slot; non-empty step prompts
|
|
// keep their existing placeholder and replacement semantics.
|
|
func (s *Service) buildWorkflowEntryPrompt(
|
|
ctx context.Context,
|
|
taskDescription string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID, sessionID string,
|
|
isPassthrough bool,
|
|
) (string, string, error) {
|
|
basePrompt := taskDescription
|
|
if step.Prompt == "" && strings.TrimSpace(taskDescription) != "" {
|
|
claimed, err := s.repo.ClaimInitialPromptFallback(ctx, sessionID)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("failed to claim workflow prompt fallback: %w", err)
|
|
}
|
|
if !claimed {
|
|
basePrompt = ""
|
|
}
|
|
}
|
|
// A replacement session is intentionally a fresh prompt boundary. It has
|
|
// no prior claim, so the task description is eligible again after a reused
|
|
// session terminalizes during workflow entry.
|
|
prompt, promptReferenceContext := s.buildWorkflowPromptWithContext(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, false,
|
|
)
|
|
return prompt, promptReferenceContext, nil
|
|
}
|
|
|
|
// workflowInstructionsHeading/End are stable, agent-facing markers for the
|
|
// optional workflow-level prompt block. Chat collapses everything between
|
|
// them by default. Do not i18n (sent to the model, same as step prompt English).
|
|
// The end marker is required so multi-paragraph workflow prompts do not break
|
|
// the frontend split (a first-blank-line heuristic would cut mid-body).
|
|
const (
|
|
workflowInstructionsHeading = "## Workflow instructions"
|
|
workflowInstructionsEnd = "<!-- /workflow-instructions -->"
|
|
)
|
|
|
|
func (s *Service) buildWorkflowPromptWithContext(
|
|
ctx context.Context,
|
|
basePrompt string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID string,
|
|
sessionID string,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
) (string, string) {
|
|
return s.buildWorkflowPromptWithContextOptions(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt, false,
|
|
false,
|
|
)
|
|
}
|
|
|
|
func (s *Service) buildWorkflowPromptWithContextOptions(
|
|
ctx context.Context,
|
|
basePrompt string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID string,
|
|
sessionID string,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
preserveDirectPrompt bool,
|
|
promptReferencesPrepared bool,
|
|
) (string, string) {
|
|
return s.buildWorkflowPromptWithTrustedContextOptions(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt, "",
|
|
preserveDirectPrompt, promptReferencesPrepared,
|
|
)
|
|
}
|
|
|
|
// buildWorkflowPromptWithTrustedContext composes a workflow prompt without
|
|
// re-expanding an already canonical direct message. If the workflow step
|
|
// replaces the base prompt, it restores the exact trusted block so the
|
|
// accepted saved-prompt context remains available to the agent.
|
|
func (s *Service) buildWorkflowPromptWithTrustedContext(
|
|
ctx context.Context,
|
|
basePrompt string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID string,
|
|
sessionID string,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
trustedPromptContext string,
|
|
) (string, string) {
|
|
return s.buildWorkflowPromptWithTrustedContextOptions(
|
|
ctx, basePrompt, step, taskID, sessionID, isPassthrough, skipStepPrompt,
|
|
trustedPromptContext, false, false,
|
|
)
|
|
}
|
|
|
|
func (s *Service) buildWorkflowPromptWithTrustedContextOptions(
|
|
ctx context.Context,
|
|
basePrompt string,
|
|
step *wfmodels.WorkflowStep,
|
|
taskID string,
|
|
sessionID string,
|
|
isPassthrough bool,
|
|
skipStepPrompt bool,
|
|
trustedPromptContext string,
|
|
preserveDirectPrompt bool,
|
|
promptReferencesPrepared bool,
|
|
) (string, string) {
|
|
_ = sessionID
|
|
var parts []string
|
|
|
|
if block := s.workflowInstructionsBlock(ctx, step, taskID); block != "" {
|
|
parts = append(parts, block)
|
|
}
|
|
|
|
// skip_step_prompt suppresses the step prompt and its task-description
|
|
// fallback for this one entry; only the workflow-level block above (and any
|
|
// one-time move instructions appended by the caller) remain.
|
|
if !skipStepPrompt {
|
|
// {step_entry_number} is resolved against the step's own template before
|
|
// {{task_prompt}} substitution, so a literal token inside basePrompt (task
|
|
// description / direct message) is never treated as an interpolation
|
|
// target. The step is copied rather than mutated in place because it may
|
|
// be a cached/shared *wfmodels.WorkflowStep.
|
|
interpolatedStep := step
|
|
if interpolated := s.interpolateStepEntryNumberIfPresent(ctx, step.Prompt, taskID, step.ID); interpolated != step.Prompt {
|
|
stepCopy := *step
|
|
stepCopy.Prompt = interpolated
|
|
interpolatedStep = &stepCopy
|
|
}
|
|
parts = append(parts, stepPromptBodyWithOptions(interpolatedStep, taskID, basePrompt, preserveDirectPrompt))
|
|
}
|
|
|
|
joined := strings.Join(parts, "\n\n")
|
|
if trustedPromptContext != "" {
|
|
trustedBlock := sysprompt.Wrap(trustedPromptContext)
|
|
if !strings.Contains(joined, trustedBlock) {
|
|
joined += "\n\n" + trustedBlock
|
|
}
|
|
return joined, trustedPromptContext
|
|
}
|
|
if promptReferencesPrepared {
|
|
return joined, ""
|
|
}
|
|
return s.expandPromptReferencesWithContext(ctx, joined, isPassthrough)
|
|
}
|
|
|
|
// stepPromptBody renders the visible step prompt for one entry: the step's
|
|
// prompt template with {{task_prompt}} resolved to basePrompt, a step prompt
|
|
// without that placeholder used verbatim, or the base prompt when the step has
|
|
// no prompt of its own.
|
|
func stepPromptBody(step *wfmodels.WorkflowStep, taskID, basePrompt string) string {
|
|
return stepPromptBodyWithOptions(step, taskID, basePrompt, false)
|
|
}
|
|
|
|
func stepPromptBodyWithOptions(step *wfmodels.WorkflowStep, taskID, basePrompt string, preserveDirectPrompt bool) string {
|
|
if step.Prompt == "" {
|
|
return basePrompt
|
|
}
|
|
interpolatedPrompt := sysprompt.InterpolatePlaceholders(step.Prompt, taskID)
|
|
if strings.Contains(interpolatedPrompt, "{{task_prompt}}") {
|
|
return strings.Replace(interpolatedPrompt, "{{task_prompt}}", basePrompt, 1)
|
|
}
|
|
if preserveDirectPrompt && strings.TrimSpace(basePrompt) != "" {
|
|
return interpolatedPrompt + "\n\n" + basePrompt
|
|
}
|
|
// A step prompt without {{task_prompt}} is treated as the full visible prompt.
|
|
return interpolatedPrompt
|
|
}
|
|
|
|
// workflowInstructionsBlock returns the visible "## Workflow instructions"
|
|
// section when the step's workflow has a non-empty prompt. Empty/whitespace
|
|
// prompts and missing getters/workflows omit the section entirely.
|
|
func (s *Service) workflowInstructionsBlock(ctx context.Context, step *wfmodels.WorkflowStep, taskID string) string {
|
|
if s.workflowStepGetter == nil || step == nil || step.WorkflowID == "" {
|
|
return ""
|
|
}
|
|
meta, err := s.getWorkflowMeta(ctx, step.WorkflowID)
|
|
if err != nil {
|
|
if s.logger != nil {
|
|
s.logger.Warn("failed to get workflow prompt for prompt building",
|
|
zap.String("workflow_id", step.WorkflowID),
|
|
zap.Error(err))
|
|
}
|
|
return ""
|
|
}
|
|
prompt := strings.TrimSpace(meta.Prompt)
|
|
if prompt == "" {
|
|
return ""
|
|
}
|
|
interpolated := sysprompt.InterpolatePlaceholders(prompt, taskID)
|
|
interpolated = s.interpolateStepEntryNumberIfPresent(ctx, interpolated, taskID, step.ID)
|
|
interpolated = strings.TrimSpace(interpolated)
|
|
if interpolated == "" {
|
|
return ""
|
|
}
|
|
// Drop any accidental end-marker text from user content so chat split
|
|
// cannot cut the block early (frontend also prefers the final marker).
|
|
interpolated = strings.ReplaceAll(interpolated, workflowInstructionsEnd, "")
|
|
interpolated = strings.TrimSpace(interpolated)
|
|
if interpolated == "" {
|
|
return ""
|
|
}
|
|
return workflowInstructionsHeading + "\n\n" + interpolated + "\n\n" + workflowInstructionsEnd
|
|
}
|
|
|
|
// stepEntryNumberToken is the exact literal REQ-TWS-001 substitutes in
|
|
// workflow prompt templates.
|
|
const stepEntryNumberToken = "{step_entry_number}"
|
|
|
|
// stepEntryNumber resolves the 1-based ordinal of the task's current entry
|
|
// into stepID from the append-only task_step_transitions ledger, floored at
|
|
// 1: a task whose prompt is being built has entered the step at least once,
|
|
// so 0 recorded rows (a zero-row task, or an empty taskID/stepID) means the
|
|
// ledger under-counts, not that the task never entered. The result is
|
|
// therefore a lower bound on the true entry count for a task whose history
|
|
// predates the ledger's first row (2026-08-16).
|
|
func (s *Service) stepEntryNumber(ctx context.Context, taskID, stepID string) (int, error) {
|
|
if taskID == "" || stepID == "" {
|
|
return 1, nil
|
|
}
|
|
count, err := s.repo.CountStepEntries(ctx, taskID, stepID)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if count < 1 {
|
|
return 1, nil
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// interpolateStepEntryNumberIfPresent substitutes every occurrence of
|
|
// {step_entry_number} in template, issuing the count query only when the
|
|
// token is present (NFR-1: a template that does not ask must not pay). A
|
|
// count-query failure leaves the token verbatim and logs at warn rather than
|
|
// failing prompt building, so an un-migrated prompt degrades visibly instead
|
|
// of rendering an invented number.
|
|
func (s *Service) interpolateStepEntryNumberIfPresent(ctx context.Context, template, taskID, stepID string) string {
|
|
if !strings.Contains(template, stepEntryNumberToken) {
|
|
return template
|
|
}
|
|
entryNumber, err := s.stepEntryNumber(ctx, taskID, stepID)
|
|
if err != nil {
|
|
if s.logger != nil {
|
|
s.logger.Warn("failed to compute step entry number for prompt interpolation",
|
|
zap.String("task_id", taskID),
|
|
zap.String("step_id", stepID),
|
|
zap.Error(err))
|
|
}
|
|
return template
|
|
}
|
|
return sysprompt.InterpolateStepEntryNumber(template, entryNumber)
|
|
}
|
|
|
|
// expandPromptReferences resolves "@name" saved-prompt references in prompt
|
|
// via the configured PromptReferenceExpander. When no expander is set, prompt
|
|
// is returned unchanged. Passthrough sessions skip expansion entirely: the
|
|
// prompt is written raw to a PTY with no <kandev-system> stripping step, so
|
|
// the expansion's hidden wrapper block would be typed into the terminal
|
|
// verbatim instead of staying hidden.
|
|
func (s *Service) expandPromptReferences(ctx context.Context, prompt string, isPassthrough bool) string {
|
|
expanded, _ := s.expandPromptReferencesWithContext(ctx, prompt, isPassthrough)
|
|
return expanded
|
|
}
|
|
|
|
func (s *Service) expandPromptReferencesWithContext(
|
|
ctx context.Context,
|
|
prompt string,
|
|
isPassthrough bool,
|
|
) (string, string) {
|
|
if s.promptExpander == nil || isPassthrough {
|
|
return prompt, ""
|
|
}
|
|
var zapLogger *zap.Logger
|
|
if s.logger != nil {
|
|
zapLogger = s.logger.Zap()
|
|
}
|
|
return s.promptExpander.AppendReferenceExpansionsWithContext(ctx, prompt, zapLogger)
|
|
}
|
|
|
|
// PrepareDirectPrompt applies the same backend-owned saved-prompt expansion
|
|
// used by workflow prompts to a direct user message. Message handlers call it
|
|
// before persistence so the stored content and the first dispatched prompt
|
|
// share one canonical representation.
|
|
func (s *Service) PrepareDirectPrompt(
|
|
ctx context.Context,
|
|
prompt string,
|
|
isPassthrough bool,
|
|
) (string, string) {
|
|
return s.expandPromptReferencesWithContext(ctx, prompt, isPassthrough)
|
|
}
|
|
|
|
// ResumeTaskSession restarts a specific task session using its stored worktree.
|
|
func (s *Service) ResumeTaskSession(ctx context.Context, taskID, sessionID string) (*executor.TaskExecution, error) {
|
|
return s.ResumeTaskSessionWithOptions(ctx, taskID, sessionID, executor.ResumeOptions{})
|
|
}
|
|
|
|
// ResumeTaskSessionWithOptions restarts a task session with an explicit
|
|
// recovery permission. The default ResumeTaskSession path remains unchanged.
|
|
func (s *Service) ResumeTaskSessionWithOptions(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
options executor.ResumeOptions,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.resumeTaskSessionWithContinuation(ctx, taskID, sessionID, options, nil)
|
|
}
|
|
|
|
// ResumeTaskSessionAndPrompt keeps one recovery attempt alive from the initial
|
|
// resume through the prompt provider acceptance boundary. The message handler
|
|
// uses this compound operation for an internal retry so cancellation cannot
|
|
// settle the session between ResumeTaskSession and PromptTask.
|
|
func (s *Service) ResumeTaskSessionAndPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt, model string,
|
|
planMode bool,
|
|
attachments []v1.MessageAttachment,
|
|
) (*PromptResult, error) {
|
|
var result *PromptResult
|
|
_, err := s.resumeTaskSessionWithContinuation(
|
|
ctx,
|
|
taskID,
|
|
sessionID,
|
|
executor.ResumeOptions{Origin: string(launchOriginManual)},
|
|
func(resumeCtx context.Context, attempt *resumeAttempt, _ *executor.TaskExecution) error {
|
|
var promptErr error
|
|
result, promptErr = s.promptTask(
|
|
resumeCtx,
|
|
taskID,
|
|
sessionID,
|
|
prompt,
|
|
model,
|
|
planMode,
|
|
attachments,
|
|
false,
|
|
launchOriginManual,
|
|
promptTaskOptions{resumeAttempt: attempt},
|
|
)
|
|
return promptErr
|
|
},
|
|
)
|
|
return result, err
|
|
}
|
|
|
|
//
|
|
//nolint:cyclop,gocognit,funlen // Resume coordinates state validation, launch recovery, and ready-state persistence.
|
|
func (s *Service) resumeTaskSessionWithContinuation(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
options executor.ResumeOptions,
|
|
continuation func(context.Context, *resumeAttempt, *executor.TaskExecution) error,
|
|
) (*executor.TaskExecution, error) {
|
|
entryBinding := ceilingEntryBindingFromContext(ctx)
|
|
s.logger.Debug("resuming task session",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
releaseLifecycleLock := s.acquireSessionLifecycleLock(sessionID)
|
|
defer releaseLifecycleLock()
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if session.TaskID != taskID {
|
|
return nil, fmt.Errorf("task session does not belong to task")
|
|
}
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, entryBinding); err != nil {
|
|
return nil, err
|
|
}
|
|
allowCompletedResume := options.AllowCompletedSessionResume &&
|
|
session.State == models.TaskSessionStateCompleted
|
|
// The completed-session permission is valid only for the exact completed
|
|
// row admitted by the caller. Do not let an option intended for that state
|
|
// alter the ordinary FAILED/CANCELLED recovery paths.
|
|
options.AllowCompletedSessionResume = allowCompletedResume
|
|
// Completed sessions remain closed to every implicit resume path. Check this
|
|
// before looking for an executor row because cleanup commonly removes that
|
|
// row, and the caller should receive the terminal-state rejection rather
|
|
// than an incidental "no executor record" error.
|
|
if session.State == models.TaskSessionStateCompleted && !allowCompletedResume {
|
|
return nil, fmt.Errorf("session is completed and cannot be resumed; create a new session instead")
|
|
}
|
|
running, err := s.repo.GetExecutorRunningBySessionID(ctx, sessionID)
|
|
if (err != nil || running == nil) &&
|
|
session.State != models.TaskSessionStateCancelled &&
|
|
session.State != models.TaskSessionStateFailed &&
|
|
!models.HasInterruptedRecoveryPending(session.Metadata) &&
|
|
!allowCompletedResume {
|
|
// Executor record is required for non-terminal sessions. For cancelled/failed sessions
|
|
// the record may already have been cleaned up before the user clicked Resume — allow it.
|
|
// Explicit completed follow-up has the same cleanup shape and is admitted
|
|
// only through the narrow permission above.
|
|
return nil, fmt.Errorf("session is not resumable: no executor record")
|
|
}
|
|
attempt, owner, err := s.beginResumeAttempt(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !owner {
|
|
if continuation != nil {
|
|
// A compound retry cannot safely hand its prompt to an attempt owned
|
|
// by another caller. The owner may finish and remove the registry
|
|
// entry before this continuation reaches provider admission.
|
|
return nil, fmt.Errorf("%w: recovery is already owned by another caller", ErrResumeAttemptCancelled)
|
|
}
|
|
// The lifecycle lock normally prevents two callers from reaching this
|
|
// branch together. Keep the registry join behavior explicit for callers
|
|
// that entered through a different recovery path: share the completed
|
|
// result instead of launching a second provider execution.
|
|
waitCtx, cancelWait := context.WithTimeout(context.WithoutCancel(ctx), cancellationOperationTTL)
|
|
waitErr := attempt.wait(waitCtx)
|
|
cancelWait()
|
|
if waitErr != nil {
|
|
return nil, waitErr
|
|
}
|
|
if attempt.context().Err() != nil {
|
|
return nil, ErrResumeAttemptCancelled
|
|
}
|
|
if execution, ok := s.executor.GetExecutionBySession(sessionID); ok && execution != nil {
|
|
return execution, nil
|
|
}
|
|
return nil, ErrResumeAttemptCancelled
|
|
}
|
|
defer attempt.finish(s.resumeAttemptStore())
|
|
resumeCtx := cancellableResumeContext(attempt)
|
|
decorateResumeFailure := func(failure error) error {
|
|
return s.withSessionRecoveryFailureIdentity(
|
|
resumeCtx, taskID, sessionID, attempt, attempt.execution(), failure,
|
|
)
|
|
}
|
|
s.noteMissingWorktreesBeforeResume(sessionID, session)
|
|
branchRecoveryBefore := s.captureBranchRecoveryBeforeResume(resumeCtx, taskID, sessionID, options)
|
|
persistBranchRecovery := func() {
|
|
if options.AllowBranchReplacement {
|
|
s.persistBranchRecoveryWarnings(resumeCtx, taskID, sessionID, branchRecoveryBefore)
|
|
}
|
|
}
|
|
|
|
isOfficeTask, err := s.lookupOfficeTask(resumeCtx, taskID)
|
|
if err != nil {
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
return nil, decorateResumeFailure(fmt.Errorf("failed to determine office task status: %w", err))
|
|
}
|
|
if isOfficeTask {
|
|
return nil, decorateResumeFailure(errOfficeTaskResumeRequiresScheduler)
|
|
}
|
|
admissionCtx := ctx
|
|
releaseAdmission := func() {}
|
|
if isSessionOpenRecoveryContext(ctx) {
|
|
admissionCtx, releaseAdmission = s.lockCeilingEntryAdmission(ctx, taskID)
|
|
}
|
|
seam4Res, deferred, err := func() (*sessionKeyedCeilingReservation, bool, error) {
|
|
defer releaseAdmission()
|
|
if isSessionOpenRecoveryContext(ctx) {
|
|
if reason := s.sessionOpenRecoveryBlockReason(admissionCtx, taskID, session); reason != "" {
|
|
return nil, false, &sessionOpenRecoveryBlockedError{reason: reason}
|
|
}
|
|
}
|
|
return s.admitOrDeferSeam4(admissionCtx, taskID, sessionID, launchOrigin(options.Origin),
|
|
seam4ResumePayloadWithBinding(sessionID, options, entryBinding))
|
|
}()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if deferred {
|
|
return nil, nil
|
|
}
|
|
defer seam4Res.releaseIfNotConsumed()
|
|
s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, seam4Res.manualOverride, seam4Res.population, seam4Res.populationKnown, seam4Res.ceiling)
|
|
|
|
if _, err := s.resolveDynamicLaunchExecution(resumeCtx, session, session.AgentProfileID, true); err != nil {
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
return nil, decorateResumeFailure(err)
|
|
}
|
|
|
|
// Bury any open turns from the previous run before relaunching. Without
|
|
// this, startTurnForSession adopts the orphan on the next prompt and the
|
|
// UI's running timer counts from the orphan's started_at — which can be
|
|
// hours or days ago. Zero-duration completion keeps analytics honest about
|
|
// the dead window. A failure here shouldn't block the resume; the next
|
|
// completeTurnForSession sweep will mop up.
|
|
//
|
|
// Drop the activeTurns cache entry first, mirroring completeTurnForSession.
|
|
// Otherwise a stale entry would let getActiveTurnID return the now-abandoned
|
|
// turn ID without re-reading the DB, tagging new messages to a closed turn.
|
|
if s.turnService != nil {
|
|
s.activeTurns.Delete(sessionID)
|
|
if err := s.turnService.AbandonOpenTurns(resumeCtx, sessionID); err != nil {
|
|
s.logger.Warn("failed to abandon orphan turns on resume; continuing",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
if err := s.validateClaimedCeilingBinding(resumeCtx, taskID, entryBinding); err != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, err
|
|
}
|
|
dispatchCtx, releaseCeilingDispatch, err := s.commitCeilingEntryDispatch(
|
|
resumeCtx, taskID, entryBinding,
|
|
)
|
|
if err != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, err
|
|
}
|
|
execution, err := s.executor.ResumeSessionWithOptions(dispatchCtx, session, true, options)
|
|
releaseCeilingDispatch()
|
|
if execution != nil {
|
|
attempt.setExecutionID(execution.AgentExecutionID)
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
var readySession *models.TaskSession
|
|
if err != nil {
|
|
// If the execution is already running (duplicate resume request), return it as success.
|
|
if errors.Is(err, executor.ErrExecutionAlreadyRunning) {
|
|
execution, readySession, err = s.recoverAlreadyRunningResume(resumeCtx, taskID, sessionID, options)
|
|
if execution != nil {
|
|
attempt.setExecutionID(execution.AgentExecutionID)
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
if err != nil && errors.Is(err, ErrAgentNotReadyForPrompt) {
|
|
persistBranchRecovery()
|
|
return nil, decorateResumeFailure(err)
|
|
}
|
|
}
|
|
if err != nil {
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
// Task was archived while the resume was in flight — return the error
|
|
// without mutating task/session state (archive already handled cleanup).
|
|
// Check both the sentinel (early rejection) and re-read the task to catch
|
|
// the race where archive completed after the executor's archived check.
|
|
if errors.Is(err, executor.ErrTaskArchived) {
|
|
return nil, err
|
|
}
|
|
if task, taskErr := s.repo.GetTask(resumeCtx, taskID); taskErr == nil && task != nil && task.ArchivedAt != nil {
|
|
return nil, executor.ErrTaskArchived
|
|
}
|
|
persistBranchRecovery()
|
|
// Use resumeCtx (WithoutCancel) for the failure-recording writes too —
|
|
// if the caller's ctx was already cancelled (e.g. WS client navigated
|
|
// away), the SessionStateFailed and TaskStateFailed updates would
|
|
// themselves fail with "context canceled" and leave the task stuck
|
|
// looking "running" forever.
|
|
// ResumeSession launches the workspace directly rather than through
|
|
// LaunchPreparedSession. Record this failure with the same state CAS,
|
|
// persisted recovery claim, and archive-safe task CAS as early launch.
|
|
err = s.branchRecoveryError(resumeCtx, taskID, sessionID, err)
|
|
return nil, decorateResumeFailure(s.handleSessionLaunchFailure(
|
|
resumeCtx, taskID, sessionID, err, session,
|
|
))
|
|
}
|
|
}
|
|
if readySession == nil {
|
|
readySession, err = s.waitForResumedSessionReady(resumeCtx, sessionID)
|
|
if err != nil {
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
persistBranchRecovery()
|
|
return nil, decorateResumeFailure(err)
|
|
}
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(attempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(attempt)
|
|
return nil, attemptErr
|
|
}
|
|
execution.SessionState = v1.TaskSessionState(readySession.State)
|
|
seam4Res.consume()
|
|
persistBranchRecovery()
|
|
|
|
// Backfill the initial user message when a prior failed launch never got
|
|
// to recordInitialMessage. Without this, the resume can succeed and the
|
|
// agent starts replying, but the chat shows agent output with no user
|
|
// prompt above it.
|
|
//
|
|
// We use task.Description (the raw user input) rather than the
|
|
// workflow-effective prompt produced by applyWorkflowAndPlanMode. The
|
|
// effective prompt may carry a plan-mode prefix or be templated through a
|
|
// workflow step, but reconstructing the exact prompt the original launch
|
|
// sent to the agent is brittle (workflow state may have advanced since).
|
|
// Surfacing the raw description is intentionally conservative: it shows
|
|
// what the user actually typed, which is what they expect to see in chat.
|
|
if task, taskErr := s.repo.GetTask(resumeCtx, taskID); taskErr != nil {
|
|
s.logger.Warn("resume: failed to load task for initial message backfill",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(taskErr))
|
|
} else if task != nil {
|
|
s.backfillInitialUserMessageIfMissing(resumeCtx, taskID, sessionID, task.Description)
|
|
}
|
|
|
|
s.logger.Debug("task session resumed and ready for input",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
|
|
go s.ensureSessionPRWatch(context.Background(), taskID, execution.SessionID, execution.WorktreeBranch)
|
|
if continuation != nil {
|
|
return execution, decorateResumeFailure(continuation(resumeCtx, attempt, execution))
|
|
}
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
func (s *Service) captureBranchRecoveryBeforeResume(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
options executor.ResumeOptions,
|
|
) *branchRecoverySnapshot {
|
|
if !options.AllowBranchReplacement {
|
|
return nil
|
|
}
|
|
snapshot, err := s.captureBranchRecoverySnapshot(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to capture branch recovery state before resume",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return nil
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func (s *Service) waitForResumedSessionReady(ctx context.Context, sessionID string) (*models.TaskSession, error) {
|
|
return s.waitForSessionAndAgentReady(ctx, sessionID, "after resume")
|
|
}
|
|
|
|
func (s *Service) recoverAlreadyRunningResume(
|
|
resumeCtx context.Context,
|
|
taskID string,
|
|
sessionID string,
|
|
options executor.ResumeOptions,
|
|
) (*executor.TaskExecution, *models.TaskSession, error) {
|
|
existing, ok := s.executor.GetExecutionBySession(sessionID)
|
|
if !ok || existing == nil {
|
|
return nil, nil, executor.ErrExecutionAlreadyRunning
|
|
}
|
|
if err := s.bindExistingResumeAttempt(resumeCtx, sessionID); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
readySession, waitErr := s.waitForResumedSessionReady(resumeCtx, sessionID)
|
|
if waitErr == nil {
|
|
existing.SessionState = v1.TaskSessionState(readySession.State)
|
|
return existing, readySession, nil
|
|
}
|
|
if !errors.Is(waitErr, ErrAgentNotReadyForPrompt) {
|
|
return nil, nil, waitErr
|
|
}
|
|
|
|
if err := s.reapPromptUnreadyExecution(resumeCtx, sessionID, waitErr); err != nil {
|
|
return nil, nil, fmt.Errorf("%w: failed to stop prompt-unready execution: %w", ErrAgentNotReadyForPrompt, err)
|
|
}
|
|
|
|
session, err := s.repo.GetTaskSession(resumeCtx, sessionID)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to reload session after prompt-readiness recovery: %w", err)
|
|
}
|
|
if session.TaskID != taskID {
|
|
return nil, nil, fmt.Errorf("task session does not belong to task")
|
|
}
|
|
|
|
dispatchCtx, releaseCeilingDispatch, err := s.commitCeilingEntryDispatch(
|
|
resumeCtx, taskID, ceilingEntryBindingFromContext(resumeCtx),
|
|
)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
execution, err := s.executor.ResumeSessionWithOptions(dispatchCtx, session, true, options)
|
|
releaseCeilingDispatch()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
readySession, err = s.waitForResumedSessionReady(resumeCtx, sessionID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
execution.SessionState = v1.TaskSessionState(readySession.State)
|
|
return execution, readySession, nil
|
|
}
|
|
|
|
func (s *Service) waitForSessionAndAgentReady(ctx context.Context, sessionID, waitContext string) (*models.TaskSession, error) {
|
|
if err := s.waitForSessionReady(ctx, sessionID); err != nil {
|
|
return nil, fmt.Errorf("session not ready %s: %w", waitContext, err)
|
|
}
|
|
if err := s.waitForAgentPromptReady(ctx, sessionID); err != nil {
|
|
return nil, fmt.Errorf("agent not ready %s: %w", waitContext, err)
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to reload session %s: %w", waitContext, err)
|
|
}
|
|
return session, nil
|
|
}
|
|
|
|
func (s *Service) waitForStartingSessionPromptable(ctx context.Context, taskID, sessionID string) (*models.TaskSession, error) {
|
|
s.logger.Debug("waiting for starting session to become promptable",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
session, err := s.waitForSessionAndAgentReady(ctx, sessionID, "for prompt")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.checkSessionPromptable(taskID, sessionID, session.State); err != nil {
|
|
return nil, err
|
|
}
|
|
return session, nil
|
|
}
|
|
|
|
// StartSessionForWorkflowStep starts an existing session with a workflow step's prompt configuration.
|
|
// If the session is not running, it will be resumed first. Then a prompt is sent using the
|
|
// step's prompt_prefix, prompt_suffix, and plan_mode settings combined with the task description.
|
|
func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessionID, workflowStepID string) error {
|
|
ctx = withWorkflowMetaCache(ctx)
|
|
entryBinding := ceilingEntryBindingFromContext(ctx)
|
|
s.logger.Debug("starting session for workflow step",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("workflow_step_id", workflowStepID))
|
|
|
|
if workflowStepID == "" {
|
|
return fmt.Errorf("workflow_step_id is required")
|
|
}
|
|
if s.workflowStepGetter == nil {
|
|
return fmt.Errorf("workflow step getter not configured")
|
|
}
|
|
|
|
step, err := s.workflowStepGetter.GetStep(ctx, workflowStepID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get workflow step: %w", err)
|
|
}
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if session.TaskID != taskID {
|
|
return fmt.Errorf("session does not belong to task")
|
|
}
|
|
effectiveProfile, err := s.resolveStepAgentProfileForTaskID(ctx, taskID, step)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if effectiveProfile != "" && effectiveProfile != session.AgentProfileID {
|
|
return fmt.Errorf(
|
|
"workflow step profile mismatch: step %q resolves to profile %q but session %q uses profile %q; route the session before prompting",
|
|
workflowStepID,
|
|
effectiveProfile,
|
|
session.ID,
|
|
session.AgentProfileID,
|
|
)
|
|
}
|
|
|
|
dbTask, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get task: %w", err)
|
|
}
|
|
derivedEntryBinding := false
|
|
replayWorkflowStepBinding := entryBinding != nil &&
|
|
ceilingEntryKindFromContext(ctx) == models.CeilingLaunchWorkflowStepEnsure
|
|
if entryBinding == nil {
|
|
if derivedBinding, derived := s.workflowEntryBindingForStep(ctx, taskID, step, sessionID); derived {
|
|
entryBinding = derivedBinding
|
|
ctx = withCeilingEntryBinding(ctx, entryBinding)
|
|
derivedEntryBinding = true
|
|
}
|
|
}
|
|
|
|
if session.ReviewStatus == models.ReviewStatusPending {
|
|
return fmt.Errorf("session is pending approval - use Approve button to proceed or send a message to request changes")
|
|
}
|
|
preConsultRes, refused, err := s.admitOrDeferWorkflowStepEnsureWithBinding(ctx, taskID, sessionID, workflowStepID, entryBinding)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if refused {
|
|
return errSeam3WorkflowStepEnsureDeferred
|
|
}
|
|
defer preConsultRes.releaseIfNotConsumed()
|
|
s.recordManualOverrideIfAdmitted(ctx, taskID, sessionID, preConsultRes.manualOverride, preConsultRes.population, preConsultRes.populationKnown, preConsultRes.ceiling)
|
|
if replayWorkflowStepBinding {
|
|
if err := s.validateClaimedCeilingBinding(ctx, taskID, entryBinding); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
s.advanceTaskWorkflowStep(ctx, dbTask, workflowStepID, session)
|
|
if derivedEntryBinding || replayWorkflowStepBinding {
|
|
// The pre-consultation binding identifies the requested destination while
|
|
// the task still points at its source step. Once admission succeeds, the
|
|
// step transition allocates the committed entry identity. Carry that new
|
|
// identity into the prompt and dispatch guards; otherwise a legitimate
|
|
// transition would look like a successor replacing the very entry it just
|
|
// committed.
|
|
if latestTask, reloadErr := s.repo.GetTask(ctx, taskID); reloadErr == nil && latestTask != nil {
|
|
if latestBinding, bound := s.workflowEntryBindingForStep(ctx, taskID, step, sessionID); bound {
|
|
entryBinding = latestBinding
|
|
ctx = withCeilingEntryBinding(ctx, entryBinding)
|
|
} else {
|
|
entryBinding = nil
|
|
}
|
|
}
|
|
}
|
|
|
|
effectivePrompt, promptReferenceContext, err := s.buildWorkflowEntryPrompt(
|
|
ctx, dbTask.Description, step, taskID, sessionID, session.IsPassthrough,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to build workflow prompt: %w", err)
|
|
}
|
|
|
|
if err := s.ensureSessionRunningWithBinding(ctx, sessionID, session, launchOriginAutomatic, entryBinding); err != nil {
|
|
return err
|
|
}
|
|
preConsultRes.consume()
|
|
|
|
// Apply conditional session settings after a manual resume and before the
|
|
// step prompt. The helper reloads the session so a resume-created runtime
|
|
// state cannot be overwritten by the stale request snapshot.
|
|
s.applyWorkflowSessionConfigOnEnter(ctx, taskID, session, step)
|
|
stepPlanMode := step.HasOnEnterAction(wfmodels.OnEnterEnablePlanMode)
|
|
// The plan/config transforms can make an otherwise empty workflow prompt
|
|
// actionable. Decide whether to dispatch only after applying those same
|
|
// transforms that PromptTask applies downstream.
|
|
session = s.latestSession(ctx, session)
|
|
promptForEmptinessCheck := s.effectivePromptForSession(sessionID, effectivePrompt, stepPlanMode, session)
|
|
if strings.TrimSpace(promptForEmptinessCheck) == "" {
|
|
s.logger.Info("workflow step has no prompt after entry fallback",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("workflow_step_id", workflowStepID))
|
|
return nil
|
|
}
|
|
|
|
// Claimed after the actionability decision above, over content that
|
|
// excludes the handoff text, then appended last. Queue promotion and
|
|
// manual auto-start dispatch once with no replacement launch, so no
|
|
// shared stepHandoffOnce is needed here.
|
|
handoffText, _ := s.claimStepHandoffCarryText(ctx, taskID, workflowStepID)
|
|
effectivePrompt = appendStepHandoffToPrompt(effectivePrompt, handoffText)
|
|
composedLaunchPrompt := appendStepHandoffToPrompt(promptForEmptinessCheck, handoffText)
|
|
|
|
// effectivePrompt is already fully composed for this step entry (handoff
|
|
// included): if promptTask's own internal ErrExecutionNotFound recovery
|
|
// fires, it must reuse this composed prompt rather than recomposing from
|
|
// the destination step's own template and discarding the handoff.
|
|
_, err = s.promptTask(ctx, taskID, sessionID, effectivePrompt, "", stepPlanMode, nil, false, launchOriginAutomatic, promptTaskOptions{
|
|
promptAlreadyComposed: true,
|
|
fallbackLaunchPrompt: composedLaunchPrompt,
|
|
fallbackRetryPrompt: effectivePrompt,
|
|
promptReferenceContext: promptReferenceContext,
|
|
ceilingEntryBinding: entryBinding,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to prompt session: %w", err)
|
|
}
|
|
|
|
s.logger.Info("session started for workflow step",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.String("step_name", step.Name),
|
|
zap.Bool("plan_mode", stepPlanMode))
|
|
|
|
return nil
|
|
}
|
|
|
|
// advanceTaskWorkflowStep updates the task's workflow step and clears session review status if the step changed.
|
|
func (s *Service) advanceTaskWorkflowStep(ctx context.Context, task *models.Task, workflowStepID string, session *models.TaskSession) {
|
|
if task.WorkflowStepID == workflowStepID {
|
|
return
|
|
}
|
|
task.WorkflowStepID = workflowStepID
|
|
task.UpdatedAt = time.Now().UTC()
|
|
if err := s.repo.UpdateTaskPreservingDeferredLaunch(ctx, task); err != nil {
|
|
s.logger.Warn("failed to update task workflow step",
|
|
zap.String("task_id", task.ID),
|
|
zap.String("workflow_step_id", workflowStepID),
|
|
zap.Error(err))
|
|
}
|
|
if session.ReviewStatus != models.ReviewStatusNone {
|
|
if err := s.repo.UpdateSessionReviewStatus(ctx, session.ID, ""); err != nil {
|
|
s.logger.Warn("failed to clear session review status",
|
|
zap.String("session_id", session.ID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// ensureSessionRunning resumes the session if the agent is not actually running.
|
|
// After lazy recovery, a session may be in WAITING_FOR_INPUT with no agent process;
|
|
// this function detects that case and triggers a resume.
|
|
func (s *Service) ensureSessionRunning(ctx context.Context, sessionID string, session *models.TaskSession, origin launchOrigin) error {
|
|
attempt, err := s.ensureSessionRunningWithAttempt(ctx, sessionID, session, origin)
|
|
if attempt != nil {
|
|
attempt.finish(s.resumeAttemptStore())
|
|
}
|
|
return err
|
|
}
|
|
|
|
// ensureSessionRunningWithAttempt is the prompt-owned lazy resume path. A
|
|
// caller that will dispatch immediately keeps the returned attempt alive until
|
|
// provider acceptance, which closes the cancellation gap between readiness and
|
|
// prompt admission.
|
|
func (s *Service) ensureSessionRunningWithAttempt(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
origin launchOrigin,
|
|
) (*resumeAttempt, error) {
|
|
return s.ensureSessionRunningWithAttemptAndBinding(
|
|
ctx, sessionID, session, origin, ceilingEntryBindingFromContext(ctx),
|
|
)
|
|
}
|
|
|
|
func (s *Service) ensureSessionRunningWithBinding(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
origin launchOrigin,
|
|
binding *models.CeilingWorkflowEntryBinding,
|
|
) error {
|
|
attempt, err := s.ensureSessionRunningWithAttemptAndBinding(ctx, sessionID, session, origin, binding)
|
|
if attempt != nil {
|
|
attempt.finish(s.resumeAttemptStore())
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (s *Service) ensureSessionRunningWithAttemptAndBinding(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
origin launchOrigin,
|
|
binding *models.CeilingWorkflowEntryBinding,
|
|
) (*resumeAttempt, error) {
|
|
if binding != nil {
|
|
ctx = withCeilingEntryBinding(ctx, binding)
|
|
if err := s.validateClaimedCeilingBinding(ctx, session.TaskID, binding); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
releaseLifecycleLock := s.acquireSessionLifecycleLock(sessionID)
|
|
defer releaseLifecycleLock()
|
|
|
|
isOfficeTask, err := s.lookupOfficeTask(ctx, session.TaskID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to determine office task status: %w", err)
|
|
}
|
|
|
|
if s.sessionAlreadyPromptReady(ctx, sessionID) {
|
|
return nil, nil
|
|
}
|
|
|
|
// Register before checking readiness so explicit cancellation can interrupt
|
|
// an existing execution that is still booting. A ready execution completes
|
|
// the attempt below and the prompt owner keeps it through provider admission.
|
|
startupAttempt, owner, err := s.beginResumeAttempt(ctx, session.TaskID, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !owner {
|
|
return s.waitForSharedResumeAttempt(ctx, sessionID, startupAttempt)
|
|
}
|
|
return s.runResumeAttempt(ctx, sessionID, session, isOfficeTask, startupAttempt, origin)
|
|
}
|
|
|
|
func (s *Service) sessionAlreadyPromptReady(ctx context.Context, sessionID string) bool {
|
|
// A prompt against an execution that was already prompt-ready does not
|
|
// need a recovery attempt. Avoid registering one so the normal prompt path
|
|
// keeps its existing cancellation behavior. If another resume is already
|
|
// registered, join it instead of bypassing its ownership boundary.
|
|
if _, active := s.resumeAttemptStore().current(sessionID); active {
|
|
return false
|
|
}
|
|
probeCtx := context.WithoutCancel(ctx)
|
|
existing, ok := s.executor.GetExecutionBySession(sessionID)
|
|
return ok && existing != nil &&
|
|
(s.agentManager == nil || s.agentManager.IsAgentReadyForPrompt(probeCtx, sessionID))
|
|
}
|
|
|
|
func (s *Service) waitForSharedResumeAttempt(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
startupAttempt *resumeAttempt,
|
|
) (*resumeAttempt, error) {
|
|
waitCtx, cancelWait := context.WithTimeout(context.WithoutCancel(ctx), cancellationOperationTTL)
|
|
waitErr := startupAttempt.wait(waitCtx)
|
|
cancelWait()
|
|
if waitErr != nil {
|
|
return nil, waitErr
|
|
}
|
|
if startupAttempt.context().Err() != nil {
|
|
return nil, ErrResumeAttemptCancelled
|
|
}
|
|
if execution, ok := s.executor.GetExecutionBySession(sessionID); ok && execution != nil {
|
|
return nil, nil
|
|
}
|
|
return nil, ErrResumeAttemptCancelled
|
|
}
|
|
|
|
func (s *Service) runResumeAttempt(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
isOfficeTask bool,
|
|
startupAttempt *resumeAttempt,
|
|
origin launchOrigin,
|
|
) (*resumeAttempt, error) {
|
|
resumeCtx := cancellableResumeContext(startupAttempt)
|
|
preparedSession, ready, err := s.prepareExistingSessionForResume(
|
|
resumeCtx, sessionID, session, isOfficeTask,
|
|
)
|
|
if err != nil {
|
|
return s.finishResumeAttemptWithError(startupAttempt, err)
|
|
}
|
|
if ready {
|
|
return startupAttempt, nil
|
|
}
|
|
session = preparedSession
|
|
|
|
s.logger.Debug("agent not running for session, attempting resume",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("session_state", string(session.State)))
|
|
if err := s.validateContextCeilingEntry(resumeCtx, session.TaskID); err != nil {
|
|
return s.finishResumeAttemptWithError(startupAttempt, err)
|
|
}
|
|
return s.coldResumeSession(resumeCtx, sessionID, session, isOfficeTask, startupAttempt, origin)
|
|
}
|
|
|
|
func (s *Service) finishResumeAttemptWithError(
|
|
startupAttempt *resumeAttempt,
|
|
err error,
|
|
) (*resumeAttempt, error) {
|
|
if attemptErr := s.validateResumeAttempt(startupAttempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(startupAttempt)
|
|
return startupAttempt, attemptErr
|
|
}
|
|
return startupAttempt, err
|
|
}
|
|
|
|
func (s *Service) prepareExistingSessionForResume(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
isOfficeTask bool,
|
|
) (*models.TaskSession, bool, error) {
|
|
// Check if agent is genuinely running (in-memory execution store, not just DB state).
|
|
if exec, ok := s.executor.GetExecutionBySession(sessionID); !ok || exec == nil {
|
|
return session, false, nil
|
|
}
|
|
if err := s.bindExistingResumeAttempt(ctx, sessionID); err != nil {
|
|
return nil, false, err
|
|
}
|
|
s.recoverAgentPromptStreamIfNeeded(ctx, sessionID)
|
|
readinessErr := s.waitForAgentPromptReady(ctx, sessionID)
|
|
if readinessErr == nil {
|
|
return session, true, nil
|
|
}
|
|
if !errors.Is(readinessErr, ErrAgentNotReadyForPrompt) {
|
|
return nil, false, readinessErr
|
|
}
|
|
if isOfficeTask {
|
|
return nil, false, errOfficeTaskResumeRequiresScheduler
|
|
}
|
|
refreshed, reapErr := s.reapAndReloadSession(ctx, sessionID, readinessErr)
|
|
if reapErr != nil {
|
|
return nil, false, reapErr
|
|
}
|
|
return refreshed, false, nil
|
|
}
|
|
|
|
func (s *Service) bindExistingResumeAttempt(ctx context.Context, sessionID string) error {
|
|
attemptID := executor.ResumeAttemptIDFromContext(ctx)
|
|
if attemptID == "" {
|
|
return nil
|
|
}
|
|
binder, ok := s.agentManager.(resumeAttemptBinder)
|
|
if !ok {
|
|
return fmt.Errorf("agent manager cannot bind resume attempt to existing execution")
|
|
}
|
|
if err := binder.BindResumeAttempt(ctx, sessionID, attemptID); err != nil {
|
|
return fmt.Errorf("failed to bind resume attempt to existing execution: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) coldResumeSession(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
isOfficeTask bool,
|
|
startupAttempt *resumeAttempt,
|
|
origin launchOrigin,
|
|
) (*resumeAttempt, error) {
|
|
if err := s.validateContextCeilingEntry(ctx, session.TaskID); err != nil {
|
|
return s.finishResumeAttemptWithError(startupAttempt, err)
|
|
}
|
|
seam3Res, refusal := s.admitSeam3(ctx, session.TaskID, sessionID, origin)
|
|
if refusal != nil {
|
|
return startupAttempt, refusal
|
|
}
|
|
defer seam3Res.releaseIfNotConsumed()
|
|
s.recordManualOverrideIfAdmitted(ctx, session.TaskID, sessionID, seam3Res.manualOverride, seam3Res.population, seam3Res.populationKnown, seam3Res.ceiling)
|
|
|
|
// Bounded to two attempts: a fresh cold resume, and — if the launched
|
|
// agent never reports prompt-ready — one reap-and-retry, mirroring the
|
|
// already-tracked-execution branch above. Without this, a wedged launch
|
|
// right after a backend restart (the common "kandev restart" cold-resume
|
|
// shape: in-memory execution store empty, executors_running row intact)
|
|
// surfaced a bare "agent not ready after resume: ... context deadline
|
|
// exceeded" with no self-heal, unlike every other resume path in this
|
|
// file.
|
|
//
|
|
// The gate above runs once, before the first attempt: a retry here is
|
|
// recovering an already-admitted launch, not requesting a new one.
|
|
for attemptNumber := 1; ; attemptNumber++ {
|
|
retryable, err := s.attemptColdResume(ctx, sessionID, session, isOfficeTask, startupAttempt)
|
|
if err == nil {
|
|
seam3Res.consume()
|
|
if validationErr := s.validateResumeAttempt(startupAttempt); validationErr != nil {
|
|
s.cleanupCancelledResumeAttempt(startupAttempt)
|
|
return startupAttempt, validationErr
|
|
}
|
|
return startupAttempt, nil
|
|
}
|
|
if attemptNumber >= 2 || !retryable {
|
|
return s.finishResumeAttemptWithError(startupAttempt, err)
|
|
}
|
|
refreshed, reapErr := s.reapAndReloadSession(ctx, sessionID, err)
|
|
if reapErr != nil {
|
|
return s.finishResumeAttemptWithError(startupAttempt, reapErr)
|
|
}
|
|
session = refreshed
|
|
}
|
|
}
|
|
|
|
// reapAndReloadSession stops a prompt-unready execution and reloads the
|
|
// session row afterward. Shared by ensureSessionRunning's already-tracked-
|
|
// execution branch and its cold-resume retry loop so both recovery paths
|
|
// stay in lockstep.
|
|
func (s *Service) reapAndReloadSession(ctx context.Context, sessionID string, cause error) (*models.TaskSession, error) {
|
|
recoveryCtx := context.WithoutCancel(ctx)
|
|
if executor.IsCancellableResumeContext(ctx) {
|
|
recoveryCtx = ctx
|
|
}
|
|
if stopErr := s.reapPromptUnreadyExecution(recoveryCtx, sessionID, cause); stopErr != nil {
|
|
return nil, fmt.Errorf("failed to stop prompt-unready execution: %w", stopErr)
|
|
}
|
|
refreshed, refreshErr := s.repo.GetTaskSession(recoveryCtx, sessionID)
|
|
if refreshErr != nil {
|
|
return nil, fmt.Errorf("failed to reload session after prompt-readiness recovery: %w", refreshErr)
|
|
}
|
|
return refreshed, nil
|
|
}
|
|
|
|
// attemptColdResume performs one resume attempt for ensureSessionRunning's
|
|
// cold-resume loop (no execution currently tracked in memory). retryable is
|
|
// true only when the caller should reap the stuck execution and retry —
|
|
// specifically the main-path prompt-readiness timeout below, which is the
|
|
// exact "kandev restart" wedged-launch shape this loop exists to self-heal.
|
|
// The concurrent-resume-race branch (ErrExecutionAlreadyRunning) reports its
|
|
// own timeout as non-retryable: that failure is against an execution this
|
|
// attempt never launched, not a wedged cold launch, so retrying it here would
|
|
// blur two distinct failure modes the caller's tests pin down separately.
|
|
func (s *Service) attemptColdResume(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
isOfficeTask bool,
|
|
resumeAttempt *resumeAttempt,
|
|
) (retryable bool, err error) {
|
|
// If the session is in CREATED state with an existing workspace (executors_running
|
|
// row exists), the workspace was prepared but the agent was never started. Use
|
|
// LaunchPreparedSession which routes to startAgentOnExistingWorkspace to reuse
|
|
// the workspace rather than ResumeSession which tries a full LaunchAgent and
|
|
// conflicts with the existing execution.
|
|
if session.State == models.TaskSessionStateCreated {
|
|
hasRunning, _ := s.repo.HasExecutorRunningRow(ctx, sessionID)
|
|
if hasRunning {
|
|
if err := s.validateContextCeilingEntry(ctx, session.TaskID); err != nil {
|
|
return false, err
|
|
}
|
|
return false, s.startAgentOnPreparedWorkspace(ctx, sessionID, session, resumeAttempt)
|
|
}
|
|
}
|
|
if isOfficeTask {
|
|
return false, errOfficeTaskResumeRequiresScheduler
|
|
}
|
|
|
|
running, lookupErr := s.repo.GetExecutorRunningBySessionID(ctx, sessionID)
|
|
if lookupErr != nil && !errors.Is(lookupErr, models.ErrExecutorRunningNotFound) {
|
|
return false, fmt.Errorf("get executor running row: %w", lookupErr)
|
|
}
|
|
if running == nil {
|
|
return false, fmt.Errorf("%w (state: %s)", errSessionAwaitingRuntimeLaunch, session.State)
|
|
}
|
|
|
|
s.noteMissingWorktreesBeforeResume(sessionID, session)
|
|
|
|
// Use a detached context for ordinary resumes to prevent WebSocket request
|
|
// timeout from canceling the resume. Cancellable resume attempts retain
|
|
// their owner context so explicit cancellation can interrupt startup.
|
|
// The lifecycle layer publishes events.AgentBootReady (handled by handleAgentBootReady)
|
|
// when the agent's ACP session initializes — that's what unblocks waitForSessionReady,
|
|
// no flag-tracking needed.
|
|
resumeCtx := context.WithoutCancel(ctx)
|
|
if executor.IsCancellableResumeContext(ctx) {
|
|
resumeCtx = ctx
|
|
}
|
|
if err := s.admitCeilingDispatch(resumeCtx, session.TaskID); err != nil {
|
|
return false, err
|
|
}
|
|
dispatchCtx, releaseCeilingDispatch, err := s.commitCeilingEntryDispatch(
|
|
resumeCtx, session.TaskID, ceilingEntryBindingFromContext(resumeCtx),
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
execution, launchErr := s.executor.ResumeSession(dispatchCtx, session, true)
|
|
releaseCeilingDispatch()
|
|
if execution != nil && resumeAttempt != nil {
|
|
resumeAttempt.setExecutionID(execution.AgentExecutionID)
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(resumeAttempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
return false, attemptErr
|
|
}
|
|
if launchErr != nil {
|
|
if errors.Is(launchErr, executor.ErrExecutionAlreadyRunning) {
|
|
s.recoverAgentPromptStreamIfNeeded(resumeCtx, sessionID)
|
|
if readyErr := s.waitForAgentPromptReady(resumeCtx, sessionID); readyErr != nil {
|
|
return false, fmt.Errorf("agent not ready after resume race: %w", readyErr)
|
|
}
|
|
return false, nil
|
|
}
|
|
return false, s.handleSessionLaunchFailure(
|
|
resumeCtx,
|
|
session.TaskID,
|
|
sessionID,
|
|
fmt.Errorf("failed to resume session: %w", launchErr),
|
|
session,
|
|
)
|
|
}
|
|
|
|
// ResumeSession launches the agent asynchronously. Wait for it to finish
|
|
// initializing before returning, so the caller can send a prompt immediately.
|
|
//
|
|
// Use resumeCtx (context.WithoutCancel) here too, not the original ctx: the
|
|
// resume itself is already shielded from the caller's request deadline (see
|
|
// comment above), but a short-lived caller context (WebSocket request,
|
|
// MCP tool-call timeout, etc.) would otherwise still abort these polling
|
|
// waits early with a misleading "context deadline exceeded" even though the
|
|
// resume is progressing fine in the background and would succeed within its
|
|
// own bounded timeouts (waitForSessionReady's AgentLaunchTimeout launch
|
|
// budget, and waitForAgentPromptReady's 30s below).
|
|
if err := s.waitForSessionReady(resumeCtx, sessionID); err != nil {
|
|
return false, fmt.Errorf("session not ready after resume: %w", err)
|
|
}
|
|
readyErr := s.waitForAgentPromptReady(resumeCtx, sessionID)
|
|
if readyErr == nil {
|
|
s.logger.Debug("session resumed and ready for prompt")
|
|
return false, nil
|
|
}
|
|
return errors.Is(readyErr, ErrAgentNotReadyForPrompt), fmt.Errorf("agent not ready after resume: %w", readyErr)
|
|
}
|
|
|
|
func (s *Service) recoverAgentPromptStreamIfNeeded(ctx context.Context, sessionID string) {
|
|
if s.agentManager == nil || s.agentManager.IsAgentReadyForPrompt(ctx, sessionID) {
|
|
return
|
|
}
|
|
recoverer, ok := s.agentManager.(agentPromptStreamRecoverer)
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := recoverer.RecoverAgentPromptStream(ctx, sessionID); err != nil {
|
|
s.logger.Debug("agent prompt stream recovery did not make session ready",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
|
|
func (s *Service) reapPromptUnreadyExecution(ctx context.Context, sessionID string, cause error) error {
|
|
if s.agentManager == nil {
|
|
return fmt.Errorf("agent manager is not configured")
|
|
}
|
|
executionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if err != nil || executionID == "" {
|
|
if running, runErr := s.repo.GetExecutorRunningBySessionID(ctx, sessionID); runErr == nil && running != nil {
|
|
executionID = running.AgentExecutionID
|
|
}
|
|
}
|
|
if executionID == "" {
|
|
if err != nil {
|
|
return fmt.Errorf("execution id for session %s: %w", sessionID, err)
|
|
}
|
|
return fmt.Errorf("execution id for session %s not found", sessionID)
|
|
}
|
|
if !s.claimForcedExecutionCleanup(sessionID, executionID) {
|
|
return fmt.Errorf(
|
|
"execution %s teardown is already owned for session %s",
|
|
executionID,
|
|
sessionID,
|
|
)
|
|
}
|
|
|
|
s.logger.Warn("stopping prompt-unready agent execution before resume",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("agent_execution_id", executionID),
|
|
zap.Error(cause))
|
|
cleanupCtx, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), cancellationOperationTTL)
|
|
defer cancelCleanup()
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForExecution(executionID)
|
|
}
|
|
if err := s.executor.StopExecution(cleanupCtx, executionID, promptReadinessRecoveryStopReason, true); err != nil {
|
|
return err
|
|
}
|
|
s.markExecutionFailed(sessionID, executionID)
|
|
refreshed, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
taskID := ""
|
|
if refreshed != nil {
|
|
taskID = refreshed.TaskID
|
|
}
|
|
s.retireExecutionActivityAndPublish(
|
|
context.WithoutCancel(ctx),
|
|
taskID,
|
|
sessionID,
|
|
executionID,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("reload session after prompt-readiness teardown: %w", err)
|
|
}
|
|
if refreshed == nil {
|
|
return fmt.Errorf("reload session after prompt-readiness teardown: session %s not found", sessionID)
|
|
}
|
|
if isTerminalSessionState(refreshed.State) {
|
|
return &executor.SessionStateSupersededError{
|
|
SessionID: refreshed.ID,
|
|
State: refreshed.State,
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) waitForAgentPromptReady(ctx context.Context, sessionID string) error {
|
|
if s.agentManager == nil {
|
|
return nil
|
|
}
|
|
|
|
readyCtx, cancel := context.WithTimeout(ctx, agentPromptReadyTimeout)
|
|
defer cancel()
|
|
|
|
ticker := time.NewTicker(agentPromptReadyInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
if s.agentManager.IsAgentReadyForPrompt(readyCtx, sessionID) {
|
|
return nil
|
|
}
|
|
|
|
select {
|
|
case <-readyCtx.Done():
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
return fmt.Errorf("%w: %w", ErrAgentNotReadyForPrompt, readyCtx.Err())
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// startAgentOnPreparedWorkspace starts the agent subprocess on a session whose workspace
|
|
// was prepared (agentctl running) but whose agent process was never started. This avoids
|
|
// the "session already has an agent running" error from ResumeSession which tries a full
|
|
// LaunchAgent and conflicts with the existing execution in the lifecycle manager's store.
|
|
func (s *Service) startAgentOnPreparedWorkspace(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
resumeAttempt *resumeAttempt,
|
|
) error {
|
|
s.logger.Debug("session has prepared workspace but no agent, starting agent on existing workspace",
|
|
zap.String("session_id", sessionID))
|
|
|
|
// Boot ready is published as events.AgentBootReady by the lifecycle layer
|
|
// and routed to handleAgentBootReady, which flips the session to
|
|
// WAITING_FOR_INPUT — that's what waitForSessionReady polls for. No flag
|
|
// tracking required here.
|
|
launchCtx := context.WithoutCancel(ctx)
|
|
if executor.IsCancellableResumeContext(ctx) {
|
|
launchCtx = ctx
|
|
}
|
|
isOfficeTask, err := s.lookupOfficeTask(launchCtx, session.TaskID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to determine office task status: %w", err)
|
|
}
|
|
if isOfficeTask {
|
|
return errOfficePreparedResumeRequiresScheduler
|
|
}
|
|
task, err := s.scheduler.GetTask(launchCtx, session.TaskID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get task for prepared session: %w", err)
|
|
}
|
|
if err := s.validateContextCeilingEntry(launchCtx, session.TaskID); err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.ClaimTaskTitleSession(launchCtx, session.TaskID, sessionID); err != nil {
|
|
return fmt.Errorf("failed to claim first-turn task title: %w", err)
|
|
}
|
|
if err := s.validateContextCeilingEntry(launchCtx, session.TaskID); err != nil {
|
|
return err
|
|
}
|
|
var execution *executor.TaskExecution
|
|
if execution, err = s.launchPreparedSessionWithDynamicFallback(launchCtx, task, sessionID, executor.LaunchOptions{
|
|
AgentProfileID: session.AgentProfileID,
|
|
ExecutorID: session.ExecutorID,
|
|
StartAgent: true,
|
|
}); err != nil {
|
|
if execution != nil && resumeAttempt != nil {
|
|
resumeAttempt.setExecutionID(execution.AgentExecutionID)
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(resumeAttempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
return attemptErr
|
|
}
|
|
launchErr := fmt.Errorf("failed to start agent on prepared workspace: %w", err)
|
|
return s.handleSessionLaunchFailure(launchCtx, session.TaskID, sessionID, launchErr)
|
|
}
|
|
if execution != nil && resumeAttempt != nil {
|
|
resumeAttempt.setExecutionID(execution.AgentExecutionID)
|
|
}
|
|
if attemptErr := s.validateResumeAttempt(resumeAttempt); attemptErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
return attemptErr
|
|
}
|
|
|
|
// Same reasoning as ensureSessionRunning's resume path: use launchCtx
|
|
// (already WithoutCancel'd for the launch call above) so a short-lived
|
|
// caller context doesn't abort these polling waits early.
|
|
if err := s.waitForSessionReady(launchCtx, sessionID); err != nil {
|
|
return fmt.Errorf("session not ready after starting agent: %w", err)
|
|
}
|
|
if err := s.waitForAgentPromptReady(launchCtx, sessionID); err != nil {
|
|
return fmt.Errorf("agent not ready after starting agent: %w", err)
|
|
}
|
|
s.logger.Debug("agent started on prepared workspace and ready for prompt")
|
|
return nil
|
|
}
|
|
|
|
// waitForSessionReady polls the session state until the agent is ready for prompts.
|
|
func (s *Service) waitForSessionReady(ctx context.Context, sessionID string) error {
|
|
const pollInterval = 500 * time.Millisecond
|
|
maxWait := constants.AgentLaunchTimeout
|
|
// Derive a bounded context so the overall wait AND each GetTaskSession query
|
|
// inside the loop honor maxWait. Callers pass context.WithoutCancel(ctx) so
|
|
// the wait survives the caller's request timeout during resume/launch — but
|
|
// that context carries no deadline of its own, so without this a single
|
|
// blocking query could hang well past maxWait (the loop only checks the
|
|
// wall clock between iterations). Mirrors waitForAgentPromptReady.
|
|
readyCtx, cancel := context.WithTimeout(ctx, maxWait)
|
|
defer cancel()
|
|
// Ticker rather than time.After(pollInterval) in the loop: time.After
|
|
// allocates a fresh timer each iteration that lives until it fires, so a
|
|
// long wait would pile up ~maxWait/pollInterval live timers. Mirrors
|
|
// waitForAgentPromptReady.
|
|
ticker := time.NewTicker(pollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-readyCtx.Done():
|
|
return fmt.Errorf("timeout waiting for agent to become ready: %w", readyCtx.Err())
|
|
case <-ticker.C:
|
|
}
|
|
sess, err := s.repo.GetTaskSession(readyCtx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to check session state: %w", err)
|
|
}
|
|
switch sess.State {
|
|
case models.TaskSessionStateWaitingForInput:
|
|
return nil
|
|
case models.TaskSessionStateFailed:
|
|
errMsg := sess.ErrorMessage
|
|
if errMsg == "" {
|
|
errMsg = "session failed during startup"
|
|
}
|
|
return fmt.Errorf("session failed: %s", errMsg)
|
|
case models.TaskSessionStateCancelled, models.TaskSessionStateCompleted:
|
|
return fmt.Errorf("session in unexpected state: %s", sess.State)
|
|
}
|
|
}
|
|
}
|
|
|
|
// sessionNotFoundStatus is the status-response error used for both a missing
|
|
// session and one the caller does not own, so the response cannot be used to
|
|
// tell the two apart.
|
|
const sessionNotFoundStatus = "session not found"
|
|
|
|
func (s *Service) ensureTaskNotArchived(ctx context.Context, taskID string) error {
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load task: %w", err)
|
|
}
|
|
if task == nil {
|
|
return fmt.Errorf("failed to load task: task %s is nil", taskID)
|
|
}
|
|
if task.ArchivedAt != nil {
|
|
return executor.ErrTaskArchived
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetTaskSessionStatus returns the status of a task session including whether it's resumable
|
|
func (s *Service) GetTaskSessionStatus(ctx context.Context, taskID, sessionID string) (dto.TaskSessionStatusResponse, error) {
|
|
s.logger.Debug("checking task session status",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
|
|
resp := dto.TaskSessionStatusResponse{
|
|
SessionID: sessionID,
|
|
TaskID: taskID,
|
|
}
|
|
|
|
// Per-user scoping: a foreign session is indistinguishable from a missing
|
|
// one, so reuse the same response shape rather than a distinct error.
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
resp.Error = sessionNotFoundStatus
|
|
return resp, nil
|
|
}
|
|
|
|
// 1. Load session from database
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
resp.Error = sessionNotFoundStatus
|
|
return resp, nil
|
|
}
|
|
|
|
if session.TaskID != taskID {
|
|
resp.Error = "session does not belong to task"
|
|
return resp, nil
|
|
}
|
|
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return resp, fmt.Errorf("failed to load task: %w", err)
|
|
}
|
|
if task == nil {
|
|
return resp, fmt.Errorf("failed to load task: task %s is nil", taskID)
|
|
}
|
|
|
|
resp.State = string(session.State)
|
|
resp.UpdatedAt = session.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
|
resp.AgentProfileID = session.AgentProfileID
|
|
resp.AutoResumeAllowed, resp.AutoResumeBlockedReason = s.autoResumeEligibility(ctx, task, session)
|
|
if task.ArchivedAt != nil {
|
|
resp.AutoResumeAllowed = false
|
|
resp.AutoResumeBlockedReason = resumeReasonTaskArchived
|
|
resp.ResumeReason = resumeReasonTaskArchived
|
|
return resp, nil
|
|
}
|
|
s.populateExecutorStatusInfo(ctx, session, &resp)
|
|
|
|
running, runErr := s.repo.GetExecutorRunningBySessionID(ctx, sessionID)
|
|
resumeToken := ""
|
|
if runErr == nil && running != nil {
|
|
resumeToken = running.ResumeToken
|
|
resp.ACPSessionID = running.ResumeToken
|
|
resp.Runtime = running.Runtime
|
|
if running.Resumable {
|
|
resp.IsResumable = true
|
|
}
|
|
s.applyRemoteRuntimeStatus(ctx, sessionID, &resp)
|
|
}
|
|
|
|
if shouldHealStuckStartingSession(session, running) {
|
|
s.logger.Info("healing stale STARTING session state from ready runtime status",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("agent_execution_id", running.AgentExecutionID))
|
|
s.setSessionWaitingForInput(ctx, taskID, sessionID)
|
|
refreshedSession, refreshErr := s.repo.GetTaskSession(ctx, sessionID)
|
|
if refreshErr == nil && refreshedSession != nil {
|
|
session = refreshedSession
|
|
resp.State = string(session.State)
|
|
if !session.UpdatedAt.IsZero() {
|
|
resp.UpdatedAt = session.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Extract worktree info
|
|
populateWorktreeInfo(session, &resp)
|
|
s.populateEnvironmentWorkspaceInfo(ctx, session, &resp)
|
|
|
|
// 2. Check if this session's agent is running
|
|
if exec, ok := s.executor.GetExecutionBySession(sessionID); ok && exec != nil {
|
|
resp.IsAgentRunning = true
|
|
resp.NeedsResume = false
|
|
return resp, nil
|
|
}
|
|
|
|
// 3. Session can be resumed if it has a resume token
|
|
if resumeToken != "" {
|
|
// Auto-resume FAILED sessions when the runtime is resumable: PR #670 made
|
|
// terminal-state resume safe (cleanup + retry), so recover transparently
|
|
// before surfacing the error. Frontend falls back to restore_workspace if
|
|
// the resume itself fails.
|
|
if session.State == models.TaskSessionStateFailed && running != nil && running.Resumable {
|
|
out := s.validateResumeEligibility(session, resp)
|
|
if out.NeedsResume {
|
|
out.ResumeReason = resumeReasonFailedSessionResumable
|
|
}
|
|
return out, nil
|
|
}
|
|
// A session cancelled by archiving is a system side effect (Service.
|
|
// ArchiveTask / HandoffService cascade), not an explicit user stop —
|
|
// it must recover like a FAILED session once the task is unarchived
|
|
// instead of falling into the terminal-session block below. See
|
|
// models.IsArchiveCancelReason.
|
|
if isArchiveCancelledSession(session) {
|
|
if running != nil && running.Resumable {
|
|
out := s.validateResumeEligibility(session, resp)
|
|
if out.NeedsResume {
|
|
out.ResumeReason = resumeReasonArchiveCancelledResumable
|
|
}
|
|
return out, nil
|
|
}
|
|
// The persisted token's runtime reports Resumable=false — it isn't
|
|
// safe to resume with. Route through the same fresh-start result
|
|
// used when there's no token/running row at all instead of
|
|
// validateResumeEligibility, which never sets IsResumable and
|
|
// would return NeedsResume=true with IsResumable=false, a
|
|
// combination the frontend's auto-resume gate
|
|
// (needs_resume && is_resumable) can never satisfy.
|
|
return evaluateFreshStartResume(session, running, runErr, resp), nil
|
|
}
|
|
// The active-session reconciliation sweep used the exact orphan reason
|
|
// when it cancelled execution-less sessions. Treat those legacy rows like
|
|
// archive cancellations so opening the task restores the same conversation
|
|
// without replaying its prior task prompt. Other CANCELLED rows remain
|
|
// explicit user/coordinator stops and stay on workspace restore.
|
|
if isOrphanCancelledSession(session) {
|
|
if running != nil && running.Resumable {
|
|
out := s.validateResumeEligibility(session, resp)
|
|
if out.NeedsResume {
|
|
out.ResumeReason = resumeReasonOrphanCancelledResumable
|
|
}
|
|
return out, nil
|
|
}
|
|
return evaluateFreshStartResume(session, running, runErr, resp), nil
|
|
}
|
|
// Don't auto-resume other terminal sessions (CANCELLED stays stopped, COMPLETED is done).
|
|
if !isActiveSessionState(session.State) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = false
|
|
resp.NeedsResume = false
|
|
resp.NeedsWorkspaceRestore = canRestoreWorkspace(&resp)
|
|
return resp, nil
|
|
}
|
|
return s.validateResumeEligibility(session, resp), nil
|
|
}
|
|
|
|
// 4. No resume token — check if session can be started fresh.
|
|
return evaluateFreshStartResume(session, running, runErr, resp), nil
|
|
}
|
|
|
|
const (
|
|
autoResumeBlockedLaunchQueued = "launch_queued"
|
|
autoResumeBlockedOwnershipUnavailable = "ownership_unavailable"
|
|
)
|
|
|
|
func (s *Service) sessionOpenRecoveryBlockReason(
|
|
ctx context.Context,
|
|
taskID string,
|
|
session *models.TaskSession,
|
|
) string {
|
|
if s == nil || s.repo == nil || session == nil {
|
|
return autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil || task == nil {
|
|
return autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
allowed, reason := s.autoResumeEligibility(ctx, task, session)
|
|
if allowed {
|
|
return ""
|
|
}
|
|
return reason
|
|
}
|
|
|
|
// autoResumeEligibility is intentionally conservative about deferred launches.
|
|
// Passive inspection may resume a session unless a valid durable launch belongs
|
|
// to that exact session. A malformed deferred record blocks recovery instead of
|
|
// falling back to a fresh launch.
|
|
//
|
|
//nolint:cyclop // Passive recovery checks each ownership source independently.
|
|
func (s *Service) autoResumeEligibility(
|
|
ctx context.Context, task *models.Task, session *models.TaskSession,
|
|
) (bool, string) {
|
|
if session == nil || task == nil {
|
|
return false, autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
raw, present := task.Metadata[models.MetaKeyDeferredLaunch]
|
|
if !present || raw == nil {
|
|
return true, ""
|
|
}
|
|
record, ok := raw.(map[string]interface{})
|
|
if !ok {
|
|
return false, autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
if len(record) == 0 {
|
|
return true, ""
|
|
}
|
|
deferral, err := models.ReadCeilingDeferral(record)
|
|
if err != nil {
|
|
return false, autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
if deferral.Kind == models.CeilingLaunchStart {
|
|
// Seam 1 can defer before a destination session exists. Once a route is
|
|
// committed, the route destination is the only session that may own this
|
|
// record. Do not let passive recovery wake an existing predecessor while
|
|
// the sessionless workflow start is waiting, and fail closed when the
|
|
// route/binding cannot identify that destination.
|
|
destinationID := models.CeilingDeferralSessionID(task, deferral)
|
|
if destinationID == "" || !models.CeilingDeferralTargetsSession(task, deferral, destinationID) {
|
|
return false, autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
if destinationID == session.ID {
|
|
return false, autoResumeBlockedLaunchQueued
|
|
}
|
|
return true, ""
|
|
}
|
|
destinationID := models.CeilingDeferralSessionID(task, deferral)
|
|
if destinationID == "" || !models.CeilingDeferralTargetsSession(task, deferral, destinationID) {
|
|
return false, autoResumeBlockedOwnershipUnavailable
|
|
}
|
|
if destinationID == session.ID {
|
|
return false, autoResumeBlockedLaunchQueued
|
|
}
|
|
// A valid deferral owned by another session does not block passive recovery
|
|
// of this session. The queue projection is task-scoped and names its exact
|
|
// destination separately.
|
|
return true, ""
|
|
}
|
|
|
|
// evaluateFreshStartResume checks whether a session without a resume token can be
|
|
// started fresh. Sessions in error-recovery state (non-empty ErrorMessage) are marked
|
|
// resumable but not auto-resumed, so the user sees the error and can choose via action buttons.
|
|
func evaluateFreshStartResume(session *models.TaskSession, running *models.ExecutorRunning, runErr error, resp dto.TaskSessionStatusResponse) dto.TaskSessionStatusResponse {
|
|
if runErr == nil && running != nil && isActiveSessionState(session.State) {
|
|
if isErrorRecoveryState(session) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = false
|
|
resp.ResumeReason = resumeReasonErrorRecovery
|
|
return resp
|
|
}
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = true
|
|
resp.ResumeReason = "agent_not_running_fresh_start"
|
|
return resp
|
|
}
|
|
if running == nil &&
|
|
(session.State == models.TaskSessionStateWaitingForInput || session.State == models.TaskSessionStateStarting || session.State == models.TaskSessionStateRunning) &&
|
|
(runErr == nil || errors.Is(runErr, models.ErrExecutorRunningNotFound)) &&
|
|
models.HasInterruptedRecoveryPending(session.Metadata) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = true
|
|
resp.ResumeReason = resumeReasonInterruptedSessionResumable
|
|
return resp
|
|
}
|
|
// The archive cleanup (Service.ArchiveTask / HandoffService cascade) tears
|
|
// down the ExecutorRunning row entirely, so an archive-cancelled session
|
|
// reaches this function with running == nil — exactly the shape an
|
|
// unarchive-then-open observes. Treat it as fresh-start resumable rather
|
|
// than falling through to read-only workspace restore.
|
|
if isArchiveCancelledSession(session) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = true
|
|
resp.ResumeReason = resumeReasonArchiveCancelledResumable
|
|
return resp
|
|
}
|
|
if isOrphanCancelledSession(session) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = true
|
|
resp.ResumeReason = resumeReasonOrphanCancelledResumable
|
|
return resp
|
|
}
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = false
|
|
resp.NeedsResume = false
|
|
resp.NeedsWorkspaceRestore = canRestoreWorkspace(&resp)
|
|
return resp
|
|
}
|
|
|
|
func (s *Service) populateExecutorStatusInfo(ctx context.Context, session *models.TaskSession, resp *dto.TaskSessionStatusResponse) {
|
|
if session == nil || resp == nil {
|
|
return
|
|
}
|
|
resp.ExecutorID = session.ExecutorID
|
|
if session.ExecutorID == "" {
|
|
return
|
|
}
|
|
execModel, err := s.repo.GetExecutor(ctx, session.ExecutorID)
|
|
if err != nil || execModel == nil {
|
|
return
|
|
}
|
|
resp.ExecutorType = string(execModel.Type)
|
|
resp.ExecutorName = execModel.Name
|
|
resp.IsRemoteExecutor = models.IsRemoteExecutorType(execModel.Type)
|
|
resp.Capabilities.EmbeddedVscode = capabilities.SupportsEmbeddedVscode(execModel.Type, runtime.GOOS)
|
|
}
|
|
|
|
func (s *Service) applyRemoteRuntimeStatus(ctx context.Context, sessionID string, resp *dto.TaskSessionStatusResponse) {
|
|
if s.agentManager == nil || resp == nil || !resp.IsRemoteExecutor {
|
|
return
|
|
}
|
|
status, err := s.agentManager.GetRemoteRuntimeStatusBySession(ctx, sessionID)
|
|
if err != nil || status == nil {
|
|
return
|
|
}
|
|
if status.RuntimeName != "" {
|
|
resp.Runtime = status.RuntimeName
|
|
}
|
|
resp.RemoteState = status.State
|
|
resp.RemoteName = status.RemoteName
|
|
resp.RemoteStatusErr = publicRemoteStatusError(status.ErrorMessage)
|
|
if status.CreatedAt != nil && !status.CreatedAt.IsZero() {
|
|
resp.RemoteCreatedAt = status.CreatedAt.UTC().Format(time.RFC3339)
|
|
}
|
|
if !status.LastCheckedAt.IsZero() {
|
|
resp.RemoteCheckedAt = status.LastCheckedAt.UTC().Format(time.RFC3339)
|
|
}
|
|
}
|
|
|
|
const remoteStatusUnavailable = "remote executor status is unavailable"
|
|
|
|
func publicRemoteStatusError(errorMessage string) string {
|
|
if errorMessage == "" {
|
|
return ""
|
|
}
|
|
return remoteStatusUnavailable
|
|
}
|
|
|
|
// populateWorktreeInfo copies worktree path and branch into the response if present.
|
|
func canRestoreWorkspace(resp *dto.TaskSessionStatusResponse) bool {
|
|
return resp != nil && resp.WorktreePath != nil && *resp.WorktreePath != ""
|
|
}
|
|
|
|
func populateWorktreeInfo(session *models.TaskSession, resp *dto.TaskSessionStatusResponse) {
|
|
if len(session.Worktrees) == 0 {
|
|
return
|
|
}
|
|
wt := session.Worktrees[0]
|
|
if wt.WorktreePath != "" {
|
|
resp.WorktreePath = &wt.WorktreePath
|
|
}
|
|
if wt.WorktreeBranch != "" {
|
|
resp.WorktreeBranch = &wt.WorktreeBranch
|
|
}
|
|
}
|
|
|
|
func (s *Service) populateEnvironmentWorkspaceInfo(ctx context.Context, session *models.TaskSession, resp *dto.TaskSessionStatusResponse) {
|
|
if hasWorktreeStatus(resp) {
|
|
return
|
|
}
|
|
env, err := s.repo.GetTaskEnvironmentByTaskID(ctx, session.TaskID)
|
|
if err != nil || env == nil {
|
|
return
|
|
}
|
|
if session.TaskEnvironmentID != "" && env.ID != session.TaskEnvironmentID {
|
|
return
|
|
}
|
|
// WorkspacePath is the canonical task-root identity. A repository-less
|
|
// environment is still restorable, so do not require a repo row here.
|
|
if resp.WorktreePath == nil && env.WorkspacePath != "" {
|
|
resp.WorktreePath = &env.WorkspacePath
|
|
}
|
|
if len(env.Repos) == 0 {
|
|
return
|
|
}
|
|
if resp.WorktreePath == nil && env.Repos[0].WorktreePath != "" {
|
|
resp.WorktreePath = &env.Repos[0].WorktreePath
|
|
}
|
|
if resp.WorktreeBranch == nil && env.Repos[0].WorktreeBranch != "" {
|
|
resp.WorktreeBranch = &env.Repos[0].WorktreeBranch
|
|
}
|
|
}
|
|
|
|
func hasWorktreeStatus(resp *dto.TaskSessionStatusResponse) bool {
|
|
return resp != nil &&
|
|
resp.WorktreePath != nil && *resp.WorktreePath != "" &&
|
|
resp.WorktreeBranch != nil && *resp.WorktreeBranch != ""
|
|
}
|
|
|
|
// isActiveSessionState returns true for session states where lazy resume makes sense.
|
|
func isActiveSessionState(state models.TaskSessionState) bool {
|
|
switch state {
|
|
case models.TaskSessionStateWaitingForInput,
|
|
models.TaskSessionStateStarting,
|
|
models.TaskSessionStateRunning:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isErrorRecoveryState returns true when a session is in WAITING_FOR_INPUT with
|
|
// a non-empty ErrorMessage, indicating it was set by handleRecoverableFailure.
|
|
func isErrorRecoveryState(session *models.TaskSession) bool {
|
|
return session != nil &&
|
|
session.State == models.TaskSessionStateWaitingForInput &&
|
|
session.ErrorMessage != ""
|
|
}
|
|
|
|
// isArchiveCancelledSession reports whether session was cancelled by an
|
|
// archive (Service.ArchiveTask's single-task path or HandoffService's
|
|
// cascade archive) rather than an explicit user/coordinator stop. Such
|
|
// sessions must resume like a FAILED session once the owning task is
|
|
// unarchived — see models.IsArchiveCancelReason and its two constants.
|
|
func isArchiveCancelledSession(session *models.TaskSession) bool {
|
|
return session != nil &&
|
|
session.State == models.TaskSessionStateCancelled &&
|
|
models.IsArchiveCancelReason(session.ErrorMessage)
|
|
}
|
|
|
|
// isOrphanCancelledSession reports the exact legacy cancellation marker used
|
|
// by session reconciliation. Matching the reason exactly keeps explicit user
|
|
// stops and unrelated terminal states out of automatic recovery.
|
|
func isOrphanCancelledSession(session *models.TaskSession) bool {
|
|
return session != nil &&
|
|
session.State == models.TaskSessionStateCancelled &&
|
|
models.IsOrphanCancelReason(session.ErrorMessage)
|
|
}
|
|
|
|
func shouldHealStuckStartingSession(session *models.TaskSession, running *models.ExecutorRunning) bool {
|
|
if session == nil || running == nil {
|
|
return false
|
|
}
|
|
if session.State != models.TaskSessionStateStarting {
|
|
return false
|
|
}
|
|
if running.Status != "ready" {
|
|
return false
|
|
}
|
|
// Pre-refactor this also checked session.AgentExecutionID vs running.AgentExecutionID
|
|
// to skip healing on a divergent row. With the executors_running table now the
|
|
// single source of truth, that comparison is structurally always equal — drop it.
|
|
return true
|
|
}
|
|
|
|
// validateResumeEligibility performs final checks before marking a session as resumable.
|
|
func (s *Service) validateResumeEligibility(session *models.TaskSession, resp dto.TaskSessionStatusResponse) dto.TaskSessionStatusResponse {
|
|
if session.AgentProfileID == "" {
|
|
resp.Error = "session missing agent profile"
|
|
resp.IsResumable = false
|
|
return resp
|
|
}
|
|
|
|
// A missing worktree directory does not make the session unresumable.
|
|
// Archive cleanup deletes the directory and keeps the branch, so reporting
|
|
// IsResumable=false here would hide the resume affordance for every
|
|
// unarchived task — see noteMissingWorktreesBeforeResume. The resume
|
|
// itself recreates the directory through the worktree manager.
|
|
|
|
// Don't auto-resume sessions in error-recovery state.
|
|
if isErrorRecoveryState(session) {
|
|
resp.IsAgentRunning = false
|
|
resp.IsResumable = true
|
|
resp.NeedsResume = false
|
|
resp.ResumeReason = resumeReasonErrorRecovery
|
|
return resp
|
|
}
|
|
|
|
resp.IsAgentRunning = false
|
|
resp.NeedsResume = true
|
|
resp.ResumeReason = "agent_not_running"
|
|
return resp
|
|
}
|
|
|
|
// StopTask stops agent execution for a task (stops all active sessions for the task)
|
|
func (s *Service) StopTask(ctx context.Context, taskID string, reason string, force bool) error {
|
|
s.logger.Info("stopping task execution",
|
|
zap.String("task_id", taskID),
|
|
zap.String("reason", reason),
|
|
zap.Bool("force", force))
|
|
|
|
// Stop all agents for this task
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForTask(taskID)
|
|
}
|
|
if err := s.executor.StopByTaskID(ctx, taskID, reason, force); err != nil {
|
|
if !errors.Is(err, executor.ErrOrphanRecoveryIncomplete) {
|
|
return err
|
|
}
|
|
// Every session that could be reached did stop; only a registry-only
|
|
// orphan's row failed to load. Log it and still transition to REVIEW
|
|
// rather than surfacing a false "failed to stop task" to the user.
|
|
s.logger.Warn("task stop reached REVIEW with an orphan recovery load failure",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
// Move task to REVIEW state for user review
|
|
if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateReview); err != nil {
|
|
s.logger.Error("failed to update task state to REVIEW after stop",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
// Don't return error - the stop was successful
|
|
} else {
|
|
s.logger.Info("task moved to REVIEW state after stop",
|
|
zap.String("task_id", taskID))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// StopTaskForCoordinator gracefully halts every currently-observed live
|
|
// execution for taskID without accepting caller-controlled lifecycle options.
|
|
// The operation is idempotent: no accepted cancellation is not_running.
|
|
func (s *Service) StopTaskForCoordinator(ctx context.Context, taskID string) (CoordinatorTaskStopResult, error) {
|
|
if s.executor == nil {
|
|
return CoordinatorTaskStopResult{}, errors.New("coordinator stop: executor is not configured")
|
|
}
|
|
sessions, err := s.repo.ListActiveTaskSessionsByTaskID(ctx, taskID)
|
|
if err != nil {
|
|
return CoordinatorTaskStopResult{}, fmt.Errorf("coordinator stop: list active sessions for task %q: %w", taskID, err)
|
|
}
|
|
|
|
// Repository ordering is launch-oriented. Coordinator control uses stable
|
|
// identity ordering so partial failures and retries are deterministic.
|
|
sort.SliceStable(sessions, func(i, j int) bool {
|
|
return coordinatorStopSessionID(sessions[i]) < coordinatorStopSessionID(sessions[j])
|
|
})
|
|
|
|
accepted := 0
|
|
failures := make([]error, 0)
|
|
for _, candidate := range sessions {
|
|
if candidate == nil || candidate.ID == "" {
|
|
failures = append(failures, errors.New("coordinator stop: active session candidate is nil or has an empty ID"))
|
|
continue
|
|
}
|
|
changed, stopErr := s.stopTaskSessionForCoordinator(ctx, taskID, candidate.ID)
|
|
if stopErr != nil {
|
|
failures = append(failures, stopErr)
|
|
continue
|
|
}
|
|
if changed {
|
|
accepted++
|
|
}
|
|
}
|
|
if accepted > 0 {
|
|
// This helper owns Office/archive/active-state guards and publishes
|
|
// through the task-service adapter. Remaining working sessions still
|
|
// block REVIEW on a partial stop.
|
|
s.writeTaskReviewState(ctx, taskID, "")
|
|
}
|
|
if len(failures) > 0 {
|
|
s.logger.Warn("coordinator stop partially failed",
|
|
zap.String("task_id", taskID),
|
|
zap.Int("accepted", accepted),
|
|
zap.Int("failed", len(failures)))
|
|
return CoordinatorTaskStopResult{}, errors.Join(failures...)
|
|
}
|
|
if accepted == 0 {
|
|
return CoordinatorTaskStopResult{Status: CoordinatorTaskStopStatusNotRunning}, nil
|
|
}
|
|
return CoordinatorTaskStopResult{Status: CoordinatorTaskStopStatusStopped}, nil
|
|
}
|
|
|
|
func coordinatorStopSessionID(session *models.TaskSession) string {
|
|
if session == nil {
|
|
return ""
|
|
}
|
|
return session.ID
|
|
}
|
|
|
|
func (s *Service) stopTaskSessionForCoordinator(ctx context.Context, taskID, sessionID string) (bool, error) {
|
|
endCancel := s.beginCancelInFlight(sessionID)
|
|
defer endCancel()
|
|
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
lock.Lock()
|
|
|
|
result, teardownClaimed, err := s.stopTaskSessionForCoordinatorLocked(ctx, taskID, sessionID)
|
|
lock.Unlock()
|
|
release()
|
|
// The detached teardown callback is allowed to observe coordinator state;
|
|
// clear the cancellation ownership marker before handing it off. The defer
|
|
// above remains as an idempotent safety net for error returns.
|
|
endCancel()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if result.Changed && s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForSession(sessionID)
|
|
}
|
|
// Detached teardown must not observe this operation as an in-flight
|
|
// cancellation. ScheduleTeardown starts a goroutine, so relying on the
|
|
// deferred release above makes the ordering scheduler-dependent.
|
|
endCancel()
|
|
if result.Changed && teardownClaimed {
|
|
result.ScheduleTeardown()
|
|
}
|
|
return result.Changed, nil
|
|
}
|
|
|
|
// stopTaskSessionForCoordinatorLocked commits cancellation and claims teardown
|
|
// ownership while sessionID's cancelInFlight guard is held. The caller must
|
|
// release that guard before scheduling the returned teardown.
|
|
func (s *Service) stopTaskSessionForCoordinatorLocked(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
) (executor.SessionStopResult, bool, error) {
|
|
|
|
// Re-read after acquiring the shared cancel/ready/queue guard. A candidate
|
|
// may have become terminal while this call waited for another decision.
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: reload session %q: %w", sessionID, err)
|
|
}
|
|
if session == nil {
|
|
return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: reload session %q returned nil", sessionID)
|
|
}
|
|
if session.TaskID != taskID {
|
|
return executor.SessionStopResult{}, false, fmt.Errorf("coordinator stop: session %q belongs to task %q, not %q", sessionID, session.TaskID, taskID)
|
|
}
|
|
if !isCoordinatorStoppableSessionState(session.State) {
|
|
return executor.SessionStopResult{FinalState: session.State}, false, nil
|
|
}
|
|
|
|
s.taskRuntimeStateMu.Lock()
|
|
// Halt-only intent also disarms any provider-backoff retry. This must run
|
|
// even when the failed execution has already disappeared and the result is
|
|
// therefore not_running; otherwise its timer can launch replacement work.
|
|
s.retireAndClearTransientRetryState(sessionID)
|
|
result, stopErr := s.executor.StopSessionDetailed(ctx, session, coordinatorMCPStopReason, false)
|
|
if stopErr == nil && result.Changed {
|
|
// Cancellation takes effect before detached runtime teardown. Tombstone
|
|
// the execution immediately so buffered agent frames cannot recreate
|
|
// session output after the coordinator has acknowledged the stop.
|
|
s.markExecutionFailed(sessionID, result.ExecutionID)
|
|
}
|
|
teardownClaimed := stopErr == nil && result.Changed && s.claimExecutionTeardown(
|
|
sessionID,
|
|
result.ExecutionID,
|
|
executionTeardownIntentGraceful,
|
|
)
|
|
s.taskRuntimeStateMu.Unlock()
|
|
s.resolveTransientRetryMessages(context.WithoutCancel(ctx), sessionID)
|
|
if stopErr != nil {
|
|
return result, false, fmt.Errorf("coordinator stop: session %q: %w", sessionID, stopErr)
|
|
}
|
|
return result, teardownClaimed, nil
|
|
}
|
|
|
|
func isCoordinatorStoppableSessionState(state models.TaskSessionState) bool {
|
|
switch state {
|
|
case models.TaskSessionStateCreated,
|
|
models.TaskSessionStateStarting,
|
|
models.TaskSessionStateRunning,
|
|
models.TaskSessionStateWaitingForInput:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// CancelTaskExecution stops active sessions for a task without mutating task state.
|
|
// It is used by office tree controls where pause/cancel state transitions are
|
|
// handled by the office service itself.
|
|
func (s *Service) CancelTaskExecution(ctx context.Context, taskID string, reason string, force bool) error {
|
|
s.logger.Info("cancelling task execution",
|
|
zap.String("task_id", taskID),
|
|
zap.String("reason", reason),
|
|
zap.Bool("force", force))
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForTask(taskID)
|
|
}
|
|
err := s.executor.StopByTaskID(ctx, taskID, reason, force)
|
|
if err != nil && errors.Is(err, executor.ErrOrphanRecoveryIncomplete) {
|
|
// Every session that could be reached did stop; only a registry-only
|
|
// orphan's row failed to load. Log it rather than reporting a false
|
|
// cancellation failure to office tree controls.
|
|
s.logger.Warn("task execution cancelled with an orphan recovery load failure",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
// CancelTaskExecutionSynchronously stops every active session for a task and
|
|
// waits for each runtime process to exit. Archive and delete cascades use this
|
|
// boundary so an unarchive cannot race a late asynchronous stop callback.
|
|
func (s *Service) CancelTaskExecutionSynchronously(ctx context.Context, taskID, reason string, force bool) error {
|
|
sessions, err := s.repo.ListActiveTaskSessionsByTaskID(ctx, taskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var lastErr error
|
|
for _, session := range sessions {
|
|
if session == nil {
|
|
continue
|
|
}
|
|
if err := s.StopSessionSynchronously(ctx, session.ID, reason, force); err != nil {
|
|
lastErr = err
|
|
}
|
|
}
|
|
return lastErr
|
|
}
|
|
|
|
// StopSession stops agent execution for a specific session
|
|
func (s *Service) StopSession(ctx context.Context, sessionID string, reason string, force bool) error {
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
|
|
// A direct session stop is a true retry-ending transition. Retire the
|
|
// in-memory loop and its durable notice before stopping the execution.
|
|
s.resetTransientRetryWithContext(ctx, sessionID, true)
|
|
|
|
s.logger.Info("stopping session execution",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("reason", reason),
|
|
zap.Bool("force", force))
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForSession(sessionID)
|
|
}
|
|
return s.executor.Stop(ctx, sessionID, reason, force)
|
|
}
|
|
|
|
// StopSessionSynchronously is the internal cleanup variant of StopSession. It
|
|
// skips user authorization because task cleanup may run after the task/session
|
|
// rows have been removed, and it waits for the executor process to exit before
|
|
// returning to the destructive worktree cleanup path.
|
|
func (s *Service) StopSessionSynchronously(ctx context.Context, sessionID string, reason string, force bool) error {
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForSession(sessionID)
|
|
}
|
|
return s.executor.StopSessionSynchronously(ctx, sessionID, reason, force)
|
|
}
|
|
|
|
// deleteSessionAndPublishRemoval commits a session deletion before publishing
|
|
// the terminal event consumed by ordered conversation subscribers.
|
|
func (s *Service) deleteSessionAndPublishRemoval(ctx context.Context, taskID, sessionID string) error {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.deleteSessionAndCleanAttachments(ctx, session); err != nil {
|
|
return err
|
|
}
|
|
if s.eventBus != nil {
|
|
if err := s.eventBus.Publish(ctx, events.SessionRemoved, bus.NewEvent(
|
|
events.SessionRemoved,
|
|
"orchestrator",
|
|
map[string]interface{}{metaKeySessionID: sessionID, metaKeyTaskID: taskID},
|
|
)); err != nil {
|
|
s.logger.Warn("session deleted but removal event publish failed",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
return nil
|
|
|
|
}
|
|
|
|
// DeleteSession deletes a session that is not currently running.
|
|
func (s *Service) DeleteSession(ctx context.Context, sessionID string) error {
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("session not found: %w", err)
|
|
}
|
|
|
|
// Prevent deleting active sessions
|
|
switch session.State {
|
|
case models.TaskSessionStateRunning, models.TaskSessionStateStarting:
|
|
return fmt.Errorf("cannot delete session in %s state — stop it first", session.State)
|
|
}
|
|
|
|
taskID := session.TaskID
|
|
wasPrimary := session.IsPrimary
|
|
|
|
// A settled DB state can still own a workspace execution or detached
|
|
// background work. Quiesce that runtime boundary before removing the row.
|
|
if err := s.quiesceSessionExecutionBeforeDeletion(ctx, taskID, sessionID); err != nil {
|
|
return err
|
|
}
|
|
// Deletion ends the session incarnation even when no execution remains.
|
|
// Retire the retry loop before removing the row so a buffered provider
|
|
// failure cannot recreate notice state for a deleted session ID.
|
|
s.resetTransientRetryWithContext(ctx, sessionID, true)
|
|
|
|
s.logger.Info("deleting session",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("task_id", taskID),
|
|
zap.String("state", string(session.State)),
|
|
zap.Bool("was_primary", wasPrimary))
|
|
if err := s.deleteSessionAndPublishError(ctx, session); err != nil {
|
|
return fmt.Errorf("failed to delete session: %w", err)
|
|
}
|
|
// Remove ephemeral state only when it is still bound to the deleted
|
|
// incarnation. This purge includes reserved rows and cannot erase a
|
|
// replacement session that reuses the textual session ID.
|
|
if s.messageQueue != nil {
|
|
identity := messagequeue.QueueSessionIdentity{
|
|
TaskID: taskID,
|
|
SessionID: sessionID,
|
|
SessionIncarnationID: session.QueueIncarnationID,
|
|
}
|
|
if err := s.messageQueue.PurgeDeletedSession(ctx, identity); err != nil {
|
|
return fmt.Errorf("failed to purge deleted session queue: %w", err)
|
|
}
|
|
}
|
|
// The row is gone, so retire the detached-launch attestation and parked
|
|
// projection before any later session event can reuse this ID.
|
|
s.clearParkedProjectionOnSessionDeleted(ctx, taskID, sessionID)
|
|
|
|
// Drop the in-memory git snapshot throttle entries for an environment only
|
|
// after its session has been removed. The cache is environment-scoped, so a
|
|
// session ID cannot identify the entries and must not be used as the key.
|
|
if s.gitSnapshotCache != nil && session.TaskEnvironmentID != "" {
|
|
s.gitSnapshotCache.forget(session.TaskEnvironmentID)
|
|
}
|
|
// Same reasoning for the push-detection tracker. Multi-repo sessions
|
|
// accumulate one entry per repo; pushTrackerForget walks them all.
|
|
s.pushTrackerForget(sessionID)
|
|
// And the foreground/background turn-activity signal. Execution teardown
|
|
// normally retires it after the final owner exits; deletion forcibly
|
|
// invalidates any trailing token so the removed session cannot be recreated
|
|
// through a stale activity pointer.
|
|
s.clearTurnActivity(sessionID)
|
|
|
|
// Queue rows and per-session policy are removed atomically with the task
|
|
// session row. The repository callback publishes a session-scoped event
|
|
// when it owns the purge; focused compositions use the task-scoped fallback.
|
|
if !s.sessionQueuePurgeNotifierRegistered {
|
|
s.publishTaskQueueStatusEvent(ctx, taskID, "")
|
|
}
|
|
|
|
// Auto-promote another session if we deleted the primary
|
|
if wasPrimary {
|
|
s.promoteNextPrimaryAfterRemoval(ctx, taskID, sessionID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) deleteSessionAndCleanAttachments(ctx context.Context, session *models.TaskSession) error {
|
|
var deletedAttachments []*models.TaskMessageAttachment
|
|
var err error
|
|
if deleter, ok := s.repo.(sessionAttachmentDeleter); ok {
|
|
deletedAttachments, err = deleter.DeleteTaskSessionWithAttachments(ctx, session)
|
|
} else {
|
|
err = s.repo.DeleteTaskSession(ctx, session)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if remover, ok := s.attachmentReader.(attachmentBytesRemover); ok {
|
|
remover.RemoveBytes(deletedAttachments)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) deleteSessionAndPublishError(ctx context.Context, session *models.TaskSession) error {
|
|
lock, release := s.acquireTaskSessionErrorGuard(session.TaskID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
|
|
if err := s.deleteSessionAndPublishRemoval(ctx, session.TaskID, session.ID); err != nil {
|
|
return err
|
|
}
|
|
s.publishDeletedSessionError(ctx, session.TaskID, session.ID)
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) publishDeletedSessionError(ctx context.Context, taskID, sessionID string) {
|
|
if s.eventBus == nil {
|
|
return
|
|
}
|
|
eventCtx := context.WithoutCancel(ctx)
|
|
if err := s.publishTaskSessionErrorEvent(eventCtx, taskID, sessionID, false, nil); err != nil {
|
|
s.logger.Warn("failed to publish deleted task session error event",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
|
|
retainedSessionID, lastError, err := s.newestRetainedSessionError(eventCtx, taskID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to reload retained task session error after delete",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if lastError == nil {
|
|
return
|
|
}
|
|
if err := s.publishTaskSessionErrorEvent(eventCtx, taskID, retainedSessionID, true, lastError); err != nil {
|
|
s.logger.Warn("failed to republish retained task session error after delete",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", retainedSessionID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
func (s *Service) newestRetainedSessionError(
|
|
ctx context.Context,
|
|
taskID string,
|
|
) (string, *models.LastAgentError, error) {
|
|
sessions, err := s.repo.ListTaskSessions(ctx, taskID)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
var newestSessionID string
|
|
var newest models.LastAgentError
|
|
found := false
|
|
for _, session := range sessions {
|
|
if session == nil {
|
|
continue
|
|
}
|
|
lastError, ok := models.LoadLastAgentError(session.Metadata)
|
|
if !ok || lastError.IsDismissed() {
|
|
continue
|
|
}
|
|
if found && !lastError.OccurredAt.After(newest.OccurredAt) {
|
|
continue
|
|
}
|
|
newestSessionID = session.ID
|
|
newest = lastError
|
|
found = true
|
|
}
|
|
if !found {
|
|
return "", nil, nil
|
|
}
|
|
return newestSessionID, &newest, nil
|
|
}
|
|
|
|
func (s *Service) publishTaskSessionErrorEvent(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
active bool,
|
|
lastError *models.LastAgentError,
|
|
) error {
|
|
eventData := map[string]interface{}{
|
|
"task_id": taskID,
|
|
"session_id": sessionID,
|
|
"active": active,
|
|
}
|
|
if active && lastError != nil {
|
|
eventData["message"] = lastError.Message
|
|
eventData["occurred_at"] = lastError.OccurredAt.Format(time.RFC3339Nano)
|
|
eventData["stamp"] = lastError.Stamp()
|
|
eventData["agent_execution_id"] = lastError.AgentExecutionID
|
|
eventData["execution_id"] = lastError.ExecutionID
|
|
if lastError.Phase != "" {
|
|
eventData["phase"] = lastError.Phase
|
|
}
|
|
if lastError.AttemptID != "" {
|
|
eventData["attempt_id"] = lastError.AttemptID
|
|
}
|
|
if len(lastError.Causes) > 0 {
|
|
eventData["causes"] = append([]models.AgentErrorCause(nil), lastError.Causes...)
|
|
}
|
|
if lastError.Details != "" {
|
|
eventData["details"] = lastError.Details
|
|
}
|
|
if lastError.TaskRepositoryID != "" {
|
|
eventData["task_repository_id"] = lastError.TaskRepositoryID
|
|
}
|
|
if lastError.Code != "" {
|
|
eventData["category"] = lastError.Code
|
|
}
|
|
if len(lastError.RecoveryActions) > 0 {
|
|
eventData["recovery_actions"] = append([]string(nil), lastError.RecoveryActions...)
|
|
}
|
|
if lastError.RemediationURL != "" {
|
|
eventData["remediation_url"] = lastError.RemediationURL
|
|
}
|
|
}
|
|
return s.eventBus.Publish(ctx, events.TaskSessionErrorChanged, bus.NewEvent(
|
|
events.TaskSessionErrorChanged,
|
|
"orchestrator",
|
|
eventData,
|
|
))
|
|
}
|
|
|
|
// purgeDeletedSessionQueue invalidates in-process edit state and publishes the
|
|
// queue update after the task repository commits the durable session purge.
|
|
// Request cancellation must not prevent this post-commit notification.
|
|
func (s *Service) purgeDeletedSessionQueue(ctx context.Context, taskID, sessionID string) {
|
|
cleanupCtx := context.WithoutCancel(ctx)
|
|
if s.messageQueue == nil {
|
|
return
|
|
}
|
|
s.messageQueue.InvalidateEditLeasesForSession(sessionID)
|
|
s.publishTaskQueueStatusEvent(cleanupCtx, taskID, sessionID)
|
|
}
|
|
|
|
// cancelDeletedSessionQueue removes file-backed prompt attachments left on a
|
|
// deleted session. Queue rows and their status notification are owned by the
|
|
// repository's post-commit callback when that callback is registered. Focused
|
|
// compositions without the callback retain the direct queue purge fallback.
|
|
func (s *Service) cancelDeletedSessionQueue(ctx context.Context, taskID, sessionID string) {
|
|
cleanupCtx := context.WithoutCancel(ctx)
|
|
if s.messageQueue == nil {
|
|
if s.sessionAttachmentCleaner != nil {
|
|
if err := s.sessionAttachmentCleaner.DeleteSessionMessageAttachments(cleanupCtx, taskID, sessionID); err != nil {
|
|
s.logger.Warn("failed to remove session attachment bytes after session delete",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
return
|
|
}
|
|
_ = s.messageQueue.WithSessionAdmission(cleanupCtx, sessionID, func(admittedCtx context.Context) error {
|
|
if s.sessionAttachmentCleaner != nil {
|
|
if err := s.sessionAttachmentCleaner.DeleteSessionMessageAttachments(admittedCtx, taskID, sessionID); err != nil {
|
|
s.logger.Warn("failed to remove session attachment bytes after session delete",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
if s.sessionQueuePurgeNotifierRegistered {
|
|
return nil
|
|
}
|
|
if _, err := s.messageQueue.PurgeSession(admittedCtx, sessionID); err != nil {
|
|
s.logger.Warn("failed to purge queued prompts after session delete",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
return nil
|
|
})
|
|
if !s.sessionQueuePurgeNotifierRegistered {
|
|
s.publishTaskQueueStatusEvent(cleanupCtx, taskID, sessionID)
|
|
}
|
|
}
|
|
|
|
// quiesceSessionExecutionBeforeDeletion stops the in-memory lifecycle
|
|
// execution, if one exists, before a session row is removed. A runtime that
|
|
// explicitly reports the exact execution as absent is already quiesced; other
|
|
// stop failures preserve the row because detaching a possibly-live execution
|
|
// would let trailing frames recreate activity after deletion.
|
|
func (s *Service) quiesceSessionExecutionBeforeDeletion(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
) error {
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForSession(sessionID)
|
|
}
|
|
if s.agentManager == nil {
|
|
return nil
|
|
}
|
|
executionID, executionErr := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if executionErr != nil || executionID == "" {
|
|
return nil
|
|
}
|
|
stopErr := s.executor.StopExecution(ctx, executionID, "session deleted", true)
|
|
if stopErr != nil && !errors.Is(stopErr, agentruntime.ErrNotFound) {
|
|
return fmt.Errorf("failed to stop session execution before deletion: %w", stopErr)
|
|
}
|
|
if stopErr != nil {
|
|
s.logger.Debug("session execution already absent during deletion",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("agent_execution_id", executionID),
|
|
zap.Error(stopErr))
|
|
}
|
|
s.markExecutionFailed(sessionID, executionID)
|
|
s.retireExecutionActivityAndPublish(
|
|
context.WithoutCancel(ctx), taskID, sessionID, executionID,
|
|
)
|
|
return nil
|
|
}
|
|
|
|
// promoteNextPrimaryAfterRemoval picks the best remaining session as primary
|
|
// after a session is deleted. Prefers RUNNING > active > any remaining.
|
|
func (s *Service) promoteNextPrimaryAfterRemoval(ctx context.Context, taskID, deletedSessionID string) {
|
|
sessions, err := s.repo.ListTaskSessions(ctx, taskID)
|
|
if err != nil || len(sessions) == 0 {
|
|
return
|
|
}
|
|
var candidate string
|
|
for _, sess := range sessions {
|
|
if sess.ID == deletedSessionID {
|
|
continue
|
|
}
|
|
if sess.State == models.TaskSessionStateRunning {
|
|
candidate = sess.ID
|
|
break
|
|
}
|
|
if candidate == "" {
|
|
candidate = sess.ID
|
|
} else if isActiveSessionState(sess.State) {
|
|
// Prefer active over terminal
|
|
candidate = sess.ID
|
|
}
|
|
}
|
|
if candidate != "" {
|
|
if err := s.SetPrimarySession(ctx, candidate); err != nil {
|
|
s.logger.Warn("failed to auto-promote primary after delete",
|
|
zap.String("task_id", taskID),
|
|
zap.String("candidate", candidate),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// SetPrimarySession marks a session as the primary session for its task
|
|
// and broadcasts a task.updated event so the frontend reflects the change.
|
|
func (s *Service) SetPrimarySession(ctx context.Context, sessionID string) error {
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.repo.SetSessionPrimary(ctx, sessionID); err != nil {
|
|
return fmt.Errorf("failed to set session as primary: %w", err)
|
|
}
|
|
s.publishPrimarySessionUpdate(ctx, "", sessionID)
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) publishPrimarySessionUpdate(ctx context.Context, taskID, sessionID string) {
|
|
// Broadcast task.updated so frontend updates the primary star indicator.
|
|
// The task service's publisher loads primary-session info from the DB,
|
|
// which already reflects the SetSessionPrimary write above.
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to fetch session after setting primary", zap.Error(err))
|
|
return
|
|
}
|
|
if taskID == "" {
|
|
taskID = session.TaskID
|
|
}
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to fetch task after setting primary", zap.Error(err))
|
|
return
|
|
}
|
|
s.publishTaskUpdated(ctx, task)
|
|
}
|
|
|
|
// maxSessionNameLength caps user-supplied session names to keep tab labels sane.
|
|
const maxSessionNameLength = 120
|
|
|
|
// RenameSession sets the user-supplied name for a session and broadcasts a
|
|
// session.state_changed event (same state) so all clients update the tab label.
|
|
// An empty name clears the custom label, falling back to the derived title.
|
|
func (s *Service) RenameSession(ctx context.Context, sessionID, name string) error {
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
|
|
name = strings.TrimSpace(name)
|
|
// Truncate by runes, not bytes — a byte slice could split a multi-byte
|
|
// UTF-8 sequence and persist an invalid string.
|
|
if runes := []rune(name); len(runes) > maxSessionNameLength {
|
|
name = string(runes[:maxSessionNameLength])
|
|
}
|
|
// Fetch BEFORE the write so a post-write lookup failure can never leave a
|
|
// durably renamed row without its broadcast (clients would render stale
|
|
// labels until the next full hydration).
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load session for rename: %w", err)
|
|
}
|
|
if err := s.repo.RenameTaskSession(ctx, sessionID, name); err != nil {
|
|
return fmt.Errorf("failed to rename session: %w", err)
|
|
}
|
|
session.Name = name
|
|
updatedAt := time.Now().UTC()
|
|
s.publishTaskSessionStateChanged(ctx, session.TaskID, sessionID,
|
|
session.State, session.State, session.ErrorMessage, &updatedAt, session)
|
|
return nil
|
|
}
|
|
|
|
// StopExecution stops agent execution for a specific execution ID.
|
|
func (s *Service) StopExecution(ctx context.Context, executionID string, reason string, force bool) error {
|
|
s.logger.Info("stopping execution",
|
|
zap.String("execution_id", executionID),
|
|
zap.String("reason", reason),
|
|
zap.Bool("force", force))
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForExecution(executionID)
|
|
}
|
|
return s.executor.StopExecution(ctx, executionID, reason, force)
|
|
}
|
|
|
|
// CaptureArchiveSnapshot captures git state (commits, cumulative diff) for a session before archiving.
|
|
// This preserves the final git state for historical purposes.
|
|
func (s *Service) CaptureArchiveSnapshot(ctx context.Context, sessionID string) error {
|
|
s.logger.Info("capturing archive snapshot", zap.String("session_id", sessionID))
|
|
|
|
baseCommit, baseBranch, err := s.resolveArchiveBaseCommitAndBranch(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Skip only if we have neither baseCommit nor baseBranch for merge-base calculation
|
|
if baseCommit == "" && baseBranch == "" {
|
|
s.logger.Debug("no base_commit or base_branch available, skipping archive snapshot capture",
|
|
zap.String("session_id", sessionID))
|
|
return nil
|
|
}
|
|
|
|
// Capture commits - baseBranch can be used for merge-base even if baseCommit is empty
|
|
if !s.captureCommitsForTrigger(ctx, sessionID, baseCommit, baseBranch, commitCaptureTriggerArchive) {
|
|
// Agent not running, skip diff capture as well
|
|
return nil
|
|
}
|
|
|
|
// Diff capture requires baseCommit
|
|
if baseCommit == "" {
|
|
s.logger.Debug("no base_commit available, skipping archive diff capture",
|
|
zap.String("session_id", sessionID))
|
|
return nil
|
|
}
|
|
|
|
s.captureArchiveDiff(ctx, sessionID, baseCommit)
|
|
return nil
|
|
}
|
|
|
|
// resolveArchiveBaseCommitAndBranch retrieves the base commit and branch for archive snapshot capture.
|
|
// It first checks the session's stored values, falling back to git status if empty.
|
|
func (s *Service) resolveArchiveBaseCommitAndBranch(ctx context.Context, sessionID string) (string, string, error) {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
|
|
baseCommit := session.BaseCommitSHA
|
|
baseBranch := session.BaseBranch
|
|
|
|
if baseCommit != "" {
|
|
return baseCommit, baseBranch, nil
|
|
}
|
|
|
|
// Fallback: try to get base commit from git status for legacy sessions
|
|
status, err := s.agentManager.GetGitStatus(ctx, sessionID)
|
|
if err != nil {
|
|
s.logger.Debug("failed to get git status for base commit fallback",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return "", baseBranch, nil
|
|
}
|
|
if status != nil && status.BaseCommit != "" {
|
|
s.logger.Debug("using git status base commit as fallback for archive",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("base_commit", status.BaseCommit))
|
|
return status.BaseCommit, baseBranch, nil
|
|
}
|
|
return "", baseBranch, nil
|
|
}
|
|
|
|
// captureCommitsForTrigger fetches commits from baseCommit to HEAD and
|
|
// persists them tagged with trigger (see persistSessionCommit). If
|
|
// targetBranch is provided, uses dynamic merge-base for accurate filtering.
|
|
// Returns false if the agent is not running (caller should skip remaining
|
|
// capture that also needs it, e.g. archive's diff capture).
|
|
func (s *Service) captureCommitsForTrigger(ctx context.Context, sessionID, baseCommit, targetBranch string, trigger commitCaptureTrigger) bool {
|
|
logResult, err := s.agentManager.GetGitLog(ctx, sessionID, baseCommit, 0, targetBranch) // 0 = no limit
|
|
if err != nil {
|
|
s.logger.Warn("failed to capture git log",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("trigger", string(trigger)),
|
|
zap.Error(err))
|
|
return true // Continue with diff capture even if log fails
|
|
}
|
|
if logResult == nil {
|
|
s.logger.Debug("agent not running, skipping commit capture",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("trigger", string(trigger)))
|
|
return false
|
|
}
|
|
s.recordCommitCaptureFetchFailures(sessionID, trigger, logResult)
|
|
if logResult.Success && len(logResult.Commits) > 0 {
|
|
s.persistCommits(ctx, sessionID, trigger, logResult.Commits)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// recordCommitCaptureFetchFailures makes a GetGitLog-level failure
|
|
// observable instead of silently dropping it. logResult can fail two ways
|
|
// that are otherwise indistinguishable from "nothing to capture this time":
|
|
// a total result-level failure (logResult.Success == false, single-repo git
|
|
// command failure or every repo failed in a multi-repo fan-out), or a
|
|
// partial multi-repo failure (Success == true because at least one repo
|
|
// succeeded, but PerRepoErrors names the ones that didn't). A multi-repo
|
|
// total failure sets BOTH Success == false AND populates PerRepoErrors for
|
|
// every repo (see agentctl/server/api/git.go's mergeGitLogResults), so this
|
|
// records one failure per PerRepoErrors entry when present, and only falls
|
|
// back to counting the top-level Success == false once when PerRepoErrors is
|
|
// empty (the single-repo case) - never both, to avoid double-counting.
|
|
func (s *Service) recordCommitCaptureFetchFailures(sessionID string, trigger commitCaptureTrigger, logResult *client.GitLogResult) {
|
|
if len(logResult.PerRepoErrors) > 0 {
|
|
for _, repoErr := range logResult.PerRepoErrors {
|
|
recordCommitCaptureFetchFailed(trigger)
|
|
s.logger.Warn("git log capture failed for repository",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("trigger", string(trigger)),
|
|
zap.String("repository_name", repoErr.RepositoryName),
|
|
zap.String("error", repoErr.Error))
|
|
}
|
|
return
|
|
}
|
|
if !logResult.Success {
|
|
recordCommitCaptureFetchFailed(trigger)
|
|
s.logger.Warn("git log capture reported failure",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("trigger", string(trigger)),
|
|
zap.String("error", logResult.Error))
|
|
}
|
|
}
|
|
|
|
// captureSessionCommitsSweep reconciles task_session_commits against
|
|
// agentctl while the agent process is still alive. The live event-driven
|
|
// path (handleGitCommitCreated) can miss a commit that was pushed just
|
|
// before agentctl's next poll tick (see filterLocalCommits' upstream-commit
|
|
// filter), and this sweep is the only trigger guaranteed to run with the
|
|
// agent still running - unlike archive capture, whose GetGitLog call
|
|
// usually finds the agent process already gone. Cost is one
|
|
// `git log --shortstat base..HEAD`, the same command agentctl's own poller
|
|
// already runs on every tick. Best-effort: errors are logged and swallowed,
|
|
// matching captureGitStatusSnapshot alongside which this is called.
|
|
//
|
|
// Called from handleAgentCompletedLocked while it holds the per-session
|
|
// cancel-in-flight mutex (acquireCancelInFlightGuard) - the same mutex
|
|
// stopTaskSessionForCoordinator and DeleteSession need to serve a user's
|
|
// Stop/Cancel/Delete for this session. GetGitLog is a real git shellout
|
|
// through agentctl with no bound of its own beyond the agentctl HTTP
|
|
// client's 60s default timeout, so an undecorated ctx here could hold that
|
|
// mutex - and therefore Stop/Cancel/Delete - for up to a minute on every
|
|
// single turn completion. Bound it the same way the pre-existing archive
|
|
// capture call site already does (service_tasks.go's CaptureArchiveSnapshot:
|
|
// "Use a bounded timeout to prevent blocking the archive operation if
|
|
// agentctl is stuck"). A timeout here falls through the existing
|
|
// warn-and-continue path in captureCommitsForTrigger; archive capture
|
|
// remains the reconciliation pass, so nothing is permanently lost.
|
|
func (s *Service) captureSessionCommitsSweep(ctx context.Context, sessionID string) {
|
|
sweepCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
|
|
baseCommit, baseBranch, err := s.resolveArchiveBaseCommitAndBranch(sweepCtx, sessionID)
|
|
if err != nil {
|
|
s.logger.Debug("failed to resolve base commit for commit sweep",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if baseCommit == "" && baseBranch == "" {
|
|
return
|
|
}
|
|
s.captureCommitsForTrigger(sweepCtx, sessionID, baseCommit, baseBranch, commitCaptureTriggerSweep)
|
|
}
|
|
|
|
// captureGitStatusSnapshot fetches the current (cached) git status from agentctl
|
|
// and saves it as a DB snapshot. Called when a session's execution completes so
|
|
// the status is available when clients subscribe later.
|
|
func (s *Service) captureGitStatusSnapshot(ctx context.Context, sessionID string) {
|
|
_, _ = s.saveGitStatusSnapshot(ctx, sessionID, false)
|
|
}
|
|
|
|
// captureGitStatusSnapshotWithRetry attempts a fresh capture with up to 3
|
|
// retries at 1-second intervals if the first attempt returns stale 0/0 data
|
|
// (caused by git lock contention between concurrent worktrees). Returns
|
|
// immediately without retrying when the environment or execution is missing,
|
|
// or when the agent genuinely has no file changes.
|
|
func (s *Service) captureGitStatusSnapshotWithRetry(ctx context.Context, sessionID string) {
|
|
for attempt := 0; attempt < 3; attempt++ {
|
|
if attempt > 0 {
|
|
time.Sleep(time.Second)
|
|
}
|
|
wrote, noExec := s.saveGitStatusSnapshot(ctx, sessionID, true)
|
|
if noExec {
|
|
return // No execution exists — retrying won't help
|
|
}
|
|
if wrote {
|
|
return // Successfully wrote a snapshot (may be 0/0 for no-change turns, that's fine)
|
|
}
|
|
// saveGitStatusSnapshot returned false without noExec — means fresh
|
|
// returned 0/0 and was skipped due to possible lock contention. Retry.
|
|
}
|
|
}
|
|
|
|
// saveGitStatusSnapshot is the shared implementation for snapshot capture.
|
|
// When fresh is true, it bypasses the workspace tracker's poll cache.
|
|
// Returns (wrote, noExecution): wrote=true if a snapshot was persisted,
|
|
// noExecution=true if the environment or active execution is unavailable.
|
|
func (s *Service) saveGitStatusSnapshot(ctx context.Context, sessionID string, fresh bool) (wrote, noExecution bool) {
|
|
taskEnvironmentID, ok := s.resolveGitSnapshotEnvironmentID(ctx, sessionID)
|
|
if !ok {
|
|
// A snapshot cannot be captured without an environment. Stop the
|
|
// retry loop because another attempt cannot resolve this identity.
|
|
return false, true
|
|
}
|
|
var status *client.GitStatusResult
|
|
var err error
|
|
if fresh {
|
|
status, err = s.agentManager.GetGitStatusFresh(ctx, sessionID)
|
|
} else {
|
|
status, err = s.agentManager.GetGitStatus(ctx, sessionID)
|
|
}
|
|
if err != nil {
|
|
s.logger.Debug("failed to capture git status snapshot",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return false, false
|
|
}
|
|
if status == nil {
|
|
return false, true // No execution — caller should not retry
|
|
}
|
|
if !status.Success {
|
|
return false, false
|
|
}
|
|
|
|
// When a fresh query returns zero additions AND zero deletions, the result
|
|
// may be stale due to git lock contention between concurrent worktrees
|
|
// (merge-base computation fails transiently). Don't overwrite a potentially
|
|
// better live_monitor snapshot that already has the correct non-zero values.
|
|
if fresh && status.BranchAdditions == 0 && status.BranchDeletions == 0 {
|
|
return false, false
|
|
}
|
|
|
|
metadata := map[string]interface{}{
|
|
"repository_name": status.RepositoryName,
|
|
"timestamp": status.Timestamp,
|
|
"modified": status.Modified,
|
|
"added": status.Added,
|
|
"deleted": status.Deleted,
|
|
"untracked": status.Untracked,
|
|
"renamed": status.Renamed,
|
|
"branch_additions": status.BranchAdditions,
|
|
"branch_deletions": status.BranchDeletions,
|
|
"comparison_target": status.ComparisonTarget,
|
|
"comparison_status": status.ComparisonStatus,
|
|
"comparison_error_code": status.ComparisonErrorCode,
|
|
}
|
|
|
|
if err := s.repo.CreateGitSnapshot(ctx, &models.GitSnapshot{
|
|
TaskEnvironmentID: taskEnvironmentID,
|
|
SessionID: sessionID,
|
|
SnapshotType: models.SnapshotTypeStatusUpdate,
|
|
Branch: status.Branch,
|
|
RemoteBranch: status.RemoteBranch,
|
|
HeadCommit: status.HeadCommit,
|
|
BaseCommit: status.BaseCommit,
|
|
Ahead: status.Ahead,
|
|
Behind: status.Behind,
|
|
Files: status.Files,
|
|
TriggeredBy: gitSnapshotTriggeredByAgentCompleted,
|
|
Metadata: metadata,
|
|
}); err != nil {
|
|
s.logger.Warn("failed to save git status snapshot",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return false, false
|
|
}
|
|
|
|
s.logger.Debug("saved git status snapshot",
|
|
zap.String("task_environment_id", taskEnvironmentID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("branch", status.Branch),
|
|
zap.Bool("fresh", fresh))
|
|
return true, false
|
|
}
|
|
|
|
// captureArchiveDiff fetches and saves the cumulative diff from baseCommit to the working tree
|
|
// (including uncommitted/unstaged changes).
|
|
func (s *Service) captureArchiveDiff(ctx context.Context, sessionID, baseCommit string) {
|
|
taskEnvironmentID, ok := s.resolveGitSnapshotEnvironmentID(ctx, sessionID)
|
|
if !ok {
|
|
return
|
|
}
|
|
diffResult, err := s.agentManager.GetCumulativeDiff(ctx, sessionID, baseCommit)
|
|
if err != nil {
|
|
s.logger.Warn("failed to capture cumulative diff for archive",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if diffResult == nil || !diffResult.Success {
|
|
return
|
|
}
|
|
|
|
snapshot := &models.GitSnapshot{
|
|
TaskEnvironmentID: taskEnvironmentID,
|
|
SessionID: sessionID,
|
|
SnapshotType: models.SnapshotTypeArchive,
|
|
HeadCommit: diffResult.HeadCommit,
|
|
BaseCommit: diffResult.BaseCommit,
|
|
Files: diffResult.Files,
|
|
}
|
|
status, statusErr := s.agentManager.GetGitStatusFresh(ctx, sessionID)
|
|
if statusErr != nil {
|
|
s.logger.Warn("failed to capture git status metadata for archive",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(statusErr))
|
|
} else if status != nil && status.Success {
|
|
snapshot.Branch = status.Branch
|
|
snapshot.RemoteBranch = status.RemoteBranch
|
|
snapshot.Ahead = status.Ahead
|
|
snapshot.Behind = status.Behind
|
|
snapshot.Metadata = archiveGitStatusMetadata(status, diffResult.Files)
|
|
}
|
|
|
|
if err := s.repo.CreateGitSnapshot(ctx, snapshot); err != nil {
|
|
s.logger.Warn("failed to save archive snapshot",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
|
|
s.logger.Debug("saved archive snapshot",
|
|
zap.String("task_environment_id", taskEnvironmentID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("head_commit", diffResult.HeadCommit),
|
|
zap.Int("total_commits", diffResult.TotalCommits))
|
|
}
|
|
|
|
func archiveGitStatusMetadata(status *client.GitStatusResult, files map[string]interface{}) map[string]interface{} {
|
|
if status == nil {
|
|
return nil
|
|
}
|
|
return map[string]interface{}{
|
|
"timestamp": status.Timestamp,
|
|
// The archive's files map is a cumulative diff, while these status
|
|
// lists describe the working tree at capture time. Keep an explicit
|
|
// count for summary consumers so they do not add two different views.
|
|
"changed_files": len(files),
|
|
"modified": status.Modified,
|
|
"added": status.Added,
|
|
"deleted": status.Deleted,
|
|
"untracked": status.Untracked,
|
|
"renamed": status.Renamed,
|
|
"remote_ahead": status.RemoteAhead,
|
|
"remote_behind": status.RemoteBehind,
|
|
"remote_head_commit": status.RemoteHeadCommit,
|
|
"branch_additions": status.BranchAdditions,
|
|
"branch_deletions": status.BranchDeletions,
|
|
}
|
|
}
|
|
|
|
// parseCommitTime parses a commit timestamp from git log output.
|
|
// Returns UTC time to ensure consistent timestamps across environments.
|
|
func parseCommitTime(s string) time.Time {
|
|
t, err := time.Parse(time.RFC3339, s)
|
|
if err != nil {
|
|
return time.Now().UTC()
|
|
}
|
|
return t.UTC()
|
|
}
|
|
|
|
// persistCommits persists a batch of commits fetched via GetGitLog, tagged
|
|
// with trigger. Shared by archive capture and the per-turn reconcile sweep
|
|
// (captureSessionCommitsSweep) - see persistSessionCommit for the
|
|
// idempotent single-commit write and writer-health counters.
|
|
func (s *Service) persistCommits(ctx context.Context, sessionID string, trigger commitCaptureTrigger, commits []*client.GitCommitInfo) {
|
|
for _, commit := range commits {
|
|
s.persistSessionCommit(ctx, sessionID, trigger, &models.SessionCommit{
|
|
CommitSHA: commit.CommitSHA,
|
|
ParentSHA: commit.ParentSHA,
|
|
AuthorName: commit.AuthorName,
|
|
AuthorEmail: commit.AuthorEmail,
|
|
CommitMessage: commit.CommitMessage,
|
|
CommittedAt: parseCommitTime(commit.CommittedAt),
|
|
FilesChanged: commit.FilesChanged,
|
|
Insertions: commit.Insertions,
|
|
Deletions: commit.Deletions,
|
|
})
|
|
}
|
|
s.logger.Debug("persisted commits",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("trigger", string(trigger)),
|
|
zap.Int("count", len(commits)))
|
|
}
|
|
|
|
// persistSessionCommit is the single write path shared by the live commit
|
|
// event (handleGitCommitCreated), the per-turn reconcile sweep
|
|
// (captureSessionCommitsSweep), and archive capture (saveArchiveCommits). It
|
|
// records writer-health counters (commit_capture_*, see
|
|
// commit_capture_metrics.go) so a trigger that silently stops firing is
|
|
// observable, and treats CreateSessionCommit reporting a duplicate
|
|
// (session_id, commit_sha) as a normal outcome, not a failure - the same
|
|
// underlying git commit is expected to be observed by more than one trigger.
|
|
func (s *Service) persistSessionCommit(ctx context.Context, sessionID string, trigger commitCaptureTrigger, commit *models.SessionCommit) {
|
|
if s.repo == nil {
|
|
return
|
|
}
|
|
recordCommitCaptureObserved(trigger)
|
|
commit.SessionID = sessionID
|
|
inserted, err := s.repo.CreateSessionCommit(ctx, commit)
|
|
if err != nil {
|
|
recordCommitCaptureFailed(trigger)
|
|
s.logger.Warn("failed to persist session commit",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("commit_sha", commit.CommitSHA),
|
|
zap.String("trigger", string(trigger)),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if inserted {
|
|
recordCommitCaptureInserted(trigger)
|
|
} else {
|
|
recordCommitCaptureDuplicate(trigger)
|
|
}
|
|
}
|
|
|
|
// PromptTask sends a follow-up prompt to a running agent for a task session.
|
|
// If planMode is true, a plan mode prefix is prepended to the prompt.
|
|
// Attachments (images) are passed through to the agent if provided.
|
|
func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment, dispatchOnly bool) (*PromptResult, error) {
|
|
return s.promptTask(ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, launchOriginManual, promptTaskOptions{})
|
|
}
|
|
|
|
type promptTaskOptions struct {
|
|
claimEntryID string
|
|
lifecyclePrompt bool
|
|
afterClaim func() error
|
|
afterDispatch func() error
|
|
// beforeDispatch runs once before the final dispatch admission boundary.
|
|
beforeDispatch func() error
|
|
// afterDispatchAdmission runs once after final dispatch admission succeeds
|
|
// and before the first provider or model-switch I/O that can carry the
|
|
// prompt. Queue receipts use this boundary because admission can reject a
|
|
// stale claim.
|
|
afterDispatchAdmission func() error
|
|
disableDispatchRetry bool
|
|
preservePromptContext bool
|
|
// reserveTurnUntilDispatch persists detached-resume ownership before agentctl
|
|
// dispatch, while delaying the visible turn.started event until acceptance.
|
|
reserveTurnUntilDispatch bool
|
|
promptDispatchRecovery *models.PromptDispatchRecovery
|
|
expectedCurrentTurnID string
|
|
requireNonterminalSession bool
|
|
// onAccepted runs at the agentctl acceptance boundary, before PromptTask
|
|
// waits for the turn to finish. Automation callers use it to bind durable
|
|
// attempt identity to the exact turn.
|
|
onAccepted func(turnID string)
|
|
// promptAccepted is owned by promptTask and keeps the replay cache alive
|
|
// only when this prompt reaches provider acceptance. It is process-local
|
|
// plumbing and is never passed to a caller.
|
|
promptAccepted *atomic.Bool
|
|
expectedSessionIdentity *messagequeue.QueueSessionIdentity
|
|
// configModeOverride preserves the launch-time mode for a deferred
|
|
// workflow prompt whose raw queue content is intentionally empty.
|
|
configModeOverride *bool
|
|
// promptAlreadyComposed and fallbackRetryPrompt mirror the composed-prompt
|
|
// seam autoStartStepPrompt's own ErrExecutionNotFound branch uses (see
|
|
// fallbackFreshLaunchOnMissingExecution). When promptAlreadyComposed is
|
|
// true, handlePromptDispatchFailure's own internal ErrExecutionNotFound
|
|
// recovery relaunches via startCreatedSessionWithComposedPrompt
|
|
// (fallbackRetryPrompt as its dispatch value) instead of the public
|
|
// StartCreatedSession, so a caller that already composed the prompt itself
|
|
// (e.g. appending a claimed step handoff) is not silently recomposed from
|
|
// the destination step's own template.
|
|
promptAlreadyComposed bool
|
|
// fallbackUsesEffectivePrompt upgrades an already-composed recovery prompt
|
|
// with the session's effective plan/config transforms before fresh launch.
|
|
fallbackUsesEffectivePrompt bool
|
|
// initialCreatePromptPassthrough keeps a creation-admission marker alive
|
|
// while a transient retry creates the next turn, then rebinds it to the
|
|
// execution admitted for that retry before provider dispatch.
|
|
initialCreatePromptPassthrough bool
|
|
// fallbackLaunchPrompt is the fully composed prompt for fresh-launch
|
|
// recovery. The normal dispatch still receives the raw prompt so it can
|
|
// apply session transforms exactly once.
|
|
fallbackLaunchPrompt string
|
|
fallbackRetryPrompt string
|
|
// promptReferenceContext is the exact expansion returned while composing
|
|
// this workflow entry. Recovery uses it to preserve the trusted block.
|
|
promptReferenceContext string
|
|
// resumeAttempt keeps a compound resume-and-prompt operation under one
|
|
// ownership record. The outer resume operation finishes it after provider
|
|
// acceptance or the retry's terminal result.
|
|
resumeAttempt *resumeAttempt
|
|
// ceilingEntryBinding pins replayed workflow work to the committed route
|
|
// and destination that admitted it.
|
|
ceilingEntryBinding *models.CeilingWorkflowEntryBinding
|
|
}
|
|
|
|
type promptDispatchOutcome struct {
|
|
mu sync.Mutex
|
|
accepted bool
|
|
publicationErr error
|
|
turnID string
|
|
onAccepted func(turnID string)
|
|
}
|
|
|
|
func (o *promptDispatchOutcome) recordAccepted(publicationErr error) {
|
|
if o == nil {
|
|
return
|
|
}
|
|
o.mu.Lock()
|
|
o.accepted = true
|
|
o.publicationErr = publicationErr
|
|
o.mu.Unlock()
|
|
}
|
|
|
|
func (o *promptDispatchOutcome) snapshot() (bool, error) {
|
|
if o == nil {
|
|
return false, nil
|
|
}
|
|
o.mu.Lock()
|
|
defer o.mu.Unlock()
|
|
return o.accepted, o.publicationErr
|
|
}
|
|
|
|
// acceptedPromptDispatchError means agentctl accepted the prompt, but durable
|
|
// publication or later transport handling failed. Detached clarification
|
|
// callers must not reopen the already-delivered answer on this error.
|
|
type acceptedPromptDispatchError struct {
|
|
err error
|
|
}
|
|
|
|
func (e *acceptedPromptDispatchError) Error() string {
|
|
return fmt.Sprintf("prompt accepted but post-dispatch handling failed: %v", e.err)
|
|
}
|
|
|
|
func (e *acceptedPromptDispatchError) Unwrap() error { return e.err }
|
|
|
|
func (*acceptedPromptDispatchError) DetachedResumeAccepted() bool { return true }
|
|
|
|
func wrapAcceptedPromptDispatchFailure(
|
|
dispatchAccepted bool,
|
|
failureErr, publicationErr error,
|
|
) error {
|
|
if !dispatchAccepted {
|
|
return failureErr
|
|
}
|
|
combined := errors.Join(failureErr, publicationErr)
|
|
if combined == nil {
|
|
return nil
|
|
}
|
|
return &acceptedPromptDispatchError{err: combined}
|
|
}
|
|
|
|
func (options promptTaskOptions) executorContext(ctx context.Context) context.Context {
|
|
if options.preservePromptContext {
|
|
return ctx
|
|
}
|
|
return context.WithoutCancel(ctx)
|
|
}
|
|
|
|
func (promptTaskOptions) failureContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
|
// Preserve a small bounded window for state rollback after either the
|
|
// dispatch deadline or an ordinary transport request expires; otherwise a
|
|
// cancelled caller context can leave RUNNING state behind.
|
|
return context.WithTimeout(context.WithoutCancel(ctx), promptFailureCleanupTimeout)
|
|
}
|
|
|
|
// promptTask is PromptTask's implementation. Its options carry queued-dispatch
|
|
// ownership and the bounded-context exception. When options.claimEntryID is
|
|
// non-empty, this acquires sessionID's cancelInFlight guard around a second
|
|
// ownership check and the "mark the session RUNNING" step immediately before
|
|
// startTurnForSession and the (potentially long-blocking) executor.Prompt call
|
|
// below it. The worker claims the handoff before visible side effects; this
|
|
// check keeps the final session transition tied to that same ownership record.
|
|
//
|
|
// A bare (unguarded) "check the token, then separately call
|
|
// setSessionRunning" would still let two settling dispatches for the same
|
|
// session both pass their own check before either actually marks the
|
|
// session RUNNING — the exact double-dispatch this mechanism exists to
|
|
// prevent, just narrowed rather than closed. Serializing the check
|
|
// *and* the mark through the same per-session guard used for every other
|
|
// cancel/take-and-dispatch decision (see the Service.cancelInFlight field
|
|
// doc comment) makes "exactly one dispatch wins" a property of mutual
|
|
// exclusion instead of a racy read. Direct prompts release the guard after
|
|
// this fast, DB-only step. Queued prompts transfer the guard to promptTask and
|
|
// release it at provider acceptance, while lifecycle prompts hold the same
|
|
// guard until agentctl accepts the prompt. Neither path lets cancellation
|
|
// supersede a claimed queued turn in the gap before provider dispatch.
|
|
//
|
|
// Every return path *before* this step (invalid session, not promptable,
|
|
// ensureSessionRunning failure, a model switch) is covered by
|
|
// executeQueuedMessage's own deferred cleanup instead
|
|
// (clearQueuedDispatchInFlightIfCurrent), which is safe since none of them
|
|
// block on an agent turn.
|
|
func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment, dispatchOnly bool, origin launchOrigin, options promptTaskOptions) (*PromptResult, error) {
|
|
s.logPromptTaskCall(taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly)
|
|
if options.ceilingEntryBinding == nil {
|
|
options.ceilingEntryBinding = ceilingEntryBindingFromContext(ctx)
|
|
}
|
|
if options.ceilingEntryBinding != nil {
|
|
ctx = withCeilingEntryBinding(ctx, options.ceilingEntryBinding)
|
|
}
|
|
if bindingErr := s.validateContextCeilingEntry(ctx, taskID); bindingErr != nil {
|
|
return nil, bindingErr
|
|
}
|
|
// Check resume-attempt ownership before any repository read can observe a
|
|
// compound resume's cancellation as a generic context error.
|
|
if err := s.validatePromptTaskPreconditions(sessionID, options.resumeAttempt); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
session, foregroundClaim, err := s.prepareSessionAndForegroundClaimForPrompt(ctx, taskID, sessionID, options)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// After a lazy backend restart the session may be WAITING_FOR_INPUT with no agent process yet.
|
|
_, hadExecutionBeforeEnsure := s.executor.GetExecutionBySession(sessionID)
|
|
resumedForPrompt := options.resumeAttempt != nil || !hadExecutionBeforeEnsure
|
|
resumeAttempt, finishResumeAttempt, resumePromptCtx, err := s.resolveAndAdmitResumeAttempt(
|
|
ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, origin, options, session, foregroundClaim,
|
|
)
|
|
if finishResumeAttempt != nil {
|
|
defer finishResumeAttempt()
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Reload the session so executor.Prompt uses the freshly-persisted
|
|
// AgentExecutionID, then begin the foreground dispatch on it.
|
|
session, effectivePrompt, foregroundDispatch, err := s.beginForegroundDispatchForPrompt(
|
|
resumePromptCtx, taskID, sessionID, prompt, planMode, session, foregroundClaim,
|
|
options.configModeOverride,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if options.fallbackUsesEffectivePrompt {
|
|
options.fallbackLaunchPrompt = effectivePrompt
|
|
options.fallbackRetryPrompt = effectivePrompt
|
|
}
|
|
// A queued prompt may restart the provider as part of model switching
|
|
// before the normal claim helper runs. Keep the same session guard across
|
|
// that I/O and transfer it into the claim so Send Now cannot admit a
|
|
// replacement between the restart and provider acceptance. Resume-owned
|
|
// switches already hold this boundary through modelSwitchGuard below.
|
|
var queuedDispatchGuard *lockedCancelInFlightGuard
|
|
if options.claimEntryID != "" && !options.lifecyclePrompt && options.resumeAttempt == nil {
|
|
queuedDispatchGuard = s.lockCancelInFlightGuard(sessionID)
|
|
defer func() {
|
|
if queuedDispatchGuard != nil {
|
|
queuedDispatchGuard.release()
|
|
}
|
|
}()
|
|
if err := s.waitForCancellationWithGuard(
|
|
resumePromptCtx, sessionID, queuedDispatchGuard.unlock, queuedDispatchGuard.relock,
|
|
); err != nil {
|
|
s.rollbackForegroundDispatchOnFailure(resumePromptCtx, taskID, sessionID, foregroundDispatch)
|
|
return nil, err
|
|
}
|
|
}
|
|
runBeforeDispatch := runBeforeDispatchOnce(options.beforeDispatch)
|
|
runAfterDispatchAdmission := runBeforeDispatchOnce(options.afterDispatchAdmission)
|
|
|
|
// Keep the replay payload only after this provider attempt is accepted.
|
|
// Admission and dispatch failures must not retain prompt attachments until
|
|
// a later session event happens to replace or consume the cache.
|
|
var promptAccepted atomic.Bool
|
|
options.promptAccepted = &promptAccepted
|
|
defer func() {
|
|
if !promptAccepted.Load() {
|
|
s.lastTurnPrompt.Delete(sessionID)
|
|
}
|
|
}()
|
|
|
|
// Cache the replay identity and acquire the model-switch guard before switching.
|
|
modelSwitchGuard, err := s.prepareModelSwitchGuard(
|
|
resumePromptCtx, taskID, sessionID, prompt, model, planMode, attachments, options, session,
|
|
foregroundDispatch, resumeAttempt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if modelSwitchGuard != nil {
|
|
defer func() {
|
|
if modelSwitchGuard != nil {
|
|
modelSwitchGuard.release()
|
|
}
|
|
}()
|
|
}
|
|
|
|
switchResult, switchHandled, modelSwitchGuard, switchErr := s.resolveModelSwitchAttempt(
|
|
resumePromptCtx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, runBeforeDispatch,
|
|
runAfterDispatchAdmission,
|
|
resumeAttempt, modelSwitchGuard,
|
|
)
|
|
if switchHandled {
|
|
if switchErr == nil {
|
|
promptAccepted.Store(true)
|
|
}
|
|
return switchResult, switchErr
|
|
}
|
|
|
|
session, rollback, releaseDispatchGuard, err := s.claimAndGuardDispatch(
|
|
resumePromptCtx, taskID, sessionID, foregroundClaim, options, foregroundDispatch, resumeAttempt,
|
|
queuedDispatchGuard,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if releaseDispatchGuard != nil {
|
|
// The agentctl dispatch callback is the acceptance boundary; releasing here keeps
|
|
// terminalization exclusive with admission without holding the guard for the whole turn.
|
|
defer releaseDispatchGuard()
|
|
}
|
|
|
|
return s.runPromptTurn(
|
|
resumePromptCtx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments, dispatchOnly,
|
|
effectivePrompt, session, rollback, options, foregroundDispatch, runBeforeDispatch,
|
|
runAfterDispatchAdmission,
|
|
releaseDispatchGuard, resumeAttempt,
|
|
)
|
|
}
|
|
|
|
// runPromptTurn is promptTask's dispatch tail: it validates and runs the
|
|
// dispatch boundary, prepares the dispatch callback, invokes the executor,
|
|
// and folds the outcome into promptTask's single PromptResult/error return.
|
|
func (s *Service) runPromptTurn(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt string,
|
|
planMode, resumedForPrompt bool,
|
|
attachments []v1.MessageAttachment,
|
|
dispatchOnly bool,
|
|
effectivePrompt string,
|
|
session *models.TaskSession,
|
|
rollback promptClaimRollback,
|
|
options promptTaskOptions,
|
|
foregroundDispatch *foregroundDispatch,
|
|
runBeforeDispatch func() error,
|
|
runAfterDispatchAdmission func() error,
|
|
releaseDispatchGuard func(),
|
|
resumeAttempt *resumeAttempt,
|
|
) (*PromptResult, error) {
|
|
promptCtx, session, earlyResult, err := s.validateAndRunDispatchBoundary(
|
|
ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments,
|
|
session, rollback, options, foregroundDispatch, runBeforeDispatch, runAfterDispatchAdmission, releaseDispatchGuard,
|
|
)
|
|
if err != nil || earlyResult != nil {
|
|
return earlyResult, err
|
|
}
|
|
dispatchCtx, releaseCeilingDispatch, err := s.commitCeilingEntryDispatch(
|
|
promptCtx, taskID, ceilingEntryBindingFromContext(promptCtx),
|
|
)
|
|
if err != nil {
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
return nil, s.failPromptDispatch(
|
|
ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, err,
|
|
)
|
|
}
|
|
defer releaseCeilingDispatch()
|
|
promptCtx = dispatchCtx
|
|
onDispatched, dispatchOutcome := s.preparePromptDispatchCallback(
|
|
promptCtx, taskID, sessionID, session, rollback, options, foregroundDispatch, releaseDispatchGuard,
|
|
resumeAttempt,
|
|
)
|
|
result, execErr := s.executor.PromptWithDispatchCallback(
|
|
promptCtx, taskID, sessionID, effectivePrompt, attachments, dispatchOnly,
|
|
onDispatched, session,
|
|
)
|
|
return s.finishPromptExecutorDispatch(
|
|
ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments,
|
|
rollback, options, foregroundDispatch, releaseDispatchGuard, result, execErr, dispatchOutcome, resumeAttempt,
|
|
)
|
|
}
|
|
|
|
// resolveResumeAttempt returns the requested resume attempt unchanged when
|
|
// the caller already holds one (options.resumeAttempt, a compound resume's
|
|
// own cancellable attempt), or acquires a fresh one via
|
|
// ensureSessionRunningWithAttempt otherwise. owns reports whether promptTask
|
|
// itself is responsible for finishing the returned attempt.
|
|
func (s *Service) resolveResumeAttempt(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
origin launchOrigin,
|
|
requested *resumeAttempt,
|
|
) (attempt *resumeAttempt, owns bool, err error) {
|
|
if requested != nil {
|
|
return requested, false, nil
|
|
}
|
|
attempt, err = s.ensureSessionRunningWithAttempt(ctx, sessionID, session, origin)
|
|
return attempt, attempt != nil, err
|
|
}
|
|
|
|
// resolveAndAdmitResumeAttempt resolves promptTask's resume attempt and admits
|
|
// it in one step, returning a finish func for the caller to defer (nil when
|
|
// promptTask does not own the attempt) instead of deferring it here, since a
|
|
// defer inside this helper would fire when this helper returns rather than
|
|
// when promptTask itself does.
|
|
func (s *Service) resolveAndAdmitResumeAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt, model string,
|
|
planMode bool,
|
|
attachments []v1.MessageAttachment,
|
|
dispatchOnly bool,
|
|
origin launchOrigin,
|
|
options promptTaskOptions,
|
|
session *models.TaskSession,
|
|
foregroundClaim *foregroundClaim,
|
|
) (resumeAttempt *resumeAttempt, finish func(), resumePromptCtx context.Context, err error) {
|
|
resumeAttempt, owns, err := s.resolveResumeAttempt(ctx, sessionID, session, origin, options.resumeAttempt)
|
|
if resumeAttempt != nil && owns {
|
|
finish = func() { resumeAttempt.finish(s.resumeAttemptStore()) }
|
|
}
|
|
resumePromptCtx, err = s.admitResumeAttemptForPrompt(
|
|
ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, options,
|
|
foregroundClaim, resumeAttempt, err,
|
|
)
|
|
return resumeAttempt, finish, resumePromptCtx, err
|
|
}
|
|
|
|
// runBeforeDispatchOnce wraps a caller-supplied pre-dispatch hook so it runs
|
|
// at most once: nil beforeDispatch is a no-op, and a successful call clears
|
|
// the hook so a later retry within the same promptTask attempt does not run
|
|
// it again.
|
|
func runBeforeDispatchOnce(beforeDispatch func() error) func() error {
|
|
return func() error {
|
|
if beforeDispatch == nil {
|
|
return nil
|
|
}
|
|
if err := beforeDispatch(); err != nil {
|
|
return err
|
|
}
|
|
beforeDispatch = nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// resolveModelSwitchAttempt runs attemptModelSwitchForPrompt and folds its
|
|
// three-way outcome (not handled / handled-but-resume-attempt-lost /
|
|
// handled) together with modelSwitchGuard's release into promptTask's single
|
|
// remaining decision: whether to return switchResult/switchErr immediately.
|
|
// The returned modelSwitchGuard reflects the same release-then-nil update
|
|
// promptTask's own deferred cleanup closure observes, since Go closures
|
|
// capture the reassigned variable, not a snapshot of it.
|
|
func (s *Service) resolveModelSwitchAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID, model, effectivePrompt string,
|
|
session *models.TaskSession,
|
|
foregroundDispatch *foregroundDispatch,
|
|
runBeforeDispatch func() error,
|
|
runAfterDispatchAdmission func() error,
|
|
resumeAttempt *resumeAttempt,
|
|
modelSwitchGuard *lockedCancelInFlightGuard,
|
|
) (switchResult *PromptResult, handled bool, remainingGuard *lockedCancelInFlightGuard, switchErr error) {
|
|
result, handledSwitch, attemptErr := s.attemptModelSwitchForPrompt(
|
|
ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, runBeforeDispatch,
|
|
runAfterDispatchAdmission, resumeAttempt,
|
|
)
|
|
if !handledSwitch {
|
|
if modelSwitchGuard != nil {
|
|
modelSwitchGuard.release()
|
|
modelSwitchGuard = nil
|
|
}
|
|
return nil, false, modelSwitchGuard, nil
|
|
}
|
|
if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
return nil, true, modelSwitchGuard, resumeErr
|
|
}
|
|
if modelSwitchGuard != nil {
|
|
modelSwitchGuard.release()
|
|
modelSwitchGuard = nil
|
|
}
|
|
return result, true, modelSwitchGuard, attemptErr
|
|
}
|
|
|
|
// acquireModelSwitchGuard locks the resume attempt's admission guard for a
|
|
// prompt that both resumes a session and switches its model, when both
|
|
// conditions apply. It returns (nil, nil) when no guard is needed.
|
|
func (s *Service) acquireModelSwitchGuard(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
foregroundDispatch *foregroundDispatch,
|
|
requiresModelSwitch bool,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*lockedCancelInFlightGuard, error) {
|
|
if !requiresModelSwitch || resumeAttempt == nil {
|
|
return nil, nil
|
|
}
|
|
guard, err := s.lockResumeAttemptAdmission(ctx, sessionID, resumeAttempt)
|
|
if err != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch)
|
|
return nil, err
|
|
}
|
|
return guard, nil
|
|
}
|
|
|
|
// prepareModelSwitchGuard caches and reserves the replay identity for the turn,
|
|
// then acquires the model-switch admission guard, if one is needed, in one
|
|
// step. Both must happen before any model-switch attempt.
|
|
func (s *Service) prepareModelSwitchGuard(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt, model string,
|
|
planMode bool,
|
|
attachments []v1.MessageAttachment,
|
|
options promptTaskOptions,
|
|
session *models.TaskSession,
|
|
foregroundDispatch *foregroundDispatch,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*lockedCancelInFlightGuard, error) {
|
|
s.rememberTurnPromptWithAccepted(sessionID, prompt, model, planMode, attachments, options.onAccepted)
|
|
requiresModelSwitch := modelSwitchRequired(session, model)
|
|
return s.acquireModelSwitchGuard(ctx, taskID, sessionID, foregroundDispatch, requiresModelSwitch, resumeAttempt)
|
|
}
|
|
|
|
// admitResumeAttemptForPrompt resolves promptTask's resume-scoped context and
|
|
// validates the resume attempt both before and after handling ensureErr (the
|
|
// error from ensureSessionRunningWithAttempt), so a cancellation landing in
|
|
// either window is caught rather than only the one checked first. ensureErr,
|
|
// when non-nil, is classified via classifyEnsureSessionRunningFailureForPrompt.
|
|
func (s *Service) admitResumeAttemptForPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
prompt, model string,
|
|
planMode bool,
|
|
attachments []v1.MessageAttachment,
|
|
dispatchOnly bool,
|
|
options promptTaskOptions,
|
|
foregroundClaim *foregroundClaim,
|
|
resumeAttempt *resumeAttempt,
|
|
ensureErr error,
|
|
) (context.Context, error) {
|
|
resumePromptCtx := ctx
|
|
if resumeAttempt != nil {
|
|
resumePromptCtx = cancellableResumeContext(resumeAttempt)
|
|
if err := s.failPromptOnResumeAttempt(resumePromptCtx, taskID, sessionID, foregroundClaim, resumeAttempt); err != nil {
|
|
return resumePromptCtx, err
|
|
}
|
|
}
|
|
if ensureErr != nil {
|
|
s.releaseForegroundClaimOnFailure(resumePromptCtx, taskID, sessionID, foregroundClaim)
|
|
return resumePromptCtx, s.classifyEnsureSessionRunningFailureForPrompt(
|
|
ctx, taskID, sessionID, prompt, model, planMode, attachments, dispatchOnly, options, ensureErr,
|
|
)
|
|
}
|
|
if err := s.failPromptOnResumeAttempt(resumePromptCtx, taskID, sessionID, foregroundClaim, resumeAttempt); err != nil {
|
|
return resumePromptCtx, err
|
|
}
|
|
return resumePromptCtx, nil
|
|
}
|
|
|
|
// acquireResumeAttemptDispatchGuard is promptTask's post-claim resume-attempt
|
|
// guard step. When existingReleaseDispatchGuard is nil (claimDispatchAndAcquireGuard
|
|
// did not already acquire one), it locks the resume attempt's admission guard
|
|
// and returns the new release func for the caller to defer; a non-nil
|
|
// existingReleaseDispatchGuard is already deferred by the caller, so this only
|
|
// re-validates the attempt and returns (nil, nil) on either success or a nil
|
|
// resumeAttempt.
|
|
func (s *Service) acquireResumeAttemptDispatchGuard(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
foregroundDispatch *foregroundDispatch,
|
|
rollback promptClaimRollback,
|
|
options promptTaskOptions,
|
|
resumeAttempt *resumeAttempt,
|
|
existingReleaseDispatchGuard func(),
|
|
) (func(), error) {
|
|
if resumeAttempt == nil {
|
|
return nil, nil
|
|
}
|
|
if existingReleaseDispatchGuard != nil {
|
|
if err := s.validateResumeAttempt(resumeAttempt); err != nil {
|
|
return nil, s.failPromptDispatch(ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, err)
|
|
}
|
|
return nil, nil
|
|
}
|
|
guard, err := s.lockResumeAttemptAdmission(ctx, sessionID, resumeAttempt)
|
|
if err != nil {
|
|
return nil, s.failPromptDispatch(ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, err)
|
|
}
|
|
var releaseOnce sync.Once
|
|
return func() {
|
|
releaseOnce.Do(func() {
|
|
guard.release()
|
|
})
|
|
}, nil
|
|
}
|
|
|
|
// failPromptOnResumeAttempt returns the resume attempt's validation error, if
|
|
// any, after releasing the foreground claim and cleaning up the cancelled
|
|
// attempt. promptTask's early admission checks all share this cleanup.
|
|
func (s *Service) failPromptOnResumeAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
foregroundClaim *foregroundClaim,
|
|
resumeAttempt *resumeAttempt,
|
|
) error {
|
|
if err := s.validateResumeAttempt(resumeAttempt); err != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// failPromptDispatch cleans up a resume attempt that failed after dispatch
|
|
// admission began, rolling back both the foreground dispatch claim and the
|
|
// prompt claim under a bounded failure context, and returns err unchanged so
|
|
// callers can return it directly.
|
|
func (s *Service) failPromptDispatch(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
foregroundDispatch *foregroundDispatch,
|
|
rollback promptClaimRollback,
|
|
options promptTaskOptions,
|
|
resumeAttempt *resumeAttempt,
|
|
err error,
|
|
) error {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
return err
|
|
}
|
|
|
|
// finishPromptExecutorDispatch is promptTask's tail: it turns
|
|
// executor.PromptWithDispatchCallback's outcome, plus whatever onDispatched
|
|
// recorded via dispatchOutcome, into the one PromptResult/error pair promptTask
|
|
// returns.
|
|
func (s *Service) finishPromptExecutorDispatch(
|
|
ctx context.Context, taskID, sessionID, prompt string, planMode, resumedForPrompt bool,
|
|
attachments []v1.MessageAttachment, rollback promptClaimRollback, options promptTaskOptions,
|
|
foregroundDispatch *foregroundDispatch, releaseDispatchGuard func(),
|
|
result *executor.PromptResult, execErr error, dispatchOutcome *promptDispatchOutcome,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*PromptResult, error) {
|
|
dispatchAccepted, publicationErr := dispatchOutcome.snapshot()
|
|
if dispatchAccepted && options.promptAccepted != nil {
|
|
options.promptAccepted.Store(true)
|
|
}
|
|
if execErr != nil {
|
|
// Missing-execution recovery reacquires the cancel guard while it resets
|
|
// the session. Release dispatch admission before entering that path.
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
return nil, resumeErr
|
|
}
|
|
return s.finishPromptDispatchFailure(
|
|
ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments,
|
|
rollback, options, execErr, foregroundDispatch, dispatchAccepted, publicationErr,
|
|
)
|
|
}
|
|
if resumeErr := s.validateResumeAttempt(resumeAttempt); resumeErr != nil {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
return nil, resumeErr
|
|
}
|
|
if publicationErr != nil {
|
|
return nil, &acceptedPromptDispatchError{err: publicationErr}
|
|
}
|
|
return &PromptResult{StopReason: result.StopReason, AgentMessage: result.AgentMessage, TurnID: rollback.turnID}, nil
|
|
}
|
|
|
|
// validateAndRunDispatchBoundary resolves promptTask's queued-dispatch
|
|
// identity check, pre-admission hook, and post-admission hook — the checks
|
|
// that must all pass immediately before the executor call — rolling back the
|
|
// foreground dispatch and prompt claim on either failure. Only bounded-ack
|
|
// callers preserve request context past this point; ordinary prompts can take
|
|
// minutes, so the executor context is resolved here and threaded back out.
|
|
func (s *Service) validateAndRunDispatchBoundary(
|
|
ctx context.Context, taskID, sessionID, prompt string, planMode, resumedForPrompt bool,
|
|
attachments []v1.MessageAttachment, session *models.TaskSession, rollback promptClaimRollback,
|
|
options promptTaskOptions, foregroundDispatch *foregroundDispatch, runBeforeDispatch func() error,
|
|
runAfterDispatchAdmission func() error,
|
|
releaseDispatchGuard func(),
|
|
) (promptCtx context.Context, resolvedSession *models.TaskSession, earlyResult *PromptResult, err error) {
|
|
promptCtx = options.executorContext(ctx)
|
|
session, identityValidationErr := s.validateQueuedPromptDispatch(
|
|
promptCtx, rollback.sessionIdentity, taskID, sessionID, session,
|
|
)
|
|
if identityValidationErr != nil {
|
|
// Missing-execution recovery reacquires the session cancellation guard.
|
|
// Release the transferred queued-dispatch guard before entering that
|
|
// recovery path, while retaining it through successful provider admission.
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
result, err := s.finishPromptDispatchFailure(
|
|
ctx, taskID, sessionID, prompt, planMode, resumedForPrompt, attachments,
|
|
rollback, options, identityValidationErr, foregroundDispatch, false, nil,
|
|
)
|
|
return promptCtx, nil, result, err
|
|
}
|
|
if bindingErr := s.validateContextCeilingEntry(promptCtx, taskID); bindingErr != nil {
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
return promptCtx, nil, nil, bindingErr
|
|
}
|
|
boundaryErr := runBeforeDispatch()
|
|
if boundaryErr == nil {
|
|
boundaryErr = s.admitCeilingDispatch(promptCtx, taskID)
|
|
}
|
|
if boundaryErr == nil {
|
|
boundaryErr = runAfterDispatchAdmission()
|
|
}
|
|
if boundaryErr != nil {
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
return promptCtx, nil, nil, boundaryErr
|
|
}
|
|
return promptCtx, session, nil, nil
|
|
}
|
|
|
|
// preparePromptDispatchCallback binds the turn, arms the interactive-prompt
|
|
// attempt evidence, and builds the onDispatched callback executor.
|
|
// PromptWithDispatchCallback invokes once agentctl accepts the prompt. For a
|
|
// resumed turn, ownership transfer wraps the durable afterDispatch hook and
|
|
// identity publication; the nonterminal dispatch guard releases last.
|
|
func (s *Service) preparePromptDispatchCallback(
|
|
promptCtx context.Context, taskID, sessionID string, session *models.TaskSession,
|
|
rollback promptClaimRollback, options promptTaskOptions, foregroundDispatch *foregroundDispatch,
|
|
releaseDispatchGuard func(),
|
|
resumeAttempt *resumeAttempt,
|
|
) (onDispatched func(), dispatchOutcome *promptDispatchOutcome) {
|
|
s.bindPromptTurnID(promptCtx, session, rollback.turnID)
|
|
s.beginInteractivePromptAttempt(
|
|
promptCtx,
|
|
sessionID,
|
|
session.AgentExecutionID,
|
|
s.isDynamicPromptSession(session),
|
|
)
|
|
dispatchOutcome = &promptDispatchOutcome{}
|
|
dispatchOutcome.turnID = rollback.turnID
|
|
dispatchOutcome.onAccepted = options.onAccepted
|
|
if options.promptAccepted != nil {
|
|
originalOnAccepted := dispatchOutcome.onAccepted
|
|
dispatchOutcome.onAccepted = func(turnID string) {
|
|
options.promptAccepted.Store(true)
|
|
if originalOnAccepted != nil {
|
|
originalOnAccepted(turnID)
|
|
}
|
|
}
|
|
}
|
|
acceptedExecutionID := session.AgentExecutionID
|
|
onDispatched = s.promptDispatchCallbackForIdentity(
|
|
promptCtx, taskID, sessionID, rollback.sessionIdentity,
|
|
acceptedExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome,
|
|
)
|
|
if options.afterDispatch != nil {
|
|
originalOnDispatched := onDispatched
|
|
onDispatched = func() {
|
|
durableOutcomeErr := options.afterDispatch()
|
|
originalOnDispatched()
|
|
_, publicationErr := dispatchOutcome.snapshot()
|
|
dispatchOutcome.recordAccepted(errors.Join(durableOutcomeErr, publicationErr))
|
|
}
|
|
}
|
|
if resumeAttempt != nil {
|
|
originalOnDispatched := onDispatched
|
|
onDispatched = func() {
|
|
// Transfer startup ownership before any durable post-acceptance hook
|
|
// or identity publication can fail and before the dispatch guard releases.
|
|
s.acceptResumeAttemptAtPromptAcceptance(acceptedExecutionID, resumeAttempt)
|
|
originalOnDispatched()
|
|
}
|
|
}
|
|
if releaseDispatchGuard != nil {
|
|
originalOnDispatched := onDispatched
|
|
onDispatched = func() {
|
|
originalOnDispatched()
|
|
// Keep the cancellation guard through the complete acceptance
|
|
// callback. The callback binds the accepted turn and publishes its
|
|
// durable ownership; releasing first would let CancelAgent settle
|
|
// the session between provider acceptance and those mutations.
|
|
releaseDispatchGuard()
|
|
}
|
|
}
|
|
return onDispatched, dispatchOutcome
|
|
}
|
|
|
|
func (s *Service) acceptResumeAttemptAtPromptAcceptance(
|
|
expectedExecutionID string,
|
|
attempt *resumeAttempt,
|
|
) bool {
|
|
if attempt == nil {
|
|
return false
|
|
}
|
|
if expectedExecutionID == "" {
|
|
expectedExecutionID = attempt.execution()
|
|
}
|
|
if expectedExecutionID == "" {
|
|
return false
|
|
}
|
|
// Provider acceptance is the irreversible boundary for startup teardown.
|
|
// Use the execution admitted before the provider call; stale-incarnation
|
|
// publication remains separately fenced by promptDispatchCallbackForIdentity.
|
|
return s.resumeAttemptStore().accept(attempt, expectedExecutionID)
|
|
}
|
|
|
|
func (s *Service) finishPromptDispatchFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt string,
|
|
planMode, resumedForPrompt bool,
|
|
attachments []v1.MessageAttachment,
|
|
rollback promptClaimRollback,
|
|
options promptTaskOptions,
|
|
promptErr error,
|
|
foregroundDispatch *foregroundDispatch,
|
|
dispatchAccepted bool,
|
|
publicationErr error,
|
|
) (*PromptResult, error) {
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
if dispatchAccepted && rollback.reservedTurn != nil {
|
|
// Agentctl acceptance made the turn authoritative. A later completion
|
|
// error may reconcile session state, but must not delete this turn.
|
|
rollback.reservedTurnAccepted = true
|
|
}
|
|
failureResult, failureErr := s.handlePromptDispatchFailure(
|
|
failureCtx, taskID, sessionID, prompt, planMode, resumedForPrompt && !options.disableDispatchRetry,
|
|
attachments, rollback, options.lifecyclePrompt, dispatchAccepted, promptErr,
|
|
options.promptAlreadyComposed, options.fallbackLaunchPrompt, options.fallbackRetryPrompt,
|
|
options.promptReferenceContext,
|
|
)
|
|
return failureResult, wrapAcceptedPromptDispatchFailure(
|
|
dispatchAccepted,
|
|
failureErr,
|
|
publicationErr,
|
|
)
|
|
}
|
|
|
|
func (s *Service) reloadPromptSession(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
fallback *models.TaskSession,
|
|
) *models.TaskSession {
|
|
reloaded, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || reloaded == nil {
|
|
return fallback
|
|
}
|
|
return reloaded
|
|
}
|
|
|
|
// beginForegroundDispatchForPrompt reloads the session after ensureSessionRunning
|
|
// (so executor.Prompt uses the freshly-persisted AgentExecutionID rather than a
|
|
// pointer staled by a concurrent write), re-applies config/plan-mode prompt
|
|
// transforms against the reloaded session, and begins the foreground dispatch.
|
|
// It releases the foreground claim itself on the "already in flight" failure,
|
|
// mirroring what promptTask's own failure paths do elsewhere.
|
|
func (s *Service) beginForegroundDispatchForPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt string,
|
|
planMode bool,
|
|
session *models.TaskSession,
|
|
foregroundClaim *foregroundClaim,
|
|
configModeOverride *bool,
|
|
) (*models.TaskSession, string, *foregroundDispatch, error) {
|
|
session = s.reloadPromptSession(ctx, sessionID, session)
|
|
effectivePrompt := s.effectivePromptForSessionWithConfigMode(
|
|
sessionID, prompt, planMode, session, configModeOverride,
|
|
)
|
|
activityExecutionID, _ := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
foregroundDispatch := s.beginForegroundDispatch(sessionID, foregroundClaim, activityExecutionID)
|
|
if foregroundDispatch == nil {
|
|
s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim)
|
|
return nil, "", nil, fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress)
|
|
}
|
|
return session, effectivePrompt, foregroundDispatch, nil
|
|
}
|
|
|
|
func (s *Service) bindPromptTurnID(ctx context.Context, session *models.TaskSession, turnID string) {
|
|
setter, ok := s.agentManager.(executor.PromptTurnIDSetter)
|
|
if turnID == "" || session == nil || !ok {
|
|
return
|
|
}
|
|
if err := setter.SetPromptTurnID(ctx, session.AgentExecutionID, turnID); err != nil {
|
|
s.logger.Warn("failed to bind prompt turn before dispatch",
|
|
zap.String("session_id", session.ID),
|
|
zap.String("turn_id", turnID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
|
|
func (s *Service) validatePromptTaskStart(sessionID string) error {
|
|
if sessionID == "" {
|
|
return fmt.Errorf("session_id is required")
|
|
}
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validatePromptTaskPreconditions combines promptTask's two top-of-function
|
|
// checks: the ordinary session/reset-in-progress validation, and — for a
|
|
// compound resume passing its own cancellable attempt context — an early
|
|
// ownership check so a lost race surfaces its typed error before any
|
|
// repository read can observe the cancellation as a generic context error.
|
|
func (s *Service) validatePromptTaskPreconditions(sessionID string, resumeAttempt *resumeAttempt) error {
|
|
if err := s.validatePromptTaskStart(sessionID); err != nil {
|
|
return err
|
|
}
|
|
return s.validateResumeAttempt(resumeAttempt)
|
|
}
|
|
|
|
func (s *Service) logPromptTaskCall(
|
|
taskID, sessionID, prompt, model string,
|
|
planMode bool,
|
|
attachments []v1.MessageAttachment,
|
|
dispatchOnly bool,
|
|
) {
|
|
s.logger.Debug("PromptTask called",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Int("prompt_length", len(prompt)),
|
|
zap.String("requested_model", model),
|
|
zap.Bool("plan_mode", planMode),
|
|
zap.Int("attachments_count", len(attachments)),
|
|
zap.Bool("dispatch_only", dispatchOnly))
|
|
}
|
|
|
|
func (s *Service) loadPromptableSession(ctx context.Context, taskID, sessionID string) (*models.TaskSession, error) {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if promptErr := s.checkSessionPromptable(taskID, sessionID, session.State); promptErr != nil {
|
|
if !errors.Is(promptErr, ErrSessionNotPromptable) || session.State != models.TaskSessionStateStarting {
|
|
return nil, promptErr
|
|
}
|
|
return s.waitForStartingSessionPromptable(ctx, taskID, sessionID)
|
|
}
|
|
return session, nil
|
|
}
|
|
|
|
// prepareSessionAndForegroundClaimForPrompt resolves promptTask's session (the
|
|
// nonterminal-reload variant when options.requireNonterminalSession) and takes
|
|
// its foreground claim, as one step so promptTask itself only has one error to
|
|
// check.
|
|
func (s *Service) prepareSessionAndForegroundClaimForPrompt(
|
|
ctx context.Context, taskID, sessionID string, options promptTaskOptions,
|
|
) (*models.TaskSession, *foregroundClaim, error) {
|
|
session, err := s.loadPromptableSession(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if options.requireNonterminalSession {
|
|
session, err = s.loadNonterminalWorkflowAutoStartSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
foregroundClaim, err := s.claimForegroundForPrompt(taskID, sessionID, session)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return session, foregroundClaim, nil
|
|
}
|
|
|
|
// claimAndGuardDispatch combines claimDispatchAndAcquireGuard with the
|
|
// resume-attempt dispatch guard step that follows it, so promptTask sees one
|
|
// error check and one guard-release defer instead of two of each. On failure
|
|
// it releases whatever guard claimDispatchAndAcquireGuard had already
|
|
// acquired before returning, mirroring the release the caller's own deferred
|
|
// call would otherwise have performed.
|
|
func (s *Service) claimAndGuardDispatch(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
foregroundClaim *foregroundClaim,
|
|
options promptTaskOptions,
|
|
foregroundDispatch *foregroundDispatch,
|
|
resumeAttempt *resumeAttempt,
|
|
admissionGuard *lockedCancelInFlightGuard,
|
|
) (session *models.TaskSession, rollback promptClaimRollback, releaseDispatchGuard func(), err error) {
|
|
session, rollback, releaseDispatchGuard, err = s.claimDispatchAndAcquireGuard(
|
|
ctx, taskID, sessionID, foregroundClaim, options, foregroundDispatch, resumeAttempt, admissionGuard,
|
|
)
|
|
if err != nil {
|
|
return nil, promptClaimRollback{}, nil, err
|
|
}
|
|
newDispatchGuard, err := s.acquireResumeAttemptDispatchGuard(
|
|
ctx, taskID, sessionID, foregroundDispatch, rollback, options, resumeAttempt, releaseDispatchGuard,
|
|
)
|
|
if err != nil {
|
|
if releaseDispatchGuard != nil {
|
|
releaseDispatchGuard()
|
|
}
|
|
return nil, promptClaimRollback{}, nil, err
|
|
}
|
|
if newDispatchGuard != nil {
|
|
releaseDispatchGuard = newDispatchGuard
|
|
}
|
|
if options.initialCreatePromptPassthrough && session != nil && session.IsPassthrough {
|
|
s.armInitialCreatePromptPassthrough(ctx, session, rollback.turnID)
|
|
if session.AgentExecutionID != "" {
|
|
s.bindInitialCreatePromptPassthroughExecution(ctx, session.ID, rollback.turnID, session.AgentExecutionID)
|
|
}
|
|
}
|
|
return session, rollback, releaseDispatchGuard, nil
|
|
}
|
|
|
|
// claimDispatchAndAcquireGuard performs promptTask's admission claim
|
|
// (claimPromptDispatch), publishes the foreground-activity flip a claimed
|
|
// RUNNING prompt or the earlier beginForegroundDispatch may have caused, and
|
|
// resolves the nonterminal dispatch guard — the three steps that happen
|
|
// together between "foreground dispatch begun" and "ready to call the
|
|
// executor".
|
|
func (s *Service) claimDispatchAndAcquireGuard(
|
|
ctx context.Context, taskID, sessionID string, foregroundClaim *foregroundClaim,
|
|
options promptTaskOptions, foregroundDispatch *foregroundDispatch, resumeAttempt *resumeAttempt,
|
|
admissionGuard *lockedCancelInFlightGuard,
|
|
) (*models.TaskSession, promptClaimRollback, func(), error) {
|
|
session, rollback, err := s.claimPromptDispatchWithResumeAttempt(
|
|
ctx, taskID, sessionID, options.claimEntryID, options.lifecyclePrompt,
|
|
options.reserveTurnUntilDispatch, options.promptDispatchRecovery,
|
|
options.afterClaim, foregroundClaim, options.expectedCurrentTurnID,
|
|
options.requireNonterminalSession, resumeAttempt, admissionGuard, options.expectedSessionIdentity,
|
|
)
|
|
if err != nil {
|
|
if errors.Is(err, ErrResumeAttemptCancelled) {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
s.releaseForegroundClaimOnFailure(ctx, taskID, sessionID, foregroundClaim)
|
|
return nil, promptClaimRollback{}, nil, err
|
|
}
|
|
s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch)
|
|
return nil, promptClaimRollback{}, nil, err
|
|
}
|
|
if foregroundDispatch.yieldedBeforeBegin || foregroundClaim != nil {
|
|
s.publishForegroundActivityChanged(ctx, taskID, sessionID)
|
|
}
|
|
releaseDispatchGuard, err := s.acquireNonterminalDispatchGuardForPrompt(
|
|
ctx, taskID, sessionID, options, foregroundDispatch, rollback, rollback.dispatchGuardRelease,
|
|
)
|
|
if err != nil {
|
|
return nil, promptClaimRollback{}, nil, err
|
|
}
|
|
return session, rollback, releaseDispatchGuard, nil
|
|
}
|
|
|
|
// acquireNonterminalDispatchGuardForPrompt takes the cancel-in-flight guard for
|
|
// a requireNonterminalSession prompt that did not already claim one via
|
|
// rollback.dispatchGuardRelease (existingRelease non-nil short-circuits, since
|
|
// one turn only ever needs one guard), and reloads the session under it to
|
|
// catch a concurrent terminalization race between claimPromptDispatch and
|
|
// here. Returns existingRelease unchanged when no new guard was needed.
|
|
func (s *Service) acquireNonterminalDispatchGuardForPrompt(
|
|
ctx context.Context, taskID, sessionID string, options promptTaskOptions,
|
|
foregroundDispatch *foregroundDispatch, rollback promptClaimRollback, existingRelease func(),
|
|
) (func(), error) {
|
|
if !options.requireNonterminalSession || existingRelease != nil {
|
|
return existingRelease, nil
|
|
}
|
|
guard, release := s.acquireCancelInFlightGuard(sessionID)
|
|
guard.Lock()
|
|
fresh, reloadErr := s.repo.GetTaskSession(ctx, sessionID)
|
|
if reloadErr != nil {
|
|
guard.Unlock()
|
|
release()
|
|
s.rollbackPromptDispatchOnGuardFailure(ctx, taskID, sessionID, options, foregroundDispatch, rollback)
|
|
return nil, reloadErr
|
|
}
|
|
if fresh == nil || isTerminalSessionState(fresh.State) {
|
|
guard.Unlock()
|
|
release()
|
|
s.rollbackPromptDispatchOnGuardFailure(ctx, taskID, sessionID, options, foregroundDispatch, rollback)
|
|
return nil, newWorkflowAutoStartSessionTerminalizedError(fresh)
|
|
}
|
|
var releaseOnce sync.Once
|
|
return func() {
|
|
releaseOnce.Do(func() {
|
|
guard.Unlock()
|
|
release()
|
|
})
|
|
}, nil
|
|
}
|
|
|
|
// rollbackPromptDispatchOnGuardFailure unwinds the foreground dispatch and
|
|
// prompt claim taken earlier in promptTask when the nonterminal dispatch guard
|
|
// itself fails to admit the turn.
|
|
func (s *Service) rollbackPromptDispatchOnGuardFailure(
|
|
ctx context.Context, taskID, sessionID string, options promptTaskOptions,
|
|
foregroundDispatch *foregroundDispatch, rollback promptClaimRollback,
|
|
) {
|
|
failureCtx, cancel := options.failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackForegroundDispatchOnFailure(failureCtx, taskID, sessionID, foregroundDispatch)
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
}
|
|
|
|
// loadNonterminalWorkflowAutoStartSession reloads a session under the shared
|
|
// cancellation guard before auto-start can trigger a lazy ACP resume. Manual
|
|
// prompts intentionally do not use this path and can still resume terminal
|
|
// history when explicitly requested.
|
|
func (s *Service) loadNonterminalWorkflowAutoStartSession(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
) (*models.TaskSession, error) {
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reload workflow auto-start session: %w", err)
|
|
}
|
|
if session == nil || isTerminalSessionState(session.State) {
|
|
return nil, newWorkflowAutoStartSessionTerminalizedError(session)
|
|
}
|
|
return session, nil
|
|
}
|
|
|
|
// claimForegroundForPrompt serializes two callers racing to take the
|
|
// background-idle foreground turn. For non-experiment RUNNING sessions,
|
|
// checkSessionPromptable already rejects the prompt before this helper is reached.
|
|
func (s *Service) claimForegroundForPrompt(taskID, sessionID string, session *models.TaskSession) (*foregroundClaim, error) {
|
|
if session.State != models.TaskSessionStateRunning {
|
|
return nil, nil
|
|
}
|
|
claim := s.claimForegroundTurn(sessionID)
|
|
if claim != nil {
|
|
return claim, nil
|
|
}
|
|
s.logger.Warn("rejected prompt: another prompt claimed the background-idle foreground turn first",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
return nil, fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress)
|
|
}
|
|
|
|
func (s *Service) releaseForegroundClaimOnFailure(ctx context.Context, taskID, sessionID string, claim *foregroundClaim) {
|
|
if s.releaseForegroundClaim(claim) {
|
|
s.publishForegroundActivityChanged(ctx, taskID, sessionID)
|
|
}
|
|
}
|
|
|
|
func (s *Service) rollbackForegroundDispatchOnFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
dispatch *foregroundDispatch,
|
|
) {
|
|
if s.rollbackForegroundDispatch(dispatch) {
|
|
s.publishForegroundActivityChanged(ctx, taskID, sessionID)
|
|
}
|
|
}
|
|
|
|
func (s *Service) promptDispatchCallback(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
reservedTurn *models.Turn,
|
|
dispatch *foregroundDispatch,
|
|
outcome *promptDispatchOutcome,
|
|
) func() {
|
|
return s.promptDispatchCallbackForIdentity(
|
|
ctx, taskID, sessionID, messagequeue.QueueSessionIdentity{}, "",
|
|
reservedTurn, dispatch, outcome,
|
|
)
|
|
}
|
|
|
|
func (s *Service) promptDispatchIdentityIsCurrent(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
expectedExecutionID, executionID string,
|
|
) bool {
|
|
if identity.SessionIncarnationID == "" {
|
|
return true
|
|
}
|
|
current, err := s.messageQueue.ResolveSessionIdentity(ctx, identity.TaskID, identity.SessionID)
|
|
return err == nil && current == identity && executionID == expectedExecutionID
|
|
}
|
|
|
|
func (s *Service) promptDispatchCallbackForIdentity(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
expectedExecutionID string,
|
|
reservedTurn *models.Turn,
|
|
dispatch *foregroundDispatch,
|
|
outcome *promptDispatchOutcome,
|
|
) func() {
|
|
return func() {
|
|
executionID, _ := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if !s.promptDispatchIdentityIsCurrent(ctx, identity, expectedExecutionID, executionID) {
|
|
outcome.recordAccepted(nil)
|
|
return
|
|
}
|
|
s.bindPromptAttemptToExecution(ctx, sessionID, executionID)
|
|
var publicationErr error
|
|
if reservedTurn != nil && s.turnService != nil {
|
|
// Agentctl has already accepted the prompt. Publication must not inherit
|
|
// an expiring HTTP context or the caller could receive success while the
|
|
// durable turn still looks unpublished at restart.
|
|
publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), promptFailureCleanupTimeout)
|
|
publicationErr = s.turnService.PublishReservedTurn(publishCtx, reservedTurn)
|
|
cancel()
|
|
if publicationErr != nil {
|
|
s.logger.Error("failed to persist or publish accepted prompt turn",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("turn_id", reservedTurn.ID),
|
|
zap.Error(publicationErr))
|
|
}
|
|
// The pre-dispatch attempt marker makes restart recovery and rollback
|
|
// fail closed even if this post-acceptance publication fails. Do not
|
|
// restore an active-turn cache entry when cancellation already made the
|
|
// reservation terminal or publication could not find it.
|
|
if reservedTurn.CompletedAt == nil && !errors.Is(publicationErr, sql.ErrNoRows) {
|
|
s.activeTurns.Store(sessionID, reservedTurn.ID)
|
|
s.bindAcceptedDispatchTurn(sessionID, reservedTurn.ID)
|
|
}
|
|
s.resolveReservedPromptTurn(sessionID, reservedTurn.ID, true)
|
|
}
|
|
if outcome != nil && outcome.onAccepted != nil {
|
|
acceptedTurnID := outcome.turnID
|
|
if reservedTurn != nil && reservedTurn.ID != "" {
|
|
acceptedTurnID = reservedTurn.ID
|
|
}
|
|
if acceptedTurnID != "" {
|
|
outcome.onAccepted(acceptedTurnID)
|
|
}
|
|
}
|
|
outcome.recordAccepted(publicationErr)
|
|
if s.acceptForegroundDispatch(dispatch) {
|
|
s.publishForegroundActivityChanged(ctx, taskID, sessionID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// attemptModelSwitchForPrompt runs promptTask's pre-dispatch model-switch
|
|
// handling: arming the initial-prompt-attempt evidence, running the caller's
|
|
// pre-admission hook, admitting the dispatch, and running the caller's
|
|
// post-admission hook when a switch is required, then delegating to
|
|
// trySwitchModelForPrompt. handled reports whether the caller already has its
|
|
// full response — a dispatched switch, a switch failure, or a beforeDispatch
|
|
// failure — in which case promptTask returns (result, err) directly without
|
|
// proceeding to ordinary claim/dispatch.
|
|
func (s *Service) attemptModelSwitchForPrompt(
|
|
ctx context.Context, taskID, sessionID, model, effectivePrompt string, session *models.TaskSession,
|
|
foregroundDispatch *foregroundDispatch, runBeforeDispatch func() error,
|
|
runAfterDispatchAdmission func() error, resumeAttempts ...*resumeAttempt,
|
|
) (result *PromptResult, handled bool, err error) {
|
|
var resumeAttempt *resumeAttempt
|
|
if len(resumeAttempts) > 0 {
|
|
resumeAttempt = resumeAttempts[0]
|
|
}
|
|
if modelSwitchRequired(session, model) {
|
|
s.beginInitialPromptAttempt(sessionID, s.isDynamicPromptSession(session))
|
|
admissionErr := runBeforeDispatch()
|
|
if admissionErr == nil {
|
|
admissionErr = s.admitCeilingDispatch(ctx, taskID)
|
|
}
|
|
if admissionErr == nil {
|
|
admissionErr = runAfterDispatchAdmission()
|
|
}
|
|
if admissionErr != nil {
|
|
s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, foregroundDispatch)
|
|
s.clearPromptAttemptEvidence(sessionID, "", 0)
|
|
return nil, true, admissionErr
|
|
}
|
|
}
|
|
result, handled, switchErr := s.trySwitchModelForPrompt(
|
|
ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, resumeAttempt,
|
|
)
|
|
if handled && switchErr != nil {
|
|
s.clearPromptAttemptEvidence(sessionID, "", 0)
|
|
}
|
|
return result, handled, switchErr
|
|
}
|
|
|
|
// trySwitchModelForPrompt keeps foreground admission consistent when a model
|
|
// switch either dispatches the prompt itself or fails before reaching the agent.
|
|
func (s *Service) trySwitchModelForPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, model, prompt string,
|
|
session *models.TaskSession,
|
|
dispatch *foregroundDispatch,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*PromptResult, bool, error) {
|
|
releaseHold, onDispatched, onFailure, holdErr := s.modelSwitchResumeCallbacks(session, model, resumeAttempt)
|
|
if holdErr != nil {
|
|
return nil, true, holdErr
|
|
}
|
|
result, switched, err := s.trySwitchModelWithDispatchCallbacks(
|
|
ctx, taskID, sessionID, model, prompt, session, onDispatched, onFailure,
|
|
)
|
|
if !switched && err == nil {
|
|
releaseHold()
|
|
return nil, false, nil
|
|
}
|
|
if err != nil {
|
|
releaseHold()
|
|
s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, dispatch)
|
|
return result, true, err
|
|
}
|
|
if executionID, executionErr := s.agentManager.GetExecutionIDForSession(ctx, sessionID); executionErr == nil && executionID != "" {
|
|
s.bindPromptAttemptToExecution(ctx, sessionID, executionID)
|
|
if resumeAttempt != nil {
|
|
// The replacement's initial prompt is dispatched asynchronously by
|
|
// lifecycle. Bind its immutable execution identity now; startup
|
|
// ownership transfers only from the callback installed for that prompt.
|
|
resumeAttempt.setExecutionID(executionID)
|
|
}
|
|
}
|
|
revealedBackground := s.acceptForegroundDispatch(dispatch)
|
|
if revealedBackground || dispatch.yieldedBeforeBegin || s.acceptedForegroundDispatchClaim(dispatch) {
|
|
s.publishForegroundActivityChanged(ctx, taskID, sessionID)
|
|
}
|
|
return result, true, nil
|
|
}
|
|
|
|
func (s *Service) modelSwitchResumeCallbacks(
|
|
session *models.TaskSession,
|
|
model string,
|
|
resumeAttempt *resumeAttempt,
|
|
) (release func(), onDispatched func(executionID string), onFailure func(), err error) {
|
|
release = func() {}
|
|
if resumeAttempt == nil || !modelSwitchRequired(session, model) {
|
|
return release, nil, nil, nil
|
|
}
|
|
registry := s.resumeAttemptStore()
|
|
if !registry.holdForInitialPrompt(resumeAttempt) {
|
|
s.cleanupCancelledResumeAttempt(resumeAttempt)
|
|
return nil, nil, nil, ErrResumeAttemptCancelled
|
|
}
|
|
release = func() { registry.releaseInitialPromptHold(resumeAttempt) }
|
|
onDispatched = func(executionID string) {
|
|
if s.acceptResumeAttemptAtPromptAcceptance(executionID, resumeAttempt) {
|
|
registry.finishAfterInitialPromptAcceptance(resumeAttempt)
|
|
}
|
|
}
|
|
onFailure = func() { registry.abortInitialPromptHold(resumeAttempt) }
|
|
return release, onDispatched, onFailure, nil
|
|
}
|
|
|
|
type promptClaimRollback struct {
|
|
previousSessionState models.TaskSessionState
|
|
previousTaskState v1.TaskState
|
|
taskStateClaimed bool
|
|
sessionIdentity messagequeue.QueueSessionIdentity
|
|
turnID string
|
|
createdTurn bool
|
|
reservedTurn *models.Turn
|
|
reservedTurnAccepted bool
|
|
dispatchGuardRelease func()
|
|
}
|
|
|
|
func (s *Service) claimPromptDispatch(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
lifecyclePrompt bool,
|
|
reserveTurnUntilDispatch bool,
|
|
promptDispatchRecovery *models.PromptDispatchRecovery,
|
|
afterClaim func() error,
|
|
foregroundClaim *foregroundClaim,
|
|
expectedCurrentTurnID string,
|
|
requireNonterminalSession bool,
|
|
expectedIdentities ...*messagequeue.QueueSessionIdentity,
|
|
) (*models.TaskSession, promptClaimRollback, error) {
|
|
return s.claimPromptDispatchWithResumeAttempt(
|
|
ctx, taskID, sessionID, claimEntryID, lifecyclePrompt,
|
|
reserveTurnUntilDispatch, promptDispatchRecovery, afterClaim, foregroundClaim,
|
|
expectedCurrentTurnID, requireNonterminalSession, nil, nil, expectedIdentities...,
|
|
)
|
|
}
|
|
|
|
func (s *Service) claimPromptDispatchWithResumeAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
lifecyclePrompt bool,
|
|
reserveTurnUntilDispatch bool,
|
|
promptDispatchRecovery *models.PromptDispatchRecovery,
|
|
afterClaim func() error,
|
|
foregroundClaim *foregroundClaim,
|
|
expectedCurrentTurnID string,
|
|
requireNonterminalSession bool,
|
|
resumeAttempt *resumeAttempt,
|
|
admissionGuard *lockedCancelInFlightGuard,
|
|
expectedIdentities ...*messagequeue.QueueSessionIdentity,
|
|
) (*models.TaskSession, promptClaimRollback, error) {
|
|
claimCtx := ctx
|
|
if resumeAttempt != nil {
|
|
claimCtx = context.WithoutCancel(ctx)
|
|
}
|
|
var expectedIdentity *messagequeue.QueueSessionIdentity
|
|
if len(expectedIdentities) > 0 {
|
|
expectedIdentity = expectedIdentities[0]
|
|
}
|
|
if lifecyclePrompt {
|
|
return s.claimLifecyclePromptDispatchWithResumeAttempt(
|
|
claimCtx, taskID, sessionID, claimEntryID, afterClaim, resumeAttempt,
|
|
)
|
|
}
|
|
claimArgs := []interface{}{expectedIdentity, afterClaim}
|
|
if admissionGuard != nil {
|
|
claimArgs = append(claimArgs, admissionGuard)
|
|
}
|
|
if resumeAttempt != nil {
|
|
claimArgs = append(claimArgs, resumeAttempt)
|
|
}
|
|
claimed, previousState, turnID, createdTurn, reservedTurn, dispatchGuardRelease, err := s.claimSessionRunningForPrompt(
|
|
claimCtx, taskID, sessionID, claimEntryID, reserveTurnUntilDispatch,
|
|
promptDispatchRecovery, foregroundClaim, expectedCurrentTurnID, requireNonterminalSession,
|
|
claimArgs...,
|
|
)
|
|
if err != nil {
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
rollback := promptClaimRollback{
|
|
previousSessionState: previousState,
|
|
turnID: turnID,
|
|
createdTurn: createdTurn,
|
|
reservedTurn: reservedTurn,
|
|
}
|
|
if reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID); reservation != nil {
|
|
rollback.sessionIdentity = reservation.identity
|
|
}
|
|
if expectedIdentity != nil {
|
|
rollback.sessionIdentity = *expectedIdentity
|
|
}
|
|
rollback.dispatchGuardRelease = dispatchGuardRelease
|
|
return claimed, rollback, nil
|
|
}
|
|
|
|
func (s *Service) claimLifecyclePromptDispatch(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
afterClaim func() error,
|
|
) (*models.TaskSession, promptClaimRollback, error) {
|
|
return s.claimLifecyclePromptDispatchWithResumeAttempt(
|
|
ctx, taskID, sessionID, claimEntryID, afterClaim, nil,
|
|
)
|
|
}
|
|
|
|
func (s *Service) claimLifecyclePromptDispatchWithResumeAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
afterClaim func() error,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*models.TaskSession, promptClaimRollback, error) {
|
|
session, previousSessionState, turnID, createdTurn, err := s.claimLifecycleSessionRunningWithResumeAttempt(
|
|
ctx, taskID, sessionID, claimEntryID,
|
|
resumeAttempt,
|
|
)
|
|
if err != nil {
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
rollback := promptClaimRollback{
|
|
previousSessionState: previousSessionState,
|
|
turnID: turnID,
|
|
createdTurn: createdTurn,
|
|
}
|
|
if reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID); reservation != nil {
|
|
rollback.sessionIdentity = reservation.identity
|
|
}
|
|
rollback.previousTaskState, rollback.taskStateClaimed, err = s.reconcileLifecycleClaim(
|
|
ctx, taskID, sessionID, previousSessionState, session,
|
|
)
|
|
if err != nil {
|
|
s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback)
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
dispatchGuard := s.lockCancelInFlightGuard(sessionID)
|
|
var releaseGuardOnce sync.Once
|
|
releaseDispatchGuard := func() {
|
|
releaseGuardOnce.Do(dispatchGuard.release)
|
|
}
|
|
keepDispatchGuard := false
|
|
defer func() {
|
|
if !keepDispatchGuard {
|
|
releaseDispatchGuard()
|
|
}
|
|
}()
|
|
rollback.dispatchGuardRelease = releaseDispatchGuard
|
|
if err := s.validateResumeAttempt(resumeAttempt); err != nil {
|
|
s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback)
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
if err := s.validateLifecyclePromptTurn(ctx, sessionID, rollback.turnID); err != nil {
|
|
s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback)
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
if err := s.acknowledgePromptClaim(ctx, taskID, sessionID, afterClaim, rollback); err != nil {
|
|
return nil, promptClaimRollback{}, err
|
|
}
|
|
keepDispatchGuard = true
|
|
return session, rollback, nil
|
|
}
|
|
|
|
func (s *Service) acknowledgePromptClaim(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
afterClaim func() error,
|
|
rollback promptClaimRollback,
|
|
) error {
|
|
if afterClaim == nil {
|
|
return nil
|
|
}
|
|
if err := afterClaim(); err != nil {
|
|
s.rollbackPromptClaim(ctx, taskID, sessionID, rollback)
|
|
if errors.Is(err, errLifecyclePromptReservationSuperseded) {
|
|
return err
|
|
}
|
|
return fmt.Errorf("%w: %v", errLifecyclePromptMessagePersistence, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) rollbackPromptClaimAfterAdmissionFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
rollback promptClaimRollback,
|
|
) {
|
|
failureCtx, cancel := (promptTaskOptions{}).failureContext(ctx)
|
|
defer cancel()
|
|
s.rollbackPromptClaim(failureCtx, taskID, sessionID, rollback)
|
|
}
|
|
|
|
func (s *Service) rollbackPromptClaim(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
rollback promptClaimRollback,
|
|
) {
|
|
if rollback.dispatchGuardRelease != nil {
|
|
rollback.dispatchGuardRelease()
|
|
}
|
|
if rollback.sessionIdentity.SessionIncarnationID != "" {
|
|
s.restoreLifecycleClaimForIdentity(ctx, rollback.sessionIdentity, rollback.previousSessionState)
|
|
} else {
|
|
s.restoreLifecycleClaim(ctx, taskID, sessionID, rollback.previousSessionState)
|
|
}
|
|
if rollback.taskStateClaimed {
|
|
s.restoreLifecycleTaskState(ctx, taskID, rollback.previousTaskState)
|
|
}
|
|
if rollback.createdTurn {
|
|
if rollback.reservedTurn != nil {
|
|
if !rollback.reservedTurnAccepted {
|
|
s.rollbackReservedPromptTurn(ctx, sessionID, rollback.turnID)
|
|
}
|
|
} else {
|
|
s.completeTurnIfCurrent(ctx, sessionID, rollback.turnID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) rollbackReservedPromptTurn(ctx context.Context, sessionID, turnID string) {
|
|
if s.turnService == nil || turnID == "" {
|
|
return
|
|
}
|
|
// Publication removes the private reservation. A later transport/completion
|
|
// error must not delete a turn that agentctl already accepted.
|
|
if s.reservedPromptTurnID(sessionID) != turnID {
|
|
return
|
|
}
|
|
rolledBack, err := s.turnService.RollbackReservedTurn(ctx, sessionID, turnID)
|
|
if err != nil {
|
|
// The database outcome can be ambiguous (for example, a commit error).
|
|
// Keep the live reservation unresolved so ready handling waits and later
|
|
// prompts remain blocked until restart recovery reconciles durable state.
|
|
s.logger.Error("failed to roll back reserved prompt turn; session remains quarantined",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("turn_id", turnID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
s.resolveReservedPromptTurn(sessionID, turnID, false)
|
|
if rolledBack {
|
|
s.activeTurns.CompareAndDelete(sessionID, turnID)
|
|
return
|
|
}
|
|
active, lookupErr := s.turnService.GetActiveTurn(ctx, sessionID)
|
|
if lookupErr == nil && active != nil && active.ID == turnID {
|
|
s.activeTurns.Store(sessionID, turnID)
|
|
}
|
|
}
|
|
|
|
// reconcileLifecycleClaim makes the atomic SQLite RUNNING claim observable to
|
|
// the task runtime before a lifecycle prompt creates visible work or reaches
|
|
// the executor.
|
|
func (s *Service) reconcileLifecycleClaim(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
previousState models.TaskSessionState,
|
|
session *models.TaskSession,
|
|
) (v1.TaskState, bool, error) {
|
|
previousTaskState, taskStateClaimed, err := s.claimLifecycleTaskState(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
s.restoreLifecycleClaim(ctx, taskID, sessionID, previousState)
|
|
return "", false, fmt.Errorf("%w: reconcile lifecycle task state: %v", errLifecyclePromptClaim, err)
|
|
}
|
|
if session == nil || session.State != models.TaskSessionStateRunning {
|
|
s.restoreLifecycleClaim(ctx, taskID, sessionID, previousState)
|
|
if taskStateClaimed {
|
|
s.restoreLifecycleTaskState(ctx, taskID, previousTaskState)
|
|
}
|
|
return "", false, fmt.Errorf("%w: lifecycle claim no longer owns running session", errLifecyclePromptClaim)
|
|
}
|
|
s.publishTaskSessionStateChanged(
|
|
ctx, taskID, sessionID, previousState, models.TaskSessionStateRunning,
|
|
"", &session.UpdatedAt, session,
|
|
)
|
|
return previousTaskState, taskStateClaimed, nil
|
|
}
|
|
|
|
func (s *Service) claimLifecycleTaskState(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
) (v1.TaskState, bool, error) {
|
|
s.taskRuntimeStateMu.Lock()
|
|
defer s.taskRuntimeStateMu.Unlock()
|
|
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if task == nil || taskArchived(task) || task.AssigneeAgentProfileID != "" {
|
|
return "", false, nil
|
|
}
|
|
taskState, err := s.taskRepo.GetTask(ctx, taskID)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
previousTaskState := task.State
|
|
if taskState != nil {
|
|
previousTaskState = taskState.State
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
if !runtimeSessionOwnsTaskState(session, v1.TaskStateInProgress) {
|
|
return previousTaskState, false, nil
|
|
}
|
|
updated, err := s.taskRepo.UpdateTaskStateIfSessionState(
|
|
ctx, taskID, sessionID, session.State, v1.TaskStateInProgress,
|
|
)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
return previousTaskState, updated, nil
|
|
}
|
|
|
|
func (s *Service) restoreLifecycleTaskState(
|
|
ctx context.Context,
|
|
taskID string,
|
|
previousState v1.TaskState,
|
|
) {
|
|
if previousState == "" {
|
|
return
|
|
}
|
|
updated, err := s.taskRepo.UpdateTaskStateIfCurrentIn(
|
|
ctx, taskID, previousState, []v1.TaskState{v1.TaskStateInProgress},
|
|
)
|
|
if err != nil {
|
|
s.logger.Error("failed to restore task state after lifecycle prompt rejection",
|
|
zap.String("task_id", taskID),
|
|
zap.String("previous_state", string(previousState)),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if !updated {
|
|
s.logger.Debug("skipped lifecycle task-state restore after concurrent transition",
|
|
zap.String("task_id", taskID),
|
|
zap.String("previous_state", string(previousState)))
|
|
}
|
|
}
|
|
|
|
// handlePromptDispatchFailure handles a failed executor.Prompt call from
|
|
// promptTask. If the failure is the specific "missing execution" error a
|
|
// just-resumed session can hit (see promptTask's resumedForPrompt comment),
|
|
// it falls back to a fresh launch instead of surfacing the error to the
|
|
// caller. Otherwise — or if that fallback launch itself fails — it
|
|
// delegates to handlePromptError for the caller-facing result.
|
|
// promptAlreadyComposed/fallbackRetryPrompt are the caller's promptTaskOptions
|
|
// values, forwarded to fallbackFreshLaunchOnMissingExecution so a caller that
|
|
// already composed the dispatch prompt (e.g. autoStartStepPrompt, which
|
|
// appends a claimed step handoff) is not silently recomposed from the
|
|
// destination step's own template by this internal recovery.
|
|
func (s *Service) handlePromptDispatchFailure(
|
|
ctx context.Context,
|
|
taskID, sessionID, prompt string,
|
|
planMode, resumedForPrompt bool,
|
|
attachments []v1.MessageAttachment,
|
|
rollback promptClaimRollback,
|
|
lifecyclePrompt bool,
|
|
dispatchAccepted bool,
|
|
promptErr error,
|
|
promptAlreadyComposed bool,
|
|
fallbackLaunchPrompt string,
|
|
fallbackRetryPrompt string,
|
|
promptReferenceContext string,
|
|
) (*PromptResult, error) {
|
|
if resumedForPrompt && !dispatchAccepted && !rollback.reservedTurnAccepted && rollback.reservedTurn == nil &&
|
|
errors.Is(promptErr, executor.ErrExecutionNotFound) {
|
|
s.logger.Warn("prompt after lazy resume hit missing execution; falling back to fresh launch",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
fallbackPrompt := prompt
|
|
if fallbackLaunchPrompt != "" {
|
|
fallbackPrompt = fallbackLaunchPrompt
|
|
}
|
|
if freshErr := s.fallbackFreshLaunchOnMissingExecution(
|
|
ctx, taskID, sessionID, fallbackPrompt, promptAlreadyComposed, fallbackRetryPrompt, planMode,
|
|
promptReferenceContext, false, nil, attachments, nil,
|
|
); freshErr == nil {
|
|
return &PromptResult{}, nil
|
|
} else {
|
|
promptErr = freshErr
|
|
}
|
|
}
|
|
if lifecyclePrompt {
|
|
s.rollbackPromptClaim(ctx, taskID, sessionID, rollback)
|
|
return nil, promptErr
|
|
}
|
|
if rollback.createdTurn && rollback.reservedTurn != nil && !rollback.reservedTurnAccepted {
|
|
s.rollbackReservedPromptTurn(ctx, sessionID, rollback.turnID)
|
|
}
|
|
return nil, s.handlePromptError(ctx, taskID, sessionID, rollback.previousSessionState, promptErr)
|
|
}
|
|
|
|
// effectivePromptForSession applies promptTask's config-mode and plan-mode
|
|
// prompt transforms to the raw prompt, in that order, using session's
|
|
// *current* Metadata. Each call always starts from the raw prompt argument
|
|
// (never a previously-transformed result), so it is safe for promptTask to
|
|
// call this twice — once on session load and again after a reload that
|
|
// follows ensureSessionRunning, in case metadata changed in between —
|
|
// without double-prepending either transform's system-prompt wrapper.
|
|
func (s *Service) effectivePromptForSession(sessionID, prompt string, planMode bool, session *models.TaskSession) string {
|
|
return s.effectivePromptForSessionWithConfigMode(sessionID, prompt, planMode, session, nil)
|
|
}
|
|
|
|
func (s *Service) effectivePromptForSessionWithConfigMode(
|
|
sessionID, prompt string,
|
|
planMode bool,
|
|
session *models.TaskSession,
|
|
configModeOverride *bool,
|
|
) string {
|
|
effectivePrompt := prompt
|
|
configMode := false
|
|
if session != nil {
|
|
configMode, _ = session.Metadata["config_mode"].(bool)
|
|
}
|
|
if configModeOverride != nil {
|
|
configMode = *configModeOverride
|
|
}
|
|
if configMode {
|
|
effectivePrompt = sysprompt.InjectConfigContext(sessionID, prompt)
|
|
}
|
|
if planMode {
|
|
effectivePrompt = sysprompt.InjectPlanMode(effectivePrompt)
|
|
}
|
|
return effectivePrompt
|
|
}
|
|
|
|
var (
|
|
errLifecyclePromptInactive = errors.New("lifecycle prompt task or session is inactive")
|
|
errLifecyclePromptClaim = errors.New("lifecycle prompt claim failed")
|
|
errLifecyclePromptMessagePersistence = errors.New("lifecycle prompt message persistence failed")
|
|
errLifecyclePromptReservationSuperseded = errors.New("lifecycle prompt reservation was superseded")
|
|
)
|
|
|
|
type lifecyclePromptReselectedError struct {
|
|
sessionID string
|
|
}
|
|
|
|
func (e *lifecyclePromptReselectedError) Error() string {
|
|
return fmt.Sprintf("lifecycle prompt session reselected: %s", e.sessionID)
|
|
}
|
|
|
|
// claimSessionRunningForPrompt marks sessionID RUNNING immediately before
|
|
// promptTask dispatches to the agent, and returns the session to use for
|
|
// the rest of the call (possibly reloaded).
|
|
//
|
|
// Direct and queued prompts both claim RUNNING under the per-session
|
|
// cancelInFlight guard. This makes the final "still promptable, now owned by
|
|
// this turn" decision atomic with coordinator stop; the potentially blocking
|
|
// executor.Prompt call remains outside the guard.
|
|
//
|
|
// When claimEntryID is set, the final claim happens under the same per-session
|
|
// cancelInFlightGuard every other cancel/take-and-dispatch decision in this
|
|
// file serializes through. Re-verifying
|
|
// ownership *and* promptability inside this same critical section,
|
|
// immediately before writing RUNNING, closes a gap that checking the token
|
|
// alone would leave open: nothing would otherwise stop this call from
|
|
// blindly marking the session RUNNING over top of a state some other
|
|
// guard-synchronized decision-maker (a cancel, another claim) already
|
|
// changed since promptTask's much-earlier, unguarded checkSessionPromptable
|
|
// call near its top. Reloading the session and re-running
|
|
// checkSessionPromptable here makes "is it actually still safe to mark this
|
|
// session RUNNING right now" an atomic fact, not a stale read.
|
|
func (s *Service) claimSessionRunningForPrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
reserveTurnUntilDispatch bool,
|
|
promptDispatchRecovery *models.PromptDispatchRecovery,
|
|
foregroundClaim *foregroundClaim,
|
|
expectedCurrentTurnID string,
|
|
requireNonterminalSession bool,
|
|
optionalClaimArgs ...interface{},
|
|
) (*models.TaskSession, models.TaskSessionState, string, bool, *models.Turn, func(), error) {
|
|
var (
|
|
afterClaim func() error
|
|
expectedIdentity *messagequeue.QueueSessionIdentity
|
|
startupAttempt *resumeAttempt
|
|
admissionGuard *lockedCancelInFlightGuard
|
|
)
|
|
for _, arg := range optionalClaimArgs {
|
|
switch value := arg.(type) {
|
|
case func() error:
|
|
afterClaim = value
|
|
case *messagequeue.QueueSessionIdentity:
|
|
expectedIdentity = value
|
|
case *resumeAttempt:
|
|
startupAttempt = value
|
|
case *lockedCancelInFlightGuard:
|
|
admissionGuard = value
|
|
}
|
|
}
|
|
if admissionGuard == nil {
|
|
admissionGuard = s.lockCancelInFlightGuard(sessionID)
|
|
}
|
|
keepDispatchGuard := false
|
|
defer func() {
|
|
if !keepDispatchGuard {
|
|
admissionGuard.release()
|
|
}
|
|
}()
|
|
lock := admissionGuard.mutex
|
|
// A reset-owned cancellation must not make prompt admission wait for the
|
|
// reset that already forbids it. Keep the second check below for a reset
|
|
// that starts while this prompt is waiting on an unrelated cancellation.
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return nil, "", "", false, nil, nil, ErrSessionResetInProgress
|
|
}
|
|
if err := s.waitForCancellationWithGuard(ctx, sessionID, lock.Unlock, lock.Lock); err != nil {
|
|
return nil, "", "", false, nil, nil, err
|
|
}
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return nil, "", "", false, nil, nil, ErrSessionResetInProgress
|
|
}
|
|
if err := s.validateResumeAttempt(startupAttempt); err != nil {
|
|
return nil, "", "", false, nil, nil, err
|
|
}
|
|
if s.isRouteActionInFlight(sessionID) {
|
|
return nil, "", "", false, nil, nil, fmt.Errorf("%w, route action is in progress", ErrAgentPromptInProgress)
|
|
}
|
|
if expectedCurrentTurnID != "" {
|
|
if s.turnService == nil {
|
|
return nil, "", "", false, nil, nil, errors.New("cannot verify expected prompt turn without turn service")
|
|
}
|
|
activeTurn, err := s.turnService.GetActiveTurn(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, "", "", false, nil, nil, fmt.Errorf("verify expected prompt turn: %w", err)
|
|
}
|
|
if activeTurn == nil || activeTurn.ID != expectedCurrentTurnID {
|
|
return nil, "", "", false, nil, nil, fmt.Errorf(
|
|
"clarification turn %s is no longer current",
|
|
expectedCurrentTurnID,
|
|
)
|
|
}
|
|
}
|
|
|
|
if claimEntryID != "" && !s.isCurrentQueuedDispatch(sessionID, claimEntryID) {
|
|
return nil, "", "", false, nil, nil, errQueuedDispatchSuperseded
|
|
}
|
|
freshSession, sessErr := s.repo.GetTaskSession(ctx, sessionID)
|
|
if sessErr != nil {
|
|
return nil, "", "", false, nil, nil, fmt.Errorf("reload session before claiming queued dispatch: %w", sessErr)
|
|
}
|
|
if freshSession == nil {
|
|
return nil, "", "", false, nil, nil, errQueuedDispatchSuperseded
|
|
}
|
|
reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID)
|
|
if !reservation.matchesSessionIdentity(
|
|
freshSession.TaskID,
|
|
freshSession.ID,
|
|
freshSession.QueueIncarnationID,
|
|
) {
|
|
return nil, "", "", false, nil, nil, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if expectedIdentity != nil &&
|
|
(freshSession.TaskID != expectedIdentity.TaskID ||
|
|
freshSession.ID != expectedIdentity.SessionID ||
|
|
freshSession.QueueIncarnationID != expectedIdentity.SessionIncarnationID) {
|
|
return nil, "", "", false, nil, nil, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if requireNonterminalSession && isTerminalSessionState(freshSession.State) {
|
|
return nil, "", "", false, nil, nil, newWorkflowAutoStartSessionTerminalizedError(freshSession)
|
|
}
|
|
if promptErr := s.recheckPromptableWithForegroundClaim(
|
|
taskID, sessionID, freshSession.State, foregroundClaim,
|
|
); promptErr != nil {
|
|
return nil, "", "", false, nil, nil, promptErr
|
|
}
|
|
previousState := freshSession.State
|
|
switch {
|
|
case reservation != nil && reservation.identity.SessionIncarnationID != "":
|
|
if err := s.setQueuedSessionRunningForIdentity(ctx, reservation.identity, freshSession); err != nil {
|
|
return nil, "", "", false, nil, nil, err
|
|
}
|
|
case expectedIdentity != nil:
|
|
if err := s.setQueuedSessionRunningForIdentity(ctx, *expectedIdentity, freshSession); err != nil {
|
|
return nil, "", "", false, nil, nil, err
|
|
}
|
|
default:
|
|
s.setSessionRunning(ctx, taskID, sessionID, freshSession)
|
|
}
|
|
// The guarded state transition refreshes freshSession in place, so a
|
|
// competing terminal transition remains visible before turn creation.
|
|
if requireNonterminalSession && isTerminalSessionState(freshSession.State) {
|
|
return nil, "", "", false, nil, nil, newWorkflowAutoStartSessionTerminalizedError(freshSession)
|
|
}
|
|
if isTerminalSessionState(freshSession.State) && freshSession.State != models.TaskSessionStateCompleted {
|
|
return nil, "", "", false, nil, nil, &executor.SessionStateSupersededError{
|
|
SessionID: freshSession.ID,
|
|
State: freshSession.State,
|
|
}
|
|
}
|
|
turnID, createdTurn, reservedTurn, err := s.startTurnForSessionWithOwnershipChecked(
|
|
ctx, sessionID, reserveTurnUntilDispatch, promptDispatchRecovery,
|
|
)
|
|
if err != nil {
|
|
rollback := promptClaimRollback{previousSessionState: previousState}
|
|
if reservation != nil {
|
|
rollback.sessionIdentity = reservation.identity
|
|
}
|
|
if expectedIdentity != nil {
|
|
rollback.sessionIdentity = *expectedIdentity
|
|
}
|
|
s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback)
|
|
return nil, "", "", false, nil, nil, fmt.Errorf("persist prompt turn: %w", err)
|
|
}
|
|
if reservedTurn != nil && s.turnService != nil {
|
|
if err := s.turnService.MarkReservedTurnDispatchAttempted(ctx, reservedTurn); err != nil {
|
|
rollback := promptClaimRollback{
|
|
previousSessionState: previousState,
|
|
turnID: turnID,
|
|
createdTurn: createdTurn,
|
|
reservedTurn: reservedTurn,
|
|
}
|
|
if reservation != nil {
|
|
rollback.sessionIdentity = reservation.identity
|
|
}
|
|
if expectedIdentity != nil {
|
|
rollback.sessionIdentity = *expectedIdentity
|
|
}
|
|
s.rollbackPromptClaimAfterAdmissionFailure(ctx, taskID, sessionID, rollback)
|
|
return nil, "", "", false, nil, nil, fmt.Errorf("persist prompt dispatch attempt: %w", err)
|
|
}
|
|
}
|
|
reservation = s.queuedDispatchReservationForEntry(sessionID, claimEntryID)
|
|
rollback := promptClaimRollback{
|
|
previousSessionState: previousState,
|
|
turnID: turnID,
|
|
createdTurn: createdTurn,
|
|
reservedTurn: reservedTurn,
|
|
}
|
|
if reservation != nil {
|
|
rollback.sessionIdentity = reservation.identity
|
|
}
|
|
if expectedIdentity != nil {
|
|
rollback.sessionIdentity = *expectedIdentity
|
|
}
|
|
if err := s.acknowledgePromptClaim(ctx, taskID, sessionID, afterClaim, rollback); err != nil {
|
|
return nil, "", "", false, nil, nil, err
|
|
}
|
|
// A queued reservation becomes replaceable only after the guarded session
|
|
// claim and its identity-bound visible side effects have succeeded.
|
|
if reservation != nil {
|
|
s.markAcceptedDispatchLiveLocked(sessionID, reservation)
|
|
}
|
|
// Remove only the pre-acceptance marker here. Transfer the guard to
|
|
// promptTask for queued dispatches so cancellation cannot supersede this
|
|
// reservation between live promotion and provider acceptance.
|
|
var dispatchGuardRelease func()
|
|
if claimEntryID != "" {
|
|
var releaseOnce sync.Once
|
|
dispatchGuardRelease = func() {
|
|
releaseOnce.Do(admissionGuard.release)
|
|
}
|
|
keepDispatchGuard = true
|
|
}
|
|
if claimEntryID != "" {
|
|
s.releaseQueuedDispatchPendingIfCurrent(sessionID, reservation)
|
|
}
|
|
return freshSession, previousState, turnID, createdTurn, reservedTurn, dispatchGuardRelease, nil
|
|
}
|
|
|
|
func (s *Service) validateQueuedPromptDispatch(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
taskID, sessionID string,
|
|
session *models.TaskSession,
|
|
) (*models.TaskSession, error) {
|
|
if identity.SessionIncarnationID == "" {
|
|
return session, nil
|
|
}
|
|
if s.messageQueue == nil {
|
|
return nil, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
expectedExecutionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if err != nil || expectedExecutionID == "" {
|
|
return nil, executor.ErrExecutionNotFound
|
|
}
|
|
fresh, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if fresh == nil ||
|
|
fresh.TaskID != taskID ||
|
|
fresh.ID != identity.SessionID ||
|
|
fresh.QueueIncarnationID != identity.SessionIncarnationID {
|
|
return nil, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
currentExecutionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if err != nil || currentExecutionID != expectedExecutionID {
|
|
return nil, executor.ErrExecutionNotFound
|
|
}
|
|
fresh.AgentExecutionID = expectedExecutionID
|
|
return fresh, nil
|
|
}
|
|
|
|
func (s *Service) claimLifecycleSessionRunning(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
) (*models.TaskSession, models.TaskSessionState, string, bool, error) {
|
|
return s.claimLifecycleSessionRunningWithResumeAttempt(ctx, taskID, sessionID, claimEntryID, nil)
|
|
}
|
|
|
|
func (s *Service) claimLifecycleSessionRunningWithResumeAttempt(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
resumeAttempt *resumeAttempt,
|
|
) (*models.TaskSession, models.TaskSessionState, string, bool, error) {
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
if err := s.validateLifecycleSessionAdmission(ctx, sessionID, resumeAttempt, lock); err != nil {
|
|
return nil, "", "", false, err
|
|
}
|
|
reservation, claim, err := s.claimLifecyclePrompt(ctx, taskID, sessionID, claimEntryID)
|
|
if err != nil {
|
|
return nil, "", "", false, err
|
|
}
|
|
if err := s.validateClaimedLifecyclePrompt(ctx, sessionID, claimEntryID, resumeAttempt); err != nil {
|
|
s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState)
|
|
return nil, "", "", false, err
|
|
}
|
|
freshSession, err := s.loadClaimedLifecycleSession(ctx, sessionID, reservation)
|
|
if err != nil {
|
|
s.restoreLifecycleClaimForReservation(ctx, taskID, sessionID, reservation, claim.PreviousState)
|
|
return nil, "", "", false, err
|
|
}
|
|
// Keep the lifecycle admission guard through durable turn creation. A reset
|
|
// that follows this claim must observe this turn and quiesce it before it
|
|
// replaces the provider context.
|
|
turnID, createdTurn, _, err := s.startTurnForSessionWithOwnershipChecked(ctx, sessionID, false, nil)
|
|
if err != nil {
|
|
failureCtx, cancel := (promptTaskOptions{}).failureContext(ctx)
|
|
defer cancel()
|
|
s.restoreLifecycleClaimForReservation(
|
|
failureCtx, taskID, sessionID, reservation, claim.PreviousState,
|
|
)
|
|
return nil, "", "", false, fmt.Errorf("persist lifecycle prompt turn: %w", err)
|
|
}
|
|
s.releaseQueuedDispatchPendingIfCurrent(sessionID, reservation)
|
|
return freshSession, claim.PreviousState, turnID, createdTurn, nil
|
|
}
|
|
|
|
func (s *Service) validateLifecycleSessionAdmission(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
resumeAttempt *resumeAttempt,
|
|
lock *sync.Mutex,
|
|
) error {
|
|
// Match ordinary prompt admission: a reset-owned cancellation must reject
|
|
// lifecycle admission before its cancellation wait can observe ctx.Err().
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
if err := s.waitForCancellationWithGuard(ctx, sessionID, lock.Unlock, lock.Lock); err != nil {
|
|
return err
|
|
}
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
if err := s.validateResumeAttempt(resumeAttempt); err != nil {
|
|
return err
|
|
}
|
|
if s.isRouteActionInFlight(sessionID) {
|
|
return fmt.Errorf("%w, route action is in progress", ErrAgentPromptInProgress)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) claimLifecyclePrompt(
|
|
ctx context.Context,
|
|
taskID, sessionID, claimEntryID string,
|
|
) (*queuedDispatchReservation, models.PromptableTaskSessionClaim, error) {
|
|
if claimEntryID != "" && !s.isCurrentQueuedDispatch(sessionID, claimEntryID) {
|
|
return nil, models.PromptableTaskSessionClaim{}, errQueuedDispatchSuperseded
|
|
}
|
|
if err := s.validateLifecyclePromptSelection(ctx, taskID, sessionID); err != nil {
|
|
return nil, models.PromptableTaskSessionClaim{}, err
|
|
}
|
|
reservation := s.queuedDispatchReservationForEntry(sessionID, claimEntryID)
|
|
claim, err := s.claimPromptableSessionForIdentity(ctx, taskID, sessionID, reservation)
|
|
if err != nil {
|
|
return nil, models.PromptableTaskSessionClaim{}, fmt.Errorf("%w: %v", errLifecyclePromptClaim, err)
|
|
}
|
|
switch claim.Status {
|
|
case models.PromptableTaskSessionInactive:
|
|
return nil, models.PromptableTaskSessionClaim{}, errLifecyclePromptInactive
|
|
case models.PromptableTaskSessionBusy:
|
|
return nil, models.PromptableTaskSessionClaim{}, fmt.Errorf("%w: lifecycle prompt session is busy", ErrAgentPromptInProgress)
|
|
}
|
|
return reservation, claim, nil
|
|
}
|
|
|
|
func (s *Service) validateClaimedLifecyclePrompt(
|
|
ctx context.Context,
|
|
sessionID, claimEntryID string,
|
|
resumeAttempt *resumeAttempt,
|
|
) error {
|
|
if claimEntryID != "" && !s.isCurrentQueuedDispatch(sessionID, claimEntryID) {
|
|
return errQueuedDispatchSuperseded
|
|
}
|
|
if err := s.validateResumeAttempt(resumeAttempt); err != nil {
|
|
return err
|
|
}
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) loadClaimedLifecycleSession(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
reservation *queuedDispatchReservation,
|
|
) (*models.TaskSession, error) {
|
|
freshSession, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || freshSession == nil {
|
|
return nil, errLifecyclePromptInactive
|
|
}
|
|
if !reservation.matchesSessionIdentity(
|
|
freshSession.TaskID,
|
|
freshSession.ID,
|
|
freshSession.QueueIncarnationID,
|
|
) {
|
|
return nil, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
return freshSession, nil
|
|
}
|
|
|
|
func (s *Service) claimPromptableSessionForIdentity(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
reservation *queuedDispatchReservation,
|
|
) (models.PromptableTaskSessionClaim, error) {
|
|
if reservation == nil || reservation.identity.SessionIncarnationID == "" {
|
|
return s.repo.ClaimPromptableTaskSessionIfActive(ctx, sessionID)
|
|
}
|
|
return s.repo.ClaimPromptableTaskSessionIfActiveForIdentity(
|
|
ctx,
|
|
reservation.identity.TaskID,
|
|
reservation.identity.SessionID,
|
|
reservation.identity.SessionIncarnationID,
|
|
)
|
|
}
|
|
|
|
func (s *Service) restoreLifecycleClaimForReservation(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
reservation *queuedDispatchReservation,
|
|
previousState models.TaskSessionState,
|
|
) {
|
|
if reservation != nil && reservation.identity.SessionIncarnationID != "" {
|
|
s.restoreLifecycleClaimForIdentity(ctx, reservation.identity, previousState)
|
|
return
|
|
}
|
|
s.restoreLifecycleClaim(ctx, taskID, sessionID, previousState)
|
|
}
|
|
|
|
func (s *Service) restoreLifecycleClaimForIdentity(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
previousState models.TaskSessionState,
|
|
) {
|
|
restored, updatedAt, err := s.repo.UpdateTaskSessionStateIfCurrentIdentity(
|
|
ctx,
|
|
identity.TaskID,
|
|
identity.SessionID,
|
|
identity.SessionIncarnationID,
|
|
models.TaskSessionStateRunning,
|
|
previousState,
|
|
"",
|
|
)
|
|
if err != nil {
|
|
s.logger.Error("failed to restore lifecycle prompt session state",
|
|
zap.String("task_id", identity.TaskID),
|
|
zap.String("session_id", identity.SessionID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
if !restored {
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, identity.SessionID)
|
|
if err != nil || session == nil ||
|
|
session.TaskID != identity.TaskID ||
|
|
session.QueueIncarnationID != identity.SessionIncarnationID {
|
|
return
|
|
}
|
|
s.publishTaskSessionStateChanged(
|
|
ctx,
|
|
identity.TaskID,
|
|
identity.SessionID,
|
|
models.TaskSessionStateRunning,
|
|
previousState,
|
|
"",
|
|
&updatedAt,
|
|
session,
|
|
)
|
|
s.republishTaskActivityOnSettle(
|
|
ctx, identity.TaskID, models.TaskSessionStateRunning, previousState,
|
|
)
|
|
}
|
|
|
|
func (s *Service) validateLifecyclePromptSelection(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
) error {
|
|
selectedSession, err := s.resolveTaskPRAgentSession(ctx, taskID)
|
|
if err != nil || selectedSession == nil {
|
|
task, taskErr := s.repo.GetTask(ctx, taskID)
|
|
if taskErr != nil || task == nil || task.ArchivedAt != nil {
|
|
return errLifecyclePromptInactive
|
|
}
|
|
return fmt.Errorf("%w: no promptable lifecycle session", ErrAgentPromptInProgress)
|
|
}
|
|
if selectedSession.ID != sessionID {
|
|
return &lifecyclePromptReselectedError{sessionID: selectedSession.ID}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) validateLifecyclePromptTurn(
|
|
ctx context.Context,
|
|
sessionID, turnID string,
|
|
) error {
|
|
if s.isSessionResetInProgress(sessionID) {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
if s.turnService == nil || turnID == "" {
|
|
return nil
|
|
}
|
|
activeTurn, err := s.turnService.GetActiveTurn(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("verify lifecycle prompt turn: %w", err)
|
|
}
|
|
if activeTurn == nil || activeTurn.ID != turnID {
|
|
return ErrSessionResetInProgress
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) restoreLifecycleClaim(
|
|
ctx context.Context,
|
|
taskID, sessionID string,
|
|
previousState models.TaskSessionState,
|
|
) {
|
|
s.updateTaskSessionState(ctx, taskID, sessionID, previousState, "", false)
|
|
}
|
|
|
|
func (s *Service) recheckPromptableWithForegroundClaim(
|
|
taskID, sessionID string,
|
|
state models.TaskSessionState,
|
|
claim *foregroundClaim,
|
|
) error {
|
|
if claim == nil {
|
|
return s.checkSessionPromptable(taskID, sessionID, state)
|
|
}
|
|
if state != models.TaskSessionStateRunning {
|
|
// ensureSessionRunning may resume the agent after this prompt claimed a
|
|
// background-idle RUNNING session. AgentBootReady advances the durable
|
|
// state to WAITING_FOR_INPUT before dispatch; keep the valid claim across
|
|
// that expected transition while still rejecting terminal or otherwise
|
|
// non-promptable states.
|
|
if err := s.checkSessionPromptable(taskID, sessionID, state); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !s.isForegroundClaimCurrent(sessionID, claim) {
|
|
return fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// checkSessionPromptable returns nil when the session's state accepts a new
|
|
// prompt. RUNNING is rejected with ErrAgentPromptInProgress; any other
|
|
// non-acceptable state (STARTING / CREATED / FAILED / CANCELLED) is rejected
|
|
// with ErrSessionNotPromptable so callers can distinguish "wait for the
|
|
// current turn" from "this session is not in a state where it can take a
|
|
// prompt". IDLE is acceptable: office sessions intentionally park in IDLE
|
|
// between turns (agent torn down, ACP session preserved) and the next prompt
|
|
// resumes them — see ensureSessionRunning.
|
|
func (s *Service) checkSessionPromptable(taskID, sessionID string, state models.TaskSessionState) error {
|
|
switch state {
|
|
case models.TaskSessionStateWaitingForInput,
|
|
models.TaskSessionStateCompleted,
|
|
models.TaskSessionStateIdle:
|
|
return nil
|
|
case models.TaskSessionStateRunning:
|
|
// The safe default is coarse: every RUNNING session is busy. A
|
|
// deployment may explicitly opt a persisted Claude Code session into
|
|
// ADR-0049's adapter-attested background handoff experiment. Every
|
|
// missing identity and non-Claude provider still fails closed.
|
|
if s.ForegroundActivity(sessionID) == v1.ForegroundActivityBackground {
|
|
s.logger.Debug("accepting prompt: enabled Claude foreground handoff is background-idle",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID))
|
|
return nil
|
|
}
|
|
s.logger.Warn("rejected prompt while agent is already running",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("session_state", string(state)))
|
|
return fmt.Errorf("%w, please wait for completion", ErrAgentPromptInProgress)
|
|
default:
|
|
s.logger.Warn("rejected prompt: session not ready for input",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("session_state", string(state)))
|
|
return fmt.Errorf("%w: session is in %s state", ErrSessionNotPromptable, state)
|
|
}
|
|
}
|
|
|
|
// handlePromptError reverts session state, logs the failure, transitions the
|
|
// task to REVIEW for non-transient errors, and completes the in-flight turn.
|
|
// Returns the (possibly remapped) error for the caller to surface.
|
|
func (s *Service) handlePromptError(ctx context.Context, taskID, sessionID string, previousSessionState models.TaskSessionState, err error) error {
|
|
if isTransientPromptError(err) && s.isSessionResetInProgress(sessionID) {
|
|
s.logger.Warn("prompt deferred while session reset is in progress; retry expected",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
err = ErrSessionResetInProgress
|
|
} else {
|
|
s.logger.Error("prompt failed",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
// Revert session state so it doesn't stay stuck in RUNNING. Route through the
|
|
// wrapper so WS subscribers are notified — the UI relies on the
|
|
// session.state_changed broadcast to flip the composer/pause button out of
|
|
// "Agent is running". The wrapper's terminal-state guard is also correct here:
|
|
// if a concurrent agent-failure handler moved the session to FAILED we don't
|
|
// want to overwrite that with previousSessionState. Do NOT pass the preloaded
|
|
// `session` — the wrapper must re-read the row to see any such concurrent
|
|
// terminal transition; the stale pre-RUNNING snapshot would defeat the guard.
|
|
// allowWakeFromWaiting=false — this is a revert away from RUNNING, never a
|
|
// wake transition; the flag only matters when going WAITING_FOR_INPUT → RUNNING.
|
|
s.updateTaskSessionState(ctx, taskID, sessionID, previousSessionState, "", false)
|
|
// ErrCancelEscalated means the user cancelled and the lifecycle manager had to
|
|
// force-unblock a hung agent. Service.CancelAgent owns the cancel reconcile
|
|
// (session → WAITING_FOR_INPUT, task → REVIEW, cancel message, complete
|
|
// turn); skip the REVIEW write here so we don't race that path with a
|
|
// duplicate update.
|
|
// A short transient provider error is owned by the async
|
|
// retry-with-backoff path (handleTransientFailure), which keeps the task
|
|
// in progress while it retries — so don't flap it to REVIEW here.
|
|
if !isTransientPromptError(err) && !errors.Is(err, lifecycle.ErrCancelEscalated) &&
|
|
!routingerr.IsTransientProviderError(err.Error()) {
|
|
s.writeTaskReviewState(ctx, taskID, sessionID)
|
|
}
|
|
s.completeTurnForSession(ctx, sessionID)
|
|
return err
|
|
}
|
|
|
|
// trySwitchModel handles model switching for a prompt. Returns (result, true, nil) if a switch was
|
|
// performed, (nil, false, err) on error, or (nil, false, nil) if no switch was needed.
|
|
func modelSwitchRequired(session *models.TaskSession, model string) bool {
|
|
if model == "" || session == nil {
|
|
return false
|
|
}
|
|
var currentModel string
|
|
if session.AgentProfileSnapshot != nil {
|
|
if m, ok := session.AgentProfileSnapshot["model"].(string); ok {
|
|
currentModel = m
|
|
}
|
|
}
|
|
return currentModel != model
|
|
}
|
|
|
|
func (s *Service) trySwitchModel(ctx context.Context, taskID, sessionID, model, effectivePrompt string, session *models.TaskSession) (*PromptResult, bool, error) {
|
|
return s.trySwitchModelWithDispatchCallbacks(ctx, taskID, sessionID, model, effectivePrompt, session, nil, nil)
|
|
}
|
|
|
|
func (s *Service) trySwitchModelWithDispatchCallbacks(
|
|
ctx context.Context,
|
|
taskID, sessionID, model, effectivePrompt string,
|
|
session *models.TaskSession,
|
|
onDispatched func(executionID string),
|
|
onFailure func(),
|
|
) (*PromptResult, bool, error) {
|
|
if !modelSwitchRequired(session, model) {
|
|
return nil, false, nil
|
|
}
|
|
var currentModel string
|
|
if session.AgentProfileSnapshot != nil {
|
|
if m, ok := session.AgentProfileSnapshot["model"].(string); ok {
|
|
currentModel = m
|
|
}
|
|
}
|
|
s.logger.Info("switching model",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("from", currentModel),
|
|
zap.String("to", model))
|
|
switchCtx := context.WithoutCancel(ctx)
|
|
switchResult, err := s.executor.SwitchModelWithDispatchCallbacks(
|
|
switchCtx, taskID, sessionID, model, effectivePrompt, onDispatched, onFailure,
|
|
)
|
|
if err != nil {
|
|
return nil, true, fmt.Errorf("model switch failed: %w", err)
|
|
}
|
|
s.runtimeModelBySession.Store(sessionID, model)
|
|
if switchResult.StopReason == "model_switched_in_place" {
|
|
// Agent is still running with the new model — let PromptTask send the prompt normally.
|
|
// Invalidate the message creator's model cache so the next message picks up the new model.
|
|
if s.messageCreator != nil {
|
|
s.messageCreator.InvalidateModelCache(sessionID)
|
|
}
|
|
return nil, false, nil
|
|
}
|
|
s.startTurnForSession(ctx, sessionID)
|
|
s.setSessionRunning(ctx, taskID, sessionID, session)
|
|
return &PromptResult{
|
|
StopReason: switchResult.StopReason,
|
|
AgentMessage: switchResult.AgentMessage,
|
|
}, true, nil
|
|
}
|
|
|
|
// RespondToPermission is the existing web/internal compatibility entry point.
|
|
// Option selections use the same strict audited service as external MCP;
|
|
// dismissal uses a separate generation-safe internal cancellation operation.
|
|
func (s *Service) RespondToPermission(ctx context.Context, taskID, sessionID, requestID, pendingID, optionID string, cancelled, rejected bool) error {
|
|
_ = rejected // The immutable provider option kind determines rejection.
|
|
request := ResolveAgentPermissionRequest{
|
|
TaskID: taskID, SessionID: sessionID, RequestID: requestID,
|
|
PendingID: pendingID, OptionID: optionID, Source: models.PermissionSourceWeb,
|
|
}
|
|
if cancelled {
|
|
return s.cancelAgentPermission(ctx, request)
|
|
}
|
|
_, err := s.ResolveAgentPermission(ctx, request)
|
|
return err
|
|
}
|
|
|
|
// DrainQueuedMessage dispatches one queued message for a session that is ready
|
|
// for input. It is intentionally one-at-a-time: each successful prompt will
|
|
// complete its own turn and then drain the next entry through handleAgentReady.
|
|
func (s *Service) DrainQueuedMessage(ctx context.Context, sessionID string) (bool, error) {
|
|
if sessionID == "" {
|
|
return false, fmt.Errorf("session_id is required")
|
|
}
|
|
// Acquire the guard *before* checking promptability, not after: check-
|
|
// then-lock would leave a gap where a concurrent
|
|
// QueueAndInterruptForPeerMessage (or another drain) changes the
|
|
// session's state between this call's check and its take, letting a
|
|
// manual drain request race a parent interrupt for the same entry —
|
|
// see the Service.cancelInFlight field doc comment. A blocking Lock
|
|
// (not a TryLock skip) is deliberate: unlike handleAgentReady /
|
|
// handleAgentBootReady (where a losing side can rely on the winning
|
|
// side's own take-and-dispatch, or a future turn's own agent.ready, to
|
|
// retry), a manual drain request has no other future trigger —
|
|
// skipping here could strand an already-queued message instead of
|
|
// just reporting an accurate "not promptable right now".
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
guardLocked := true
|
|
defer func() {
|
|
if guardLocked {
|
|
lock.Unlock()
|
|
}
|
|
}()
|
|
for s.isCancelInFlight(sessionID) {
|
|
lock.Unlock()
|
|
guardLocked = false
|
|
if err := s.waitForCancelInFlight(ctx, sessionID); err != nil {
|
|
return false, err
|
|
}
|
|
lock.Lock()
|
|
guardLocked = true
|
|
}
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if err := s.checkSessionPromptable(session.TaskID, sessionID, session.State); err != nil {
|
|
return false, err
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, sessionID) {
|
|
return false, ErrSessionNotPromptable
|
|
}
|
|
if s.messageQueue == nil {
|
|
return false, errors.New("message queue is not configured")
|
|
}
|
|
if err := s.messageQueue.SetAutoRun(ctx, sessionID, true); err != nil {
|
|
return false, fmt.Errorf("resume queue Auto-run: %w", err)
|
|
}
|
|
return s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID), nil
|
|
}
|
|
|
|
// DrainQueuedMessageIfAutoRun dispatches one queued message only when the
|
|
// persisted Auto-run policy is already enabled. Unlike DrainQueuedMessage, it
|
|
// never changes the policy as a side effect.
|
|
func (s *Service) DrainQueuedMessageIfAutoRun(ctx context.Context, sessionID string) (bool, error) {
|
|
if sessionID == "" {
|
|
return false, fmt.Errorf("session_id is required")
|
|
}
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
if s.isCancelInFlight(sessionID) || s.isQueuedDispatchInFlight(sessionID) || s.isSteerInFlight(sessionID) {
|
|
return false, nil
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, sessionID) {
|
|
return false, nil
|
|
}
|
|
if s.messageQueue == nil {
|
|
return false, errors.New("message queue is not configured")
|
|
}
|
|
status := s.messageQueue.GetStatus(ctx, sessionID)
|
|
if !status.AutoRun {
|
|
return false, nil
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if err := s.checkSessionPromptable(session.TaskID, sessionID, session.State); err != nil {
|
|
return false, nil
|
|
}
|
|
return s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID), nil
|
|
}
|
|
|
|
// DrainQueuedMessageForSession dispatches one message only for the exact session incarnation.
|
|
func (s *Service) DrainQueuedMessageForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity) (bool, error) {
|
|
if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" {
|
|
return false, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
lock, release := s.acquireCancelInFlightGuard(identity.SessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
guardLocked := true
|
|
defer func() {
|
|
if guardLocked {
|
|
lock.Unlock()
|
|
}
|
|
}()
|
|
for s.isCancelInFlight(identity.SessionID) {
|
|
lock.Unlock()
|
|
guardLocked = false
|
|
if err := s.waitForCancelInFlight(ctx, identity.SessionID); err != nil {
|
|
return false, err
|
|
}
|
|
lock.Lock()
|
|
guardLocked = true
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, identity.SessionID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to get session: %w", err)
|
|
}
|
|
if session == nil || session.TaskID != identity.TaskID {
|
|
return false, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil {
|
|
return false, err
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, identity.SessionID) {
|
|
return false, ErrSessionNotPromptable
|
|
}
|
|
if s.messageQueue == nil {
|
|
return false, errors.New("message queue is not configured")
|
|
}
|
|
if err := s.messageQueue.SetAutoRunForSession(ctx, identity, true); err != nil {
|
|
return false, fmt.Errorf("resume queue Auto-run: %w", err)
|
|
}
|
|
return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity)
|
|
}
|
|
|
|
// SetQueueAutoRun persists the queue policy and, when enabling an eligible
|
|
// session, immediately attempts the FIFO head. It never cancels an active turn
|
|
// and treats busy, clarification, and lifecycle guards as a successful armed
|
|
// policy with no immediate dispatch.
|
|
func (s *Service) SetQueueAutoRun(ctx context.Context, sessionID string, enabled bool) (bool, bool, error) {
|
|
if sessionID == "" {
|
|
return false, false, errors.New("session_id is required")
|
|
}
|
|
if s.messageQueue == nil {
|
|
return false, false, errors.New("message queue is not configured")
|
|
}
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
|
|
if err := s.messageQueue.SetAutoRun(ctx, sessionID, enabled); err != nil {
|
|
return false, false, fmt.Errorf("set queue Auto-run: %w", err)
|
|
}
|
|
if !enabled || s.isCancelInFlight(sessionID) || s.isQueuedDispatchInFlight(sessionID) || s.isSteerInFlight(sessionID) {
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, false, nil
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, sessionID) {
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, false, nil
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, false, fmt.Errorf("load session for queue Auto-run: %w", err)
|
|
}
|
|
if err := s.checkSessionPromptable(session.TaskID, sessionID, session.State); err != nil {
|
|
if isSessionBusyError(err) {
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, false, nil
|
|
}
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, false, err
|
|
}
|
|
dispatched := s.drainQueuedMessageForPromptableSessionLocked(ctx, sessionID)
|
|
return s.messageQueue.GetStatus(ctx, sessionID).AutoRun, dispatched, nil
|
|
}
|
|
|
|
func (s *Service) queueAutoRunDrainBlocked(sessionID string) bool {
|
|
return s.isCancelInFlight(sessionID) ||
|
|
s.isQueuedDispatchInFlight(sessionID) ||
|
|
s.isSteerInFlight(sessionID)
|
|
}
|
|
|
|
// SetQueueAutoRunForSession persists policy and dispatches only for the exact session incarnation.
|
|
func (s *Service) SetQueueAutoRunForSession(ctx context.Context, identity messagequeue.QueueSessionIdentity, enabled bool) (bool, bool, error) {
|
|
if identity.TaskID == "" || identity.SessionID == "" || identity.SessionIncarnationID == "" {
|
|
return false, false, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if s.messageQueue == nil {
|
|
return false, false, errors.New("message queue is not configured")
|
|
}
|
|
lock, release := s.acquireCancelInFlightGuard(identity.SessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
|
|
if err := s.messageQueue.SetAutoRunForSession(ctx, identity, enabled); err != nil {
|
|
return false, false, fmt.Errorf("set queue Auto-run: %w", err)
|
|
}
|
|
if !enabled || s.queueAutoRunDrainBlocked(identity.SessionID) {
|
|
return enabled, false, nil
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, identity.SessionID) {
|
|
return enabled, false, nil
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, identity.SessionID)
|
|
if err != nil {
|
|
return enabled, false, fmt.Errorf("load session for queue Auto-run: %w", err)
|
|
}
|
|
if session == nil || session.TaskID != identity.TaskID {
|
|
return enabled, false, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil {
|
|
if isSessionBusyError(err) {
|
|
return enabled, false, nil
|
|
}
|
|
return enabled, false, err
|
|
}
|
|
dispatched, err := s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity)
|
|
return enabled, dispatched, err
|
|
}
|
|
|
|
func (s *Service) resolveQueueIdentityForSession(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
) (messagequeue.QueueSessionIdentity, error) {
|
|
if s.messageQueue == nil || session == nil || session.ID == "" || session.TaskID == "" {
|
|
return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
identity, err := s.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, session.ID)
|
|
if err != nil {
|
|
return messagequeue.QueueSessionIdentity{}, err
|
|
}
|
|
if session.QueueIncarnationID != "" &&
|
|
session.QueueIncarnationID != identity.SessionIncarnationID {
|
|
return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
return identity, nil
|
|
}
|
|
|
|
func (s *Service) drainQueuedMessageForPromptableSessionForIdentity(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
) (bool, error) {
|
|
lock, release := s.acquireCancelInFlightGuard(identity.SessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
if s.isCancelInFlight(identity.SessionID) {
|
|
return false, nil
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, identity.SessionID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if session == nil ||
|
|
session.TaskID != identity.TaskID ||
|
|
session.QueueIncarnationID != identity.SessionIncarnationID {
|
|
return false, messagequeue.ErrSessionIdentityMismatch
|
|
}
|
|
if err := s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State); err != nil {
|
|
if isSessionBusyError(err) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
if s.sessionHasPendingClarification(ctx, identity.SessionID) {
|
|
return false, nil
|
|
}
|
|
return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity)
|
|
}
|
|
|
|
func (s *Service) drainQueuedMessageForPromptableSessionLockedForIdentity(ctx context.Context, identity messagequeue.QueueSessionIdentity) (bool, error) {
|
|
if s.isCancelInFlight(identity.SessionID) || s.isQueuedDispatchInFlight(identity.SessionID) || s.isSteerInFlight(identity.SessionID) {
|
|
return false, nil
|
|
}
|
|
queuedMsg, ok, autoRun, err := s.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !autoRun {
|
|
return false, nil
|
|
}
|
|
return s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok), nil
|
|
}
|
|
|
|
// cancelInFlightGuard is a per-session mutex serializing cancel/interrupt/
|
|
// queue-take decisions (see the Service.cancelInFlight field doc comment),
|
|
// paired with a reference count so the registry can safely reclaim the
|
|
// entry once every acquirer has released it — refs is guarded by
|
|
// Service.cancelInFlightMu, never by mu itself, since mu can be held for
|
|
// the caller's whole critical section while refs bookkeeping must stay
|
|
// brief and independent of that.
|
|
type cancelInFlightGuard struct {
|
|
mu sync.Mutex
|
|
refs int
|
|
}
|
|
|
|
type taskSessionErrorGuard struct {
|
|
mu sync.Mutex
|
|
refs int
|
|
}
|
|
|
|
func (s *Service) acquireTaskSessionErrorGuard(taskID string) (*sync.Mutex, func()) {
|
|
s.taskSessionErrorLocksMu.Lock()
|
|
if s.taskSessionErrorLocks == nil {
|
|
s.taskSessionErrorLocks = make(map[string]*taskSessionErrorGuard)
|
|
}
|
|
guard, ok := s.taskSessionErrorLocks[taskID]
|
|
if !ok {
|
|
guard = &taskSessionErrorGuard{}
|
|
s.taskSessionErrorLocks[taskID] = guard
|
|
}
|
|
guard.refs++
|
|
s.taskSessionErrorLocksMu.Unlock()
|
|
|
|
var released bool
|
|
release := func() {
|
|
if released {
|
|
return
|
|
}
|
|
released = true
|
|
s.taskSessionErrorLocksMu.Lock()
|
|
guard.refs--
|
|
if guard.refs == 0 {
|
|
delete(s.taskSessionErrorLocks, taskID)
|
|
}
|
|
s.taskSessionErrorLocksMu.Unlock()
|
|
}
|
|
return &guard.mu, release
|
|
}
|
|
|
|
type cancellationKind string
|
|
|
|
const (
|
|
cancellationKindExplicit cancellationKind = "explicit"
|
|
cancellationKindSilent cancellationKind = "silent"
|
|
cancellationKindPeer cancellationKind = "peer"
|
|
cancellationKindInternal cancellationKind = "internal"
|
|
cancellationKindQueueSendNow cancellationKind = "queue_send_now"
|
|
cancellationOperationTTL = 30 * time.Second
|
|
)
|
|
|
|
type cancellationIdentity struct {
|
|
executionID string
|
|
promptGeneration uint64
|
|
activityEpoch uint64
|
|
turnID string
|
|
}
|
|
|
|
type cancelOperation struct {
|
|
done chan struct{}
|
|
joined chan struct{}
|
|
joinObserved bool
|
|
err error
|
|
providerCancelErr error
|
|
providerCancelOutcomeReady bool
|
|
projectionRelease func()
|
|
kind cancellationKind
|
|
identity cancellationIdentity
|
|
identityReady bool
|
|
expectedTurnID string
|
|
expectedTurnReady bool
|
|
completionEligible bool
|
|
completionEligibilityReady bool
|
|
actions []*cancellationAction
|
|
nextAction int
|
|
explicitReconcileOnce sync.Once
|
|
explicitReconcileErr error
|
|
}
|
|
|
|
// cancellationAction is source-specific work that must run after the shared
|
|
// lifecycle cancellation/reconciliation, but before the coordinator releases
|
|
// the operation to other state-mutating paths. The action runs while holding
|
|
// the session guard, so a manual drain or ready event cannot slip between the
|
|
// shared cancel and the source's own queue/visible-message decision.
|
|
type cancellationAction struct {
|
|
done chan struct{}
|
|
run func(context.Context, *cancelOperation) (bool, error)
|
|
dispatched bool
|
|
err error
|
|
}
|
|
|
|
// acquireCancelInFlightGuard returns the shared per-session mutex guarding
|
|
// cancel/interrupt/queue-take decisions for sessionID, creating one if it
|
|
// doesn't exist yet and registering the caller's reference to it. Callers
|
|
// MUST call the returned release func exactly once when done with the
|
|
// mutex — whether or not they actually acquired it (e.g. a TryLock that
|
|
// returned false still needs release to drop its reference). Pairing every
|
|
// acquire with a release is what keeps cancelInFlight bounded by
|
|
// concurrently-active sessions: once refs drops to zero the entry is
|
|
// deleted from the map instead of accumulating one permanent entry per
|
|
// session ever created.
|
|
func (s *Service) acquireCancelInFlightGuard(sessionID string) (*sync.Mutex, func()) {
|
|
s.cancelInFlightMu.Lock()
|
|
if s.cancelInFlight == nil {
|
|
s.cancelInFlight = make(map[string]*cancelInFlightGuard)
|
|
}
|
|
guard, ok := s.cancelInFlight[sessionID]
|
|
if !ok {
|
|
guard = &cancelInFlightGuard{}
|
|
s.cancelInFlight[sessionID] = guard
|
|
}
|
|
guard.refs++
|
|
s.cancelInFlightMu.Unlock()
|
|
|
|
var released bool
|
|
release := func() {
|
|
if released {
|
|
return
|
|
}
|
|
released = true
|
|
s.cancelInFlightMu.Lock()
|
|
guard.refs--
|
|
if guard.refs == 0 {
|
|
delete(s.cancelInFlight, sessionID)
|
|
}
|
|
s.cancelInFlightMu.Unlock()
|
|
}
|
|
return &guard.mu, release
|
|
}
|
|
|
|
// lockedCancelInFlightGuard keeps the yield/reacquire protocol in one place
|
|
// for operations that must release the session guard around lifecycle I/O.
|
|
type lockedCancelInFlightGuard struct {
|
|
mutex *sync.Mutex
|
|
releaseRef func()
|
|
locked bool
|
|
}
|
|
|
|
func (s *Service) lockCancelInFlightGuard(sessionID string) *lockedCancelInFlightGuard {
|
|
mutex, release := s.acquireCancelInFlightGuard(sessionID)
|
|
mutex.Lock()
|
|
return &lockedCancelInFlightGuard{mutex: mutex, releaseRef: release, locked: true}
|
|
}
|
|
|
|
func (g *lockedCancelInFlightGuard) unlock() {
|
|
if g == nil || !g.locked {
|
|
return
|
|
}
|
|
g.mutex.Unlock()
|
|
g.locked = false
|
|
}
|
|
|
|
func (g *lockedCancelInFlightGuard) relock() {
|
|
if g == nil || g.locked {
|
|
return
|
|
}
|
|
g.mutex.Lock()
|
|
g.locked = true
|
|
}
|
|
|
|
func (g *lockedCancelInFlightGuard) release() {
|
|
if g == nil || g.releaseRef == nil {
|
|
return
|
|
}
|
|
g.unlock()
|
|
g.releaseRef()
|
|
g.releaseRef = nil
|
|
}
|
|
|
|
// claimCancellation establishes the single owner for every cancellation source
|
|
// targeting one session. The operation stays registered until its owner has
|
|
// finished lifecycle cancellation and source-specific reconciliation.
|
|
func (s *Service) claimCancellation(sessionID string, kind cancellationKind) (*cancelOperation, bool) {
|
|
operation, owner, _ := s.claimCancellationWithAction(sessionID, kind, nil)
|
|
return operation, owner
|
|
}
|
|
|
|
func (s *Service) claimCancellationWithAction(
|
|
sessionID string,
|
|
kind cancellationKind,
|
|
action func(context.Context, *cancelOperation) (bool, error),
|
|
) (*cancelOperation, bool, *cancellationAction) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if s.cancellationOperations == nil {
|
|
s.cancellationOperations = make(map[string]*cancelOperation)
|
|
}
|
|
if operation, ok := s.cancellationOperations[sessionID]; ok {
|
|
operation.markJoinedLocked()
|
|
if action == nil {
|
|
return operation, false, nil
|
|
}
|
|
registered := &cancellationAction{done: make(chan struct{}), run: action}
|
|
operation.actions = append(operation.actions, registered)
|
|
return operation, false, registered
|
|
}
|
|
operation := &cancelOperation{done: make(chan struct{}), joined: make(chan struct{}), kind: kind}
|
|
var registered *cancellationAction
|
|
if action != nil {
|
|
registered = &cancellationAction{done: make(chan struct{}), run: action}
|
|
operation.actions = append(operation.actions, registered)
|
|
}
|
|
s.cancellationOperations[sessionID] = operation
|
|
return operation, true, registered
|
|
}
|
|
|
|
// claimCancellationWithActionExclusive establishes a cancellation owner only
|
|
// when no source is already active for the session. Send Now uses this stricter
|
|
// variant because joining an explicit cancel (or another Send Now click) would
|
|
// otherwise let the shared cancellation finish with the wrong source
|
|
// semantics. The accepted result is false when a caller must fail closed.
|
|
func (s *Service) claimCancellationWithActionExclusive(
|
|
sessionID string,
|
|
kind cancellationKind,
|
|
action func(context.Context, *cancelOperation) (bool, error),
|
|
) (*cancelOperation, bool, *cancellationAction, bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if s.cancellationOperations == nil {
|
|
s.cancellationOperations = make(map[string]*cancelOperation)
|
|
}
|
|
if operation, ok := s.cancellationOperations[sessionID]; ok {
|
|
return operation, false, nil, false
|
|
}
|
|
operation := &cancelOperation{done: make(chan struct{}), joined: make(chan struct{}), kind: kind}
|
|
var registered *cancellationAction
|
|
if action != nil {
|
|
registered = &cancellationAction{done: make(chan struct{}), run: action}
|
|
operation.actions = append(operation.actions, registered)
|
|
}
|
|
s.cancellationOperations[sessionID] = operation
|
|
return operation, true, registered, true
|
|
}
|
|
|
|
func (s *Service) claimExplicitCancellation(
|
|
sessionID string,
|
|
action func(context.Context, *cancelOperation) (bool, error),
|
|
) (*cancelOperation, bool, *cancellationAction) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if s.cancellationOperations == nil {
|
|
s.cancellationOperations = make(map[string]*cancelOperation)
|
|
}
|
|
if operation, ok := s.cancellationOperations[sessionID]; ok {
|
|
if operation.kind == cancellationKindQueueSendNow {
|
|
return operation, false, nil
|
|
}
|
|
operation.markJoinedLocked()
|
|
if operation.kind == cancellationKindExplicit || action == nil {
|
|
return operation, false, nil
|
|
}
|
|
registered := &cancellationAction{done: make(chan struct{}), run: action}
|
|
operation.actions = append(operation.actions, registered)
|
|
return operation, false, registered
|
|
}
|
|
operation := &cancelOperation{
|
|
done: make(chan struct{}),
|
|
joined: make(chan struct{}),
|
|
kind: cancellationKindExplicit,
|
|
}
|
|
s.cancellationOperations[sessionID] = operation
|
|
return operation, true, nil
|
|
}
|
|
|
|
// markJoinedLocked records that another accepted cancellation source has
|
|
// attached to this operation. cancellationOperationsMu must be held.
|
|
func (operation *cancelOperation) markJoinedLocked() {
|
|
if operation == nil || operation.joinObserved {
|
|
return
|
|
}
|
|
operation.joinObserved = true
|
|
close(operation.joined)
|
|
}
|
|
|
|
func (s *Service) currentCancellation(sessionID string) *cancelOperation {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
return s.cancellationOperations[sessionID]
|
|
}
|
|
|
|
func (s *Service) setCancellationIdentity(sessionID string, operation *cancelOperation, identity cancellationIdentity) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if current := s.cancellationOperations[sessionID]; current == operation {
|
|
operation.identity = identity
|
|
operation.identityReady = true
|
|
}
|
|
}
|
|
|
|
func (s *Service) setCancellationExpectedTurn(sessionID string, operation *cancelOperation, turnID string) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if current := s.cancellationOperations[sessionID]; current == operation {
|
|
operation.expectedTurnID = turnID
|
|
operation.expectedTurnReady = true
|
|
}
|
|
}
|
|
|
|
func (s *Service) cancellationExpectedTurnSnapshot(operation *cancelOperation) (string, bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if operation == nil {
|
|
return "", false
|
|
}
|
|
return operation.expectedTurnID, operation.expectedTurnReady
|
|
}
|
|
|
|
func (s *Service) setCancellationCompletionEligible(sessionID string, operation *cancelOperation, eligible bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if current := s.cancellationOperations[sessionID]; current == operation {
|
|
operation.completionEligible = eligible
|
|
operation.completionEligibilityReady = true
|
|
}
|
|
}
|
|
|
|
func (s *Service) cancellationIdentitySnapshot(operation *cancelOperation) (cancellationIdentity, bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if operation == nil {
|
|
return cancellationIdentity{}, false
|
|
}
|
|
return operation.identity, operation.identityReady
|
|
}
|
|
|
|
func (s *Service) cancellationPreparationSnapshot(operation *cancelOperation) (cancellationIdentity, bool, bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if operation == nil {
|
|
return cancellationIdentity{}, false, false
|
|
}
|
|
return operation.identity, operation.completionEligible, operation.completionEligibilityReady
|
|
}
|
|
|
|
func (s *Service) setCancellationProviderOutcome(sessionID string, operation *cancelOperation, err error) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if current := s.cancellationOperations[sessionID]; current == operation {
|
|
operation.providerCancelErr = err
|
|
operation.providerCancelOutcomeReady = true
|
|
}
|
|
}
|
|
|
|
func (s *Service) cancellationProviderOutcomeSnapshot(operation *cancelOperation) (error, bool) {
|
|
s.cancellationOperationsMu.Lock()
|
|
defer s.cancellationOperationsMu.Unlock()
|
|
if operation == nil {
|
|
return nil, false
|
|
}
|
|
return operation.providerCancelErr, operation.providerCancelOutcomeReady
|
|
}
|
|
|
|
func (s *Service) finishCancellation(sessionID string, operation *cancelOperation, err error) {
|
|
s.finishCancellationWithActions(context.Background(), sessionID, operation, err)
|
|
}
|
|
|
|
// finishCancellationWithActions runs all source-specific actions that joined
|
|
// the operation before removing it from the coordinator. The global
|
|
// coordinator mutex is reacquired between actions; a joiner that claims the
|
|
// operation before the final removal is therefore included, while a caller
|
|
// arriving after removal starts a fresh operation and re-evaluates state.
|
|
func (s *Service) finishCancellationWithActions(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
operation *cancelOperation,
|
|
err error,
|
|
) {
|
|
for {
|
|
s.cancellationOperationsMu.Lock()
|
|
if current := s.cancellationOperations[sessionID]; current != operation {
|
|
s.cancellationOperationsMu.Unlock()
|
|
return
|
|
}
|
|
if operation.nextAction >= len(operation.actions) {
|
|
releaseProjection := operation.projectionRelease
|
|
operation.projectionRelease = nil
|
|
operation.err = err
|
|
delete(s.cancellationOperations, sessionID)
|
|
s.cancellationOperationsMu.Unlock()
|
|
if releaseProjection != nil {
|
|
releaseProjection()
|
|
}
|
|
close(operation.done)
|
|
return
|
|
}
|
|
action := operation.actions[operation.nextAction]
|
|
operation.nextAction++
|
|
s.cancellationOperationsMu.Unlock()
|
|
|
|
actionErr := err
|
|
var dispatched bool
|
|
if actionErr == nil && action.run != nil {
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
lock.Lock()
|
|
dispatched, actionErr = action.run(ctx, operation)
|
|
lock.Unlock()
|
|
release()
|
|
}
|
|
|
|
s.cancellationOperationsMu.Lock()
|
|
action.dispatched = dispatched
|
|
action.err = actionErr
|
|
close(action.done)
|
|
s.cancellationOperationsMu.Unlock()
|
|
}
|
|
}
|
|
|
|
// beginCancelInFlight keeps the legacy marker seam used by coordinator-stop
|
|
// and focused tests, but its marker is now the same ownership record used by
|
|
// explicit, silent, and peer cancellation paths.
|
|
func (s *Service) beginCancelInFlight(sessionID string) func() {
|
|
if sessionID == "" {
|
|
return func() {}
|
|
}
|
|
endProjection := s.beginCancellationProjection(sessionID)
|
|
operation, owner := s.claimCancellation(sessionID, cancellationKindInternal)
|
|
if !owner {
|
|
return endProjection
|
|
}
|
|
operation.projectionRelease = endProjection
|
|
|
|
var once sync.Once
|
|
return func() {
|
|
once.Do(func() {
|
|
s.finishCancellation(sessionID, operation, nil)
|
|
endProjection()
|
|
})
|
|
}
|
|
}
|
|
|
|
// beginCancellationProjection records accepted cancellation work for the
|
|
// backend-owned progress projection. It is deliberately separate from the
|
|
// lifecycle ownership coordinator above: stream persistence can use the
|
|
// per-session guard without changing the user-visible cancellation state.
|
|
func (s *Service) beginCancellationProjection(sessionID string) func() {
|
|
if sessionID == "" {
|
|
return func() {}
|
|
}
|
|
s.cancelOperationsMu.Lock()
|
|
if s.cancelOperations == nil {
|
|
s.cancelOperations = make(map[string]*cancellationOperationState)
|
|
}
|
|
state := s.cancelOperations[sessionID]
|
|
if state == nil {
|
|
state = &cancellationOperationState{}
|
|
s.cancelOperations[sessionID] = state
|
|
}
|
|
state.count++
|
|
startDrain := false
|
|
if state.count == 1 {
|
|
state.revision++
|
|
startDrain = s.enqueueCancellationPendingLocked(sessionID, state, true)
|
|
}
|
|
s.cancelOperationsMu.Unlock()
|
|
if startDrain {
|
|
s.drainCancellationPublicationQueue(sessionID, state)
|
|
}
|
|
|
|
var once sync.Once
|
|
return func() {
|
|
once.Do(func() {
|
|
startDrain := false
|
|
s.cancelOperationsMu.Lock()
|
|
state := s.cancelOperations[sessionID]
|
|
if state != nil && state.count > 0 {
|
|
state.count--
|
|
if state.count == 0 {
|
|
state.revision++
|
|
startDrain = s.enqueueCancellationPendingLocked(sessionID, state, false)
|
|
}
|
|
}
|
|
s.cancelOperationsMu.Unlock()
|
|
if startDrain {
|
|
s.drainCancellationPublicationQueue(sessionID, state)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// CancellationPending reports whether at least one accepted cancellation
|
|
// operation is active for sessionID. The projection is intentionally runtime
|
|
// scoped: a backend restart clears it because no cancellation work survives
|
|
// the process restart.
|
|
func (s *Service) CancellationPending(sessionID string) bool {
|
|
pending, _ := s.CancellationPendingSnapshot(sessionID)
|
|
return pending
|
|
}
|
|
|
|
// CancellationPendingSnapshot returns the pending projection and its
|
|
// process-local transition revision from one critical section. Keeping these
|
|
// values together prevents REST/boot hydration from observing a boolean from
|
|
// one generation with the revision from another.
|
|
func (s *Service) CancellationPendingSnapshot(sessionID string) (bool, uint64) {
|
|
if sessionID == "" {
|
|
return false, 0
|
|
}
|
|
s.cancelOperationsMu.Lock()
|
|
defer s.cancelOperationsMu.Unlock()
|
|
state := s.cancelOperations[sessionID]
|
|
if state == nil {
|
|
return false, 0
|
|
}
|
|
return state.count > 0, state.revision
|
|
}
|
|
|
|
func (s *Service) publishCancellationPending(sessionID string, pending bool, revision uint64) {
|
|
if s.eventBus == nil || sessionID == "" {
|
|
return
|
|
}
|
|
payload := map[string]interface{}{
|
|
"session_id": sessionID,
|
|
"cancellation_pending": pending,
|
|
"cancellation_revision": revision,
|
|
}
|
|
if err := s.eventBus.Publish(context.Background(), events.TaskSessionCancellationChanged,
|
|
bus.NewEvent(events.TaskSessionCancellationChanged, "orchestrator", payload)); err != nil {
|
|
if s.logger != nil {
|
|
s.logger.Warn("failed to publish cancellation state",
|
|
zap.String("session_id", sessionID),
|
|
zap.Bool("cancellation_pending", pending),
|
|
zap.Uint64("cancellation_revision", revision),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
type cancellationOperationState struct {
|
|
count int
|
|
revision uint64
|
|
publications cancellationPublicationQueue
|
|
}
|
|
|
|
type cancellationPublication struct {
|
|
pending bool
|
|
revision uint64
|
|
}
|
|
|
|
type cancellationPublicationQueue struct {
|
|
pending []cancellationPublication
|
|
publishing bool
|
|
}
|
|
|
|
// enqueueCancellationPendingLocked appends a transition while the count and
|
|
// revision are still protected by cancelOperationsMu. The caller must drain
|
|
// the queue after releasing that mutex when this returns true.
|
|
func (s *Service) enqueueCancellationPendingLocked(
|
|
sessionID string,
|
|
state *cancellationOperationState,
|
|
pending bool,
|
|
) bool {
|
|
if s.eventBus == nil || sessionID == "" {
|
|
return false
|
|
}
|
|
queue := &state.publications
|
|
queue.pending = append(queue.pending, cancellationPublication{pending: pending, revision: state.revision})
|
|
if queue.publishing {
|
|
return false
|
|
}
|
|
queue.publishing = true
|
|
return true
|
|
}
|
|
|
|
func (s *Service) drainCancellationPublicationQueue(
|
|
sessionID string,
|
|
state *cancellationOperationState,
|
|
) {
|
|
for {
|
|
s.cancelOperationsMu.Lock()
|
|
queue := &state.publications
|
|
if len(queue.pending) == 0 {
|
|
queue.publishing = false
|
|
s.cancelOperationsMu.Unlock()
|
|
return
|
|
}
|
|
publication := queue.pending[0]
|
|
queue.pending = queue.pending[1:]
|
|
s.cancelOperationsMu.Unlock()
|
|
|
|
s.publishCancellationPending(sessionID, publication.pending, publication.revision)
|
|
}
|
|
}
|
|
|
|
// isCancelInFlight reports whether an actual cancellation operation for
|
|
// sessionID is active or waiting on the shared guard. It deliberately does
|
|
// not inspect the guard mutex itself because stream/lifecycle handlers also
|
|
// use that mutex for non-cancelling side effects.
|
|
func (s *Service) isCancelInFlight(sessionID string) bool {
|
|
if sessionID == "" {
|
|
return false
|
|
}
|
|
return s.currentCancellation(sessionID) != nil
|
|
}
|
|
|
|
// waitForCancelInFlight blocks until all cancellation intents for sessionID
|
|
// have finished. Callers that hold the per-session guard must release it
|
|
// before waiting; the cancellation owner may need that mutex for terminal
|
|
// stream persistence and final reconciliation.
|
|
func (s *Service) waitForCancelInFlight(ctx context.Context, sessionID string) error {
|
|
operation := s.currentCancellation(sessionID)
|
|
if operation == nil {
|
|
return nil
|
|
}
|
|
return operation.wait(ctx)
|
|
}
|
|
|
|
// waitForCancellationWithGuard releases a caller-held per-session mutex while
|
|
// a cancellation owner completes, then reacquires it before returning. The
|
|
// loop closes the small handoff window where a fresh cancellation can claim
|
|
// the session immediately after the first operation finishes.
|
|
func (s *Service) waitForCancellationWithGuard(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
unlock func(),
|
|
relock func(),
|
|
) error {
|
|
for {
|
|
operation := s.currentCancellation(sessionID)
|
|
if operation == nil {
|
|
return nil
|
|
}
|
|
unlock()
|
|
err := operation.wait(ctx)
|
|
relock()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) cancellationOwnsStreamEvent(sessionID, executionID string, promptGeneration uint64) bool {
|
|
operation := s.currentCancellation(sessionID)
|
|
if operation == nil {
|
|
return true
|
|
}
|
|
identity, ready := s.cancellationIdentitySnapshot(operation)
|
|
if !ready {
|
|
// The owner has claimed the operation but has not yet reached the
|
|
// guarded identity snapshot. The stream event is ordered before the
|
|
// yielded lifecycle wait and may proceed to let the owner capture it.
|
|
return true
|
|
}
|
|
if identity.executionID != "" && executionID != "" && identity.executionID != executionID {
|
|
return false
|
|
}
|
|
if identity.promptGeneration != 0 && promptGeneration != 0 && identity.promptGeneration != promptGeneration {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (operation *cancelOperation) wait(ctx context.Context) error {
|
|
select {
|
|
case <-operation.done:
|
|
return operation.err
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (action *cancellationAction) wait(ctx context.Context) (bool, error) {
|
|
if action == nil {
|
|
return false, nil
|
|
}
|
|
select {
|
|
case <-action.done:
|
|
return action.dispatched, action.err
|
|
case <-ctx.Done():
|
|
return false, ctx.Err()
|
|
}
|
|
}
|
|
|
|
// reconcileCancelledTurn durably settles the cancelled session and its turn.
|
|
// A user cancellation may trigger workflow completion only after both pieces
|
|
// of bookkeeping have been confirmed from their authoritative stores. The
|
|
// best-effort lifecycle helpers intentionally remain unchanged for other event
|
|
// paths; this path fails closed so a persistence or turn-service failure cannot
|
|
// advance the task while the cancelled turn is still running.
|
|
func (s *Service) reconcileCancelledTurn(
|
|
ctx context.Context,
|
|
taskID string,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
requireWaiting bool,
|
|
) (*models.TaskSession, error) {
|
|
return s.reconcileCancelledTurnOwned(ctx, taskID, sessionID, session, requireWaiting, "")
|
|
}
|
|
|
|
// reconcileCancelledTurnOwned settles cancellation bookkeeping for the turn
|
|
// captured by the caller. An expected turn ID makes every authoritative check
|
|
// fail closed when a successor turn has appeared, rather than allowing the
|
|
// generic cleanup sweep to close unrelated work.
|
|
func (s *Service) reconcileCancelledTurnOwned(
|
|
ctx context.Context,
|
|
taskID string,
|
|
sessionID string,
|
|
session *models.TaskSession,
|
|
requireWaiting bool,
|
|
expectedTurnID string,
|
|
) (*models.TaskSession, error) {
|
|
if err := s.verifyCapturedCancelledTurn(ctx, sessionID, expectedTurnID); err != nil {
|
|
return nil, err
|
|
}
|
|
authoritativeSession, err := s.reconcileCancelledSessionState(ctx, taskID, session, requireWaiting)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.verifyCapturedCancelledTurn(ctx, sessionID, expectedTurnID); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.completeTurnForTaskSessionCheckedOwned(ctx, taskID, sessionID, expectedTurnID); err != nil {
|
|
return nil, fmt.Errorf("settle cancelled turn: %w", err)
|
|
}
|
|
|
|
if session == nil {
|
|
return nil, nil
|
|
}
|
|
if err := validateCancelledSessionState(session.ID, authoritativeSession, requireWaiting); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.verifyCancelledTurnClosedOwned(ctx, sessionID, expectedTurnID); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return authoritativeSession, nil
|
|
}
|
|
|
|
func (s *Service) verifyCapturedCancelledTurn(ctx context.Context, sessionID, expectedTurnID string) error {
|
|
if s.turnService == nil || expectedTurnID == "" {
|
|
return nil
|
|
}
|
|
activeTurn, err := s.turnService.GetActiveTurn(ctx, sessionID)
|
|
if err != nil {
|
|
if isNoActiveTurnError(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("inspect captured cancelled turn: %w", err)
|
|
}
|
|
if activeTurn != nil && activeTurn.ID != expectedTurnID {
|
|
return fmt.Errorf("captured cancelled turn %s was superseded by active turn %s", expectedTurnID, activeTurn.ID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) reconcileCancelledSessionState(
|
|
ctx context.Context,
|
|
taskID string,
|
|
session *models.TaskSession,
|
|
requireWaiting bool,
|
|
) (*models.TaskSession, error) {
|
|
if session == nil || !requireWaiting {
|
|
return session, nil
|
|
}
|
|
updated := s.updateTaskSessionState(
|
|
ctx,
|
|
taskID,
|
|
session.ID,
|
|
models.TaskSessionStateWaitingForInput,
|
|
"",
|
|
true,
|
|
session,
|
|
)
|
|
if updated == nil {
|
|
return nil, fmt.Errorf("persist cancelled session %s as WAITING_FOR_INPUT", session.ID)
|
|
}
|
|
if updated.State != models.TaskSessionStateWaitingForInput {
|
|
return nil, fmt.Errorf(
|
|
"cancelled session %s persisted as %s, want WAITING_FOR_INPUT",
|
|
session.ID,
|
|
updated.State,
|
|
)
|
|
}
|
|
authoritative, err := s.repo.GetTaskSession(ctx, session.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("verify cancelled session %s state: %w", session.ID, err)
|
|
}
|
|
if authoritative == nil {
|
|
return nil, fmt.Errorf(
|
|
"cancelled session %s is missing before turn settlement, want WAITING_FOR_INPUT",
|
|
session.ID,
|
|
)
|
|
}
|
|
if authoritative.State != models.TaskSessionStateWaitingForInput {
|
|
return nil, fmt.Errorf(
|
|
"cancelled session %s is %s before turn settlement, want WAITING_FOR_INPUT",
|
|
session.ID,
|
|
authoritative.State,
|
|
)
|
|
}
|
|
return authoritative, nil
|
|
}
|
|
|
|
func validateCancelledSessionState(
|
|
sessionID string,
|
|
authoritativeSession *models.TaskSession,
|
|
requireWaiting bool,
|
|
) error {
|
|
if authoritativeSession == nil {
|
|
return fmt.Errorf("cancelled session %s has no authoritative state", sessionID)
|
|
}
|
|
if requireWaiting && authoritativeSession.State != models.TaskSessionStateWaitingForInput {
|
|
return fmt.Errorf(
|
|
"cancelled session %s settled as %s, want WAITING_FOR_INPUT",
|
|
sessionID,
|
|
authoritativeSession.State,
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) verifyCancelledTurnClosed(ctx context.Context, sessionID string) error {
|
|
return s.verifyCancelledTurnClosedOwned(ctx, sessionID, "")
|
|
}
|
|
|
|
func (s *Service) verifyCancelledTurnClosedOwned(ctx context.Context, sessionID, expectedTurnID string) error {
|
|
if s.turnService == nil {
|
|
return nil
|
|
}
|
|
activeTurn, err := s.turnService.GetActiveTurn(ctx, sessionID)
|
|
if err != nil {
|
|
if isNoActiveTurnError(err) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("verify cancelled turn closure: %w", err)
|
|
}
|
|
if activeTurn != nil {
|
|
if expectedTurnID != "" && activeTurn.ID != expectedTurnID {
|
|
return fmt.Errorf("captured cancelled turn %s was superseded by active turn %s", expectedTurnID, activeTurn.ID)
|
|
}
|
|
return fmt.Errorf("cancelled turn %s remains open", activeTurn.ID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CancelAgent interrupts the current agent turn without terminating the process,
|
|
// allowing the user to send a new prompt.
|
|
//
|
|
// Idempotent w.r.t. missing executions: if the agent manager reports no live execution
|
|
// for the session (ErrNoExecutionForSession), the method still reconciles the session's
|
|
// DB state (transitions to WAITING_FOR_INPUT, records the cancel message, completes the
|
|
// turn) so the user can unstick a session whose agent subprocess crashed. Other errors
|
|
// still fail the cancel.
|
|
type cancelAgentPreparation struct {
|
|
session *models.TaskSession
|
|
completionEligible bool
|
|
capturedTurnID string
|
|
cancelTurnID string
|
|
identity cancellationIdentity
|
|
}
|
|
|
|
func (s *Service) CancelAgent(ctx context.Context, sessionID string) (err error) {
|
|
if err := s.authorizeSessionControl(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
if s.repo == nil {
|
|
return errors.New("cancel agent: repository is not configured")
|
|
}
|
|
var operation *cancelOperation
|
|
var owner bool
|
|
var action *cancellationAction
|
|
operation, owner, action = s.claimExplicitCancellation(sessionID, func(actionCtx context.Context, operation *cancelOperation) (bool, error) {
|
|
return false, s.reconcileJoinedExplicitCancellationLocked(actionCtx, sessionID, operation)
|
|
})
|
|
if !owner {
|
|
if err := operation.wait(ctx); err != nil {
|
|
return err
|
|
}
|
|
if operation.kind == cancellationKindExplicit {
|
|
return nil
|
|
}
|
|
if operation.kind == cancellationKindQueueSendNow {
|
|
return ErrSendNowConflict
|
|
}
|
|
if action == nil {
|
|
return s.reconcileJoinedExplicitCancellation(ctx, sessionID, operation)
|
|
}
|
|
_, err := action.wait(ctx)
|
|
return err
|
|
}
|
|
go s.runExplicitCancellation(ctx, sessionID, operation)
|
|
return operation.wait(ctx)
|
|
}
|
|
|
|
func (s *Service) runExplicitCancellation(requestCtx context.Context, sessionID string, operation *cancelOperation) {
|
|
endProjection := s.beginCancellationProjection(sessionID)
|
|
operation.projectionRelease = endProjection
|
|
defer endProjection()
|
|
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(requestCtx), cancellationOperationTTL)
|
|
defer cancel()
|
|
err := s.runExplicitCancellationOwned(operationCtx, sessionID, operation)
|
|
s.finishCancellationWithActions(operationCtx, sessionID, operation, err)
|
|
}
|
|
|
|
func (s *Service) runExplicitCancellationOwned(ctx context.Context, sessionID string, operation *cancelOperation) (err error) {
|
|
s.logger.Debug("cancelling agent turn", zap.String("session_id", sessionID))
|
|
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
guardLocked := true
|
|
unlockGuard := func() {
|
|
if guardLocked {
|
|
lock.Unlock()
|
|
guardLocked = false
|
|
}
|
|
}
|
|
relockGuard := func() {
|
|
if !guardLocked {
|
|
lock.Lock()
|
|
guardLocked = true
|
|
}
|
|
}
|
|
defer unlockGuard()
|
|
|
|
// Invalidate startup before probing or cancelling the runtime. A resume
|
|
// continuation that is already waiting on ACP readiness will observe this
|
|
// identity fence and cannot publish a late token, failure, or prompt.
|
|
s.invalidateResumeAttempt(sessionID)
|
|
prepared, err := s.prepareCancelAgent(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
s.setCancellationIdentity(sessionID, operation, prepared.identity)
|
|
s.setCancellationCompletionEligible(sessionID, operation, prepared.completionEligible)
|
|
if err := s.cancelAgentWhileUnlocked(ctx, sessionID, operation, unlockGuard, relockGuard); err != nil {
|
|
return err
|
|
}
|
|
if err := s.finishCancelledAgentTurn(ctx, sessionID, prepared); err != nil {
|
|
return err
|
|
}
|
|
|
|
s.logger.Debug("agent turn cancelled", zap.String("session_id", sessionID))
|
|
return nil
|
|
}
|
|
|
|
// reconcileJoinedExplicitCancellation applies the user-facing part of an
|
|
// explicit cancel after joining a silent/internal operation. The joined caller
|
|
// never invokes lifecycle cancellation; it only re-evaluates the current
|
|
// session and performs the explicit source's reconciliation once.
|
|
func (s *Service) reconcileJoinedExplicitCancellation(
|
|
requestCtx context.Context,
|
|
sessionID string,
|
|
operation *cancelOperation,
|
|
) error {
|
|
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(requestCtx), cancellationOperationTTL)
|
|
defer cancel()
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
return s.reconcileJoinedExplicitCancellationLocked(operationCtx, sessionID, operation)
|
|
}
|
|
|
|
// reconcileJoinedExplicitCancellationLocked is the source-specific action for
|
|
// a user cancel that joined a silent/internal cancellation. The caller must
|
|
// already hold the session guard; the cancellation coordinator uses this form
|
|
// so the explicit reconciliation is completed before the shared operation is
|
|
// released to other state-mutating paths.
|
|
func (s *Service) reconcileJoinedExplicitCancellationLocked(
|
|
operationCtx context.Context,
|
|
sessionID string,
|
|
operation *cancelOperation,
|
|
) error {
|
|
if _, _, ready := s.cancellationPreparationSnapshot(operation); !ready {
|
|
return errors.New("joined cancellation preparation is incomplete")
|
|
}
|
|
operation.explicitReconcileOnce.Do(func() {
|
|
identity, completionEligible, ready := s.cancellationPreparationSnapshot(operation)
|
|
if !ready {
|
|
operation.explicitReconcileErr = errors.New("joined cancellation preparation became incomplete")
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(operationCtx, sessionID)
|
|
if err != nil {
|
|
operation.explicitReconcileErr = fmt.Errorf("load session after joined cancel: %w", err)
|
|
return
|
|
}
|
|
if session == nil || isTerminalSessionState(session.State) {
|
|
return
|
|
}
|
|
prepared := cancelAgentPreparation{
|
|
session: session,
|
|
completionEligible: completionEligible,
|
|
capturedTurnID: identity.turnID,
|
|
cancelTurnID: identity.turnID,
|
|
identity: identity,
|
|
}
|
|
operation.explicitReconcileErr = s.finishCancelledAgentTurn(operationCtx, sessionID, prepared)
|
|
})
|
|
return operation.explicitReconcileErr
|
|
}
|
|
|
|
func (s *Service) prepareCancelAgent(ctx context.Context, sessionID string) (cancelAgentPreparation, error) {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
s.logger.Warn("failed to get session for cancel",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
completionEligible, err := s.cancelTurnCompletionEligible(ctx, session, sessionID)
|
|
if err != nil {
|
|
return cancelAgentPreparation{}, err
|
|
}
|
|
|
|
identity, err := s.captureCancellationIdentity(ctx, sessionID)
|
|
if err != nil {
|
|
return cancelAgentPreparation{}, err
|
|
}
|
|
capturedTurnID := identity.turnID
|
|
cancelTurnID := capturedTurnID
|
|
if cancelTurnID == "" {
|
|
cancelTurnID = s.getActiveTurnID(sessionID)
|
|
}
|
|
return cancelAgentPreparation{
|
|
session: session,
|
|
completionEligible: completionEligible,
|
|
capturedTurnID: capturedTurnID,
|
|
cancelTurnID: cancelTurnID,
|
|
identity: identity,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Service) captureCancellationIdentity(ctx context.Context, sessionID string) (cancellationIdentity, error) {
|
|
executionID, promptGeneration := s.captureCancellationAgentIdentity(ctx, sessionID)
|
|
identity := cancellationIdentity{executionID: executionID, promptGeneration: promptGeneration}
|
|
if s.turnService != nil {
|
|
turnID, err := s.peekActiveTurnID(ctx, sessionID)
|
|
if err != nil {
|
|
return cancellationIdentity{}, fmt.Errorf("inspect active turn before cancel: %w", err)
|
|
}
|
|
identity.turnID = turnID
|
|
}
|
|
return identity, nil
|
|
}
|
|
|
|
func (s *Service) captureCancellationAgentIdentity(ctx context.Context, sessionID string) (string, uint64) {
|
|
if s.agentManager == nil {
|
|
return "", 0
|
|
}
|
|
executionID, err := s.agentManager.GetExecutionIDForSession(ctx, sessionID)
|
|
if err != nil && !errors.Is(err, lifecycle.ErrNoExecutionForSession) {
|
|
s.logger.Debug("could not capture execution identity before cancellation",
|
|
zap.String("session_id", sessionID), zap.Error(err))
|
|
}
|
|
|
|
generationReader, ok := s.agentManager.(interface {
|
|
GetPromptGenerationForSession(context.Context, string) (uint64, error)
|
|
})
|
|
if !ok {
|
|
return executionID, 0
|
|
}
|
|
generation, generationErr := generationReader.GetPromptGenerationForSession(ctx, sessionID)
|
|
if generationErr != nil && !errors.Is(generationErr, lifecycle.ErrNoExecutionForSession) {
|
|
s.logger.Debug("could not capture prompt generation before cancellation",
|
|
zap.String("session_id", sessionID), zap.Error(generationErr))
|
|
}
|
|
return executionID, generation
|
|
}
|
|
|
|
func (s *Service) cancelTurnCompletionEligible(ctx context.Context, session *models.TaskSession, sessionID string) (bool, error) {
|
|
if session == nil {
|
|
return false, nil
|
|
}
|
|
switch session.State {
|
|
case models.TaskSessionStateRunning, models.TaskSessionStateStarting:
|
|
return true, nil
|
|
case models.TaskSessionStateWaitingForInput:
|
|
activeTurnID, err := s.peekActiveTurnID(ctx, sessionID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("inspect active turn before cancel retry: %w", err)
|
|
}
|
|
return activeTurnID != "", nil
|
|
default:
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
func (s *Service) cancelAgentWhileUnlocked(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
operation *cancelOperation,
|
|
unlockGuard, relockGuard func(),
|
|
) error {
|
|
if s.agentManager == nil {
|
|
return nil
|
|
}
|
|
// The lifecycle manager waits for the in-flight prompt to consume terminal
|
|
// stream frames delivered through the same per-session guard. Keep the
|
|
// operation marker and guard registry reference active, but release only the
|
|
// mutex around that blocking wait.
|
|
unlockGuard()
|
|
cancelErr := s.agentManager.CancelAgent(ctx, sessionID)
|
|
relockGuard()
|
|
s.setCancellationProviderOutcome(sessionID, operation, cancelErr)
|
|
if cancelErr == nil {
|
|
return nil
|
|
}
|
|
switch {
|
|
case errors.Is(cancelErr, lifecycle.ErrNoExecutionForSession):
|
|
// A crashed or unregistered process still needs DB reconciliation so the
|
|
// session does not remain stuck.
|
|
s.logger.Error("agent process appears to have crashed: no live execution for session on cancel",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(cancelErr))
|
|
case errors.Is(cancelErr, lifecycle.ErrCancelEscalated):
|
|
// The lifecycle manager already unblocked the prompt and marked the
|
|
// execution ready; reconcile the session state below.
|
|
s.logger.Warn("agent did not acknowledge cancel; reconciling session state",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(cancelErr))
|
|
default:
|
|
return fmt.Errorf("cancel agent: %w", cancelErr)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) finishCancelledAgentTurn(ctx context.Context, sessionID string, prepared cancelAgentPreparation) error {
|
|
session := prepared.session
|
|
if session != nil {
|
|
reconciled, err := s.reconcileCancelledTurnOwned(
|
|
ctx,
|
|
session.TaskID,
|
|
sessionID,
|
|
session,
|
|
prepared.completionEligible,
|
|
prepared.capturedTurnID,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("reconcile cancelled turn: %w", err)
|
|
}
|
|
if reconciled != nil {
|
|
session = reconciled
|
|
}
|
|
} else if _, err := s.reconcileCancelledTurnOwned(ctx, "", sessionID, nil, false, prepared.capturedTurnID); err != nil {
|
|
return fmt.Errorf("reconcile cancelled turn: %w", err)
|
|
}
|
|
if s.messageQueue != nil {
|
|
var (
|
|
paused bool
|
|
err error
|
|
)
|
|
if session != nil {
|
|
var identity messagequeue.QueueSessionIdentity
|
|
identity, err = s.messageQueue.ResolveSessionIdentity(ctx, session.TaskID, sessionID)
|
|
if err == nil {
|
|
paused, err = s.messageQueue.PauseAutoRunIfPendingForSession(ctx, identity)
|
|
}
|
|
} else {
|
|
paused, err = s.messageQueue.PauseAutoRunIfPending(ctx, sessionID)
|
|
}
|
|
if err != nil {
|
|
s.logger.Warn("failed to pause queued messages after cancel",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
} else if paused {
|
|
s.publishQueueStatusEvent(ctx, sessionID)
|
|
}
|
|
}
|
|
|
|
s.recordCancelledAgentMessage(ctx, session, sessionID, prepared.cancelTurnID)
|
|
s.reconcileCancelledAgentWorkflow(ctx, session, prepared.completionEligible)
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) recordCancelledAgentMessage(ctx context.Context, session *models.TaskSession, sessionID, cancelTurnID string) {
|
|
if s.messageCreator == nil || session == nil {
|
|
return
|
|
}
|
|
metadata := map[string]interface{}{
|
|
"cancelled": true,
|
|
"variant": "warning",
|
|
}
|
|
if err := s.messageCreator.CreateSessionMessage(
|
|
ctx,
|
|
session.TaskID,
|
|
"Turn cancelled by user",
|
|
sessionID,
|
|
string(v1.MessageTypeStatus),
|
|
cancelTurnID,
|
|
metadata,
|
|
false,
|
|
); err != nil {
|
|
s.logger.Warn("failed to create cancel message",
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
|
|
func (s *Service) reconcileCancelledAgentWorkflow(ctx context.Context, session *models.TaskSession, completionEligible bool) {
|
|
if session == nil {
|
|
return
|
|
}
|
|
transitioned := false
|
|
if completionEligible {
|
|
transitioned = s.processOnTurnCompleteViaEngineWithCause(
|
|
ctx, session.TaskID, session, turnCompletionCauseUserCancellation,
|
|
)
|
|
}
|
|
s.reconcileCancelledTaskReview(ctx, session.TaskID, session.ID, transitioned)
|
|
}
|
|
|
|
func (s *Service) reconcileCancelledTaskReview(ctx context.Context, taskID, sessionID string, transitioned bool) {
|
|
if transitioned {
|
|
task, err := s.repo.GetTask(ctx, taskID)
|
|
if err == nil && task != nil && task.WorkflowStepID != "" && s.workflowStepGetter != nil {
|
|
step, stepErr := s.workflowStepGetter.GetStep(ctx, task.WorkflowStepID)
|
|
if stepErr == nil && step != nil && s.workflowStepIsTerminal(ctx, step.ID) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
s.writeTaskReviewState(ctx, taskID, sessionID)
|
|
}
|
|
|
|
// takeAndDispatchEntryLocked takes entryID from sessionID's queue and
|
|
// dispatches it, falling back to draining the FIFO head only if the
|
|
// targeted take doesn't find it (a benign not-found, e.g. already taken by
|
|
// a concurrent path). A genuine repository error on the targeted take is
|
|
// propagated instead of falling back — see cancelAndTakeForPeerMessage's
|
|
// doc comment for why. The caller must already hold sessionID's
|
|
// cancelInFlight lock; this neither acquires nor releases it.
|
|
//
|
|
// Deliberately does NOT check isQueuedDispatchInFlight itself: callers
|
|
// that reach here have either just performed a cancel
|
|
// (cancelAndTakeForPeerMessage) — which supersedes whatever another
|
|
// dispatch was in the middle of settling, so a stale in-flight marker from
|
|
// that dispatch must not block taking a *different* entry here — or have
|
|
// already confirmed the session is genuinely idle (takeIfPromptableLocked,
|
|
// which checks the marker itself before delegating here). See
|
|
// drainQueuedMessageForPromptableSessionLocked for the *non-cancelling*
|
|
// FIFO-head drain path, which does check the marker directly.
|
|
func (s *Service) takeAndDispatchEntryLocked(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
entryID string,
|
|
) (bool, error) {
|
|
if s.messageQueue != nil {
|
|
queuedMsg, ok, err := s.messageQueue.TakeQueuedEntryForSession(ctx, identity, entryID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("take targeted queued message: %w", err)
|
|
}
|
|
if s.dispatchTakenQueuedMessageForSession(ctx, identity, queuedMsg, ok) {
|
|
return true, nil
|
|
}
|
|
}
|
|
return s.drainQueuedMessageForPromptableSessionLockedForIdentity(ctx, identity)
|
|
}
|
|
|
|
// takeIfPromptableLocked takes and dispatches entryID only if sessionID is
|
|
// currently promptable, without ever issuing a cancel — used by
|
|
// QueueAndInterruptForPeerMessage when cancelling would risk hitting a
|
|
// *different*, unrelated turn instead of the one the caller meant to
|
|
// interrupt (see its active-turn revalidation doc comment). Returns
|
|
// (false, nil), never an error, when the session turns out not to be
|
|
// promptable — or when a *different* dispatch is already settling for
|
|
// this session (isQueuedDispatchInFlight; see the Service.dispatchingQueued
|
|
// field doc comment) — mirroring cancelAndTakeForPeerMessage's own "cancel
|
|
// failed but already promptable" recovery except without ever attempting
|
|
// the cancel. Unlike cancelAndTakeForPeerMessage's path, this never
|
|
// cancels anything, so it has no way to supersede a settling dispatch the
|
|
// way a genuine cancel would — it must defer to it instead. The caller
|
|
// must already hold sessionID's cancelInFlight lock.
|
|
//
|
|
// Also defers to an admitted-but-not-yet-dispatched steer (isSteerInFlight;
|
|
// see steerInFlight's field doc comment) for the same reason: this path
|
|
// never cancels, so it has no way to supersede the steer's dispatch either.
|
|
func (s *Service) takeIfPromptableLocked(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
entryID string,
|
|
) (bool, error) {
|
|
if s.isQueuedDispatchInFlight(identity.SessionID) || s.isSteerInFlight(identity.SessionID) {
|
|
return false, nil
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, identity.SessionID)
|
|
if err != nil || session == nil ||
|
|
session.QueueIncarnationID != identity.SessionIncarnationID ||
|
|
s.checkSessionPromptable(identity.TaskID, identity.SessionID, session.State) != nil {
|
|
return false, nil
|
|
}
|
|
return s.takeAndDispatchEntryLocked(ctx, identity, entryID)
|
|
}
|
|
|
|
// cancelAndTakeForPeerMessage cancels sessionID's in-flight turn and takes
|
|
// and dispatches entryID — the specific message that triggered this
|
|
// interrupt — falling back to draining the FIFO head only if the targeted
|
|
// take doesn't find it (a benign not-found, e.g. already taken by a
|
|
// concurrent path). entryID must be non-empty: the only caller,
|
|
// QueueAndInterruptForPeerMessage, always passes the entry it just
|
|
// inserted. A genuine repository error on the targeted take is propagated
|
|
// instead of falling back: an error says nothing about which entry is
|
|
// actually at the FIFO head, and dispatching it anyway would risk
|
|
// delivering the wrong message while the caller still reports "sent" for
|
|
// the parent's.
|
|
//
|
|
// The caller must already hold sessionID's cancelInFlight lock; this
|
|
// neither acquires nor releases it — see QueueAndInterruptForPeerMessage.
|
|
//
|
|
// The returned bool reports whether this call actually dispatched a
|
|
// message; a false result means the caller's message is still only
|
|
// queued, to be delivered later by whichever drain gets to it.
|
|
//
|
|
// Deliberately mirrors cancelAgentSilent + dispatchTakenQueuedMessage rather
|
|
// than delegating to CancelAgent: the interrupt is an internal steering
|
|
// signal from the parent, not a user action, so unlike the cancel button it
|
|
// must not write a visible "Turn cancelled" message and must not move the
|
|
// task to REVIEW (writeTaskReviewStateOnCancel).
|
|
func (s *Service) cancelAndTakeForPeerMessage(
|
|
ctx context.Context,
|
|
identity messagequeue.QueueSessionIdentity,
|
|
entryID string,
|
|
unlockGuard func(),
|
|
relockGuard func(),
|
|
) (bool, error) {
|
|
return s.cancelAgentSilentWithGuardActionKind(
|
|
ctx,
|
|
identity.TaskID,
|
|
identity.SessionID,
|
|
unlockGuard,
|
|
relockGuard,
|
|
func(actionCtx context.Context) (bool, error) {
|
|
return s.takeAndDispatchEntryLocked(actionCtx, identity, entryID)
|
|
},
|
|
cancellationKindPeer,
|
|
)
|
|
}
|
|
|
|
// QueueAndInterruptForPeerMessage accepts the caller-captured session identity,
|
|
// validates it after acquiring the session guard, atomically queues prompt, and
|
|
// then interrupts the child's in-flight turn to deliver it immediately,
|
|
// instead of waiting for the turn to end naturally. Capturing the identity
|
|
// before contending for the guard prevents a delete-and-recreate race from
|
|
// retargeting an old peer operation into a replacement session incarnation.
|
|
// It is used only by
|
|
// handleMessageTask (mcp/handlers) when the sender is the target task's
|
|
// parent: a long-running child otherwise leaves its parent's control/steer
|
|
// messages parked on the FIFO queue until the child's current turn finishes
|
|
// on its own, and with several such children running in parallel the
|
|
// per-session queue can fill up to messagequeue.DefaultMaxPerSession before
|
|
// any of them are delivered.
|
|
//
|
|
// This must be one atomic operation rather than "queue, then separately
|
|
// interrupt" (the original two-step split across the handlers/orchestrator
|
|
// boundary): between an insert becoming visible and a later, separate
|
|
// interrupt call acquiring the session's cancelInFlight lock, the child's
|
|
// turn can complete naturally and handleAgentReady's normal FIFO drain can
|
|
// grab the just-queued entry and start dispatching it as an ordinary turn —
|
|
// only for the later interrupt's cancel to land on and kill that very turn,
|
|
// orphaning the parent's message mid-delivery. Taking the lock before the
|
|
// queue insert closes that: handleAgentReady and handleAgentBootReady's
|
|
// take decisions claim the exact same lock before their own take, so neither
|
|
// can ever observe — and steal — an
|
|
// entry this call is still in the process of queueing-and-claiming.
|
|
//
|
|
// The lock is a genuine, blocking claim (Lock, mirroring
|
|
// executor.Executor.getSessionLock's precedent for per-session mutual
|
|
// exclusion in this codebase) — not a try-once peek with a fallback
|
|
// unguarded insert. Working around a busy lock with an unguarded insert
|
|
// would leave that insert visible to nobody in particular: the current
|
|
// holder may have already finished its own take-or-skip decision before
|
|
// the insert lands, and nothing then guarantees a future drain (the
|
|
// session can go idle with no further agent.ready to retry on). Every
|
|
// existing holder already bounds its own hold time through an independent
|
|
// mechanism (cancelAgentSilent's underlying agent-cancel escalation
|
|
// timeout, or a single fast DB round-trip in the natural-drain paths), so
|
|
// blocking here adds no new deadlock risk.
|
|
//
|
|
// Active-turn revalidation: after acquiring the lock and queueing the
|
|
// message, this re-checks whether the turn active for sessionID is still
|
|
// the one that was active when the caller decided to interrupt (snapshotted
|
|
// via peekActiveTurnID *before* contending for the lock). If a *different*
|
|
// turn is now active — e.g. a workflow on_turn_complete transition
|
|
// auto-started a successor for this same session while this call waited
|
|
// for the guard — cancelling now would kill that unrelated successor turn
|
|
// instead of the one the parent meant to interrupt. In that case this
|
|
// falls back to takeIfPromptableLocked (dispatch directly only if the
|
|
// session happens to already be idle *and* no other dispatch is
|
|
// concurrently settling for it, per isQueuedDispatchInFlight — see the
|
|
// Service.dispatchingQueued field doc comment; otherwise leave the
|
|
// message queued for whichever turn is actually running to drain
|
|
// naturally) instead of ever calling cancelAndTakeForPeerMessage. The
|
|
// turn-identity comparison only trusts a positively-confirmed match — both
|
|
// snapshots read without error, both non-empty, and equal — before it
|
|
// will risk a cancel; any error or mismatch (including "no turn observed
|
|
// at all") takes the safe fallback. The one exception is turnService
|
|
// being unset entirely: with no turn identity obtainable at all, this
|
|
// preserves the exact unconditional-interrupt behavior every existing
|
|
// (turn-unaware) caller and test exercises.
|
|
//
|
|
// Note this deliberately does *not* also gate on isQueuedDispatchInFlight
|
|
// when the turn *is* confirmed unchanged: a cancel (via
|
|
// cancelAndTakeForPeerMessage below) supersedes whatever another dispatch
|
|
// for this same session was in the middle of settling — e.g. a second
|
|
// parent message arriving right behind a first one whose own dispatch
|
|
// hasn't yet reached PromptTask — so it must still be allowed to cancel
|
|
// and take its own entry rather than being blocked by a marker the cancel
|
|
// itself is about to invalidate.
|
|
func (s *Service) QueueAndInterruptForPeerMessage(ctx context.Context, identity messagequeue.QueueSessionIdentity, prompt string, metadata map[string]interface{}) (*messagequeue.QueuedMessage, bool, error) {
|
|
taskID := identity.TaskID
|
|
sessionID := identity.SessionID
|
|
if s.messageQueue == nil {
|
|
return nil, false, errors.New("message queue not available")
|
|
}
|
|
|
|
turnBeforeWait, turnPeekErr := s.peekActiveTurnID(ctx, sessionID)
|
|
if turnPeekErr != nil {
|
|
s.logger.Warn("failed to snapshot active turn before peer-message interrupt; will fall back to a promptable-only dispatch instead of risking a cancel of unrelated work",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.Error(turnPeekErr))
|
|
}
|
|
|
|
lock, release := s.acquireCancelInFlightGuard(sessionID)
|
|
lock.Lock()
|
|
guardLocked := true
|
|
unlockGuard := func() {
|
|
if guardLocked {
|
|
lock.Unlock()
|
|
guardLocked = false
|
|
}
|
|
}
|
|
relockGuard := func() {
|
|
if !guardLocked {
|
|
lock.Lock()
|
|
guardLocked = true
|
|
}
|
|
}
|
|
defer func() {
|
|
unlockGuard()
|
|
release()
|
|
}()
|
|
|
|
queued, err := s.messageQueue.QueueMessageWithMetadataForSession(
|
|
ctx, identity, prompt, "", messagequeue.QueuedByAgent, false, nil, metadata,
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
s.publishQueueStatusEventForIdentity(ctx, identity)
|
|
if operation := s.currentCancellation(sessionID); operation != nil && operation.kind != cancellationKindPeer {
|
|
s.logger.Debug("leaving peer message queued while a different cancellation source is in progress",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("queue_id", queued.ID))
|
|
unlockGuard()
|
|
if waitErr := s.waitForCancelInFlight(ctx, sessionID); waitErr != nil {
|
|
return queued, false, waitErr
|
|
}
|
|
return queued, false, nil
|
|
}
|
|
|
|
if s.turnService != nil {
|
|
turnNow, turnNowErr := s.peekActiveTurnID(ctx, sessionID)
|
|
confirmedUnchanged := turnPeekErr == nil && turnNowErr == nil && turnBeforeWait != "" && turnNow == turnBeforeWait
|
|
if !confirmedUnchanged {
|
|
s.logger.Warn("active turn could not be confirmed unchanged while waiting to interrupt; dispatching only if already idle instead of risking a cancel of unrelated work",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("turn_before_wait", turnBeforeWait),
|
|
zap.Bool("turn_peek_error", turnPeekErr != nil || turnNowErr != nil))
|
|
dispatched, fallbackErr := s.takeIfPromptableLocked(ctx, identity, queued.ID)
|
|
return queued, dispatched, fallbackErr
|
|
}
|
|
}
|
|
|
|
dispatched, err := s.cancelAndTakeForPeerMessage(ctx, identity, queued.ID, unlockGuard, relockGuard)
|
|
return queued, dispatched, err
|
|
}
|
|
|
|
// CompleteTask explicitly completes a task and stops all its agents
|
|
func (s *Service) CompleteTask(ctx context.Context, taskID string) error {
|
|
s.logger.Info("completing task",
|
|
zap.String("task_id", taskID))
|
|
|
|
// Stop all agents for this task (which will trigger AgentCompleted events and update session states)
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForTask(taskID)
|
|
}
|
|
if err := s.executor.StopByTaskID(ctx, taskID, "task completed by user", false); err != nil {
|
|
// If agents are already stopped, just update the task state directly
|
|
s.logger.Warn("failed to stop agents, updating task state directly",
|
|
zap.String("task_id", taskID),
|
|
zap.Error(err))
|
|
}
|
|
|
|
// Update task state to COMPLETED
|
|
if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateCompleted); err != nil {
|
|
return fmt.Errorf("failed to update task state: %w", err)
|
|
}
|
|
s.processParentChildrenCompletedForTaskState(ctx, taskID, v1.TaskStateCompleted)
|
|
|
|
s.logger.Info("task marked as COMPLETED",
|
|
zap.String("task_id", taskID))
|
|
return nil
|
|
}
|
|
|
|
// contextResetMessage is the synthetic status message the chat renders as a
|
|
// "Context reset" divider. Both the manual reset path and the workflow-driven
|
|
// on-enter reset emit this exact text so the two paths cannot drift.
|
|
const contextResetMessage = "Context reset — new conversation started"
|
|
|
|
// createContextResetMessage inserts the synthetic contextResetMessage status
|
|
// row for a session. Callers must only invoke it when an actual reset occurred
|
|
// (e.g. not for a never-started CREATED session, which has no prior
|
|
// conversation to clear).
|
|
func (s *Service) createContextResetMessage(ctx context.Context, taskID, sessionID string) {
|
|
if s.messageCreator == nil {
|
|
return
|
|
}
|
|
if err := s.messageCreator.CreateSessionMessage(
|
|
ctx, taskID,
|
|
contextResetMessage,
|
|
sessionID, string(v1.MessageTypeStatus),
|
|
s.getActiveTurnID(sessionID),
|
|
nil, false,
|
|
); err != nil {
|
|
s.logger.Warn("failed to create context reset message",
|
|
zap.String("session_id", sessionID), zap.Error(err))
|
|
}
|
|
}
|
|
|
|
// ResetAgentContext resets the agent's conversation context for a session,
|
|
// clearing conversation history while preserving the workspace environment.
|
|
func (s *Service) ResetAgentContext(ctx context.Context, sessionID string) error {
|
|
if err := s.authorizeSession(ctx, sessionID); err != nil {
|
|
return err
|
|
}
|
|
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("session not found: %w", err)
|
|
}
|
|
if session.State != models.TaskSessionStateWaitingForInput {
|
|
return fmt.Errorf("agent must be idle to reset context, current state: %s", session.State)
|
|
}
|
|
if hasRunning, hasErr := s.repo.HasExecutorRunningRow(ctx, sessionID); hasErr != nil || !hasRunning {
|
|
return fmt.Errorf("no active agent execution for session %s", sessionID)
|
|
}
|
|
|
|
// Set STARTING so frontend disables input and shows restarting state
|
|
s.updateTaskSessionState(ctx, session.TaskID, sessionID, models.TaskSessionStateStarting, "", false, session)
|
|
|
|
if ok := s.resetAgentContext(ctx, session.TaskID, session, "user_request"); !ok {
|
|
// Restore WAITING_FOR_INPUT on failure
|
|
s.setSessionWaitingForInput(ctx, session.TaskID, sessionID)
|
|
return fmt.Errorf("failed to reset agent context for session %s", sessionID)
|
|
}
|
|
|
|
// Restore WAITING_FOR_INPUT — handleAgentReady ignores events during reset
|
|
s.setSessionWaitingForInput(ctx, session.TaskID, sessionID)
|
|
|
|
s.createContextResetMessage(ctx, session.TaskID, sessionID)
|
|
return nil
|
|
}
|
|
|
|
// GetQueuedTasks returns tasks in the queue
|
|
func (s *Service) GetQueuedTasks() []*queue.QueuedTask {
|
|
return s.queue.List()
|
|
}
|