Files
kandev/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go
T
2026-09-15 23:11:43 +01:00

2747 lines
106 KiB
Go

package lifecycle
import (
"context"
"errors"
"fmt"
"maps"
"sort"
"strings"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/agent/agents"
"github.com/kandev/kandev/internal/agent/executor"
"github.com/kandev/kandev/internal/agent/runtime/activity"
agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
agentctltypes "github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
const (
freshSessionModelStateWait = 2 * time.Second
freshSessionModelStatePoll = 10 * time.Millisecond
resetAgentContextRequestTimeout = 10 * time.Second
)
// WasSessionInitialized reports whether the execution completed ACP session setup.
// Returns false if the execution is not found or init hasn't completed.
func (m *Manager) WasSessionInitialized(executionID string) bool {
exec, exists := m.executionStore.Get(executionID)
if !exists {
return false
}
return exec.isSessionInitialized()
}
// GetSessionAuthMethods returns auth methods for a session's execution.
// It first checks cached methods from agent_capabilities events. When the agent
// failed before reporting capabilities (e.g., immediate auth error), it falls
// back to static auth methods derived from the agent type.
func (m *Manager) GetSessionAuthMethods(sessionID string) []streams.AuthMethodInfo {
exec, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil
}
if methods := exec.GetAuthMethods(); len(methods) > 0 {
return methods
}
return fallbackAuthMethods(exec.AgentID)
}
// fallbackAuthMethods returns static auth methods for known agent types.
// Used when the agent failed before sending agent_capabilities (e.g., auth error
// on first prompt before the agent could report its own auth methods).
func fallbackAuthMethods(agentID string) []streams.AuthMethodInfo {
switch agentID {
case "claude-acp":
return []streams.AuthMethodInfo{
{
ID: "claude-auth-login",
Name: "Anthropic Authentication",
Description: "Log in to your Anthropic account",
TerminalAuth: &streams.TerminalAuth{
Command: "claude",
Args: []string{"auth", "login"},
Label: "Log in with Claude CLI",
},
},
}
case "auggie":
return []streams.AuthMethodInfo{
{
ID: "auggie-login",
Name: "Auggie Authentication",
Description: "Log in to your Auggie account",
TerminalAuth: &streams.TerminalAuth{
Command: "auggie",
Args: []string{"login"},
Label: "Log in with Auggie CLI",
},
},
}
default:
return nil
}
}
// PromptAgent sends a follow-up prompt to a running agent.
// Attachments (images) are passed to the agent if provided.
// When dispatchOnly is true, returns once the prompt is accepted instead of
// waiting for the agent's turn to complete.
func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment, dispatchOnly bool) (*PromptResult, error) {
return m.PromptAgentWithDispatchCallback(ctx, executionID, prompt, attachments, dispatchOnly, nil)
}
// PromptAgentWithDispatchCallback exposes agentctl acceptance to callers that
// must keep admission serialized until the queued prompt is actually dispatched.
func (m *Manager) PromptAgentWithDispatchCallback(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment, dispatchOnly bool, onDispatched func()) (*PromptResult, error) {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return nil, fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
lease, err := m.acquireActivity(ctx, activity.KindExecutionRunning)
if err != nil {
return nil, err
}
key := executionActivityKey(executionID)
m.trackActivity(key, lease)
m.setRuntimeInterest(execution.SessionID, true)
operationRelease, err := execution.acquireContextResetOperation(ctx)
if err != nil {
m.releaseActivity(key)
m.setRuntimeInterest(execution.SessionID, false)
return nil, err
}
defer operationRelease()
result, err := m.sessionManager.SendPromptWithDispatchCallback(ctx, execution, prompt, true, attachments, dispatchOnly, onDispatched)
if err != nil || !dispatchOnly {
m.releaseActivity(key)
if err != nil {
m.setRuntimeInterest(execution.SessionID, false)
}
}
return result, err
}
// SteerAgentWithDispatchCallback delivers a steer: it hands the prompt into a
// turn that is still generating rather than serializing behind it. It mirrors
// PromptAgentWithDispatchCallback's activity accounting; only the underlying
// session-manager call differs.
func (m *Manager) SteerAgentWithDispatchCallback(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment, dispatchOnly bool, onDispatched func()) (*PromptResult, error) {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return nil, fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
lease, err := m.acquireActivity(ctx, activity.KindExecutionRunning)
if err != nil {
return nil, err
}
key := executionActivityKey(executionID)
m.trackActivity(key, lease)
m.setRuntimeInterest(execution.SessionID, true)
operationRelease, err := execution.acquireContextResetOperation(ctx)
if err != nil {
m.releaseActivity(key)
m.setRuntimeInterest(execution.SessionID, false)
return nil, err
}
defer operationRelease()
result, err := m.sessionManager.SendPromptSteerWithDispatchCallback(ctx, execution, prompt, true, attachments, dispatchOnly, onDispatched)
if err != nil || !dispatchOnly {
m.releaseActivity(key)
if err != nil {
m.setRuntimeInterest(execution.SessionID, false)
}
}
return result, err
}
// cancelWaitTimeout bounds how long CancelAgent waits for the in-flight SendPrompt
// to exit after the in-flight session/prompt RPC has ended (cancel acknowledged).
// Exposed as a var (not const) so tests can shorten it without fake clocks.
var cancelWaitTimeout = 10 * time.Second
// cancelEscalationTimeout bounds the post-escalation wait after we inject a synthetic
// error onto promptDoneCh to unblock a stuck SendPrompt.
var cancelEscalationTimeout = 2 * time.Second
// CancelAgent interrupts the current agent turn without terminating the process,
// allowing the user to send a new prompt.
//
// When the agent subprocess accepts the ACP cancel but never publishes a `complete`
// event (and the update stream stays open), the in-flight SendPrompt would otherwise
// block forever. After cancelWaitTimeout we escalate by signalling promptDoneCh with
// a synthetic error so SendPrompt returns and its deferred cleanup closes
// promptFinished normally. Callers receive ErrCancelEscalated to signal that local
// lifecycle state was reconciled but the agent never confirmed the cancel, and
// higher layers (Service.CancelAgent) must still reconcile DB state.
func (m *Manager) CancelAgent(ctx context.Context, executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
return m.cancelAgentExecution(ctx, execution)
}
// CancelAgentForPrompt cancels the execution that owns a previously captured
// prompt activity snapshot. It validates the complete identity while holding
// the execution-store lock, then operates on that exact execution pointer so
// a later session lookup cannot redirect cancellation to a successor.
func (m *Manager) CancelAgentForPrompt(
ctx context.Context,
sessionID, executionID string,
generation, activityEpoch uint64,
) error {
execution, err := m.executionStore.ClaimPromptActivity(sessionID, executionID, generation, activityEpoch)
if err != nil {
if errors.Is(err, ErrExecutionNotFound) {
return fmt.Errorf("session %q: %w", sessionID, ErrNoExecutionForSession)
}
return fmt.Errorf("session %q: %w", sessionID, err)
}
return m.cancelAgentExecution(ctx, execution)
}
func (m *Manager) cancelAgentExecution(ctx context.Context, execution *AgentExecution) error {
executionID := execution.ID
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
m.logger.Info("cancelling agent turn",
zap.String("execution_id", executionID),
zap.String("task_id", execution.TaskID),
zap.String("session_id", execution.SessionID))
cancelErr := client.Cancel(ctx)
streamDisconnected := errors.Is(cancelErr, agentctlclient.ErrAgentStreamNotConnected)
if cancelErr != nil &&
!errors.Is(cancelErr, agentctlclient.ErrTurnCancelNotAcknowledged) &&
!streamDisconnected {
m.logger.Error("failed to cancel agent turn",
zap.String("execution_id", executionID),
zap.Error(cancelErr))
return fmt.Errorf("failed to cancel agent: %w", cancelErr)
}
// Don't clear buffers or mark ready here.
// The agent will respond to the original prompt with StopReason=cancelled,
// which triggers handleCompleteEvent() to properly flush buffers and mark state.
// Clearing here would race with in-flight notifications and lose content.
execution.promptFinishedMu.Lock()
ch := execution.promptFinished
execution.promptFinishedMu.Unlock()
if ch == nil {
if execution.dispatchedPromptPending.Load() {
return m.escalateStuckCancel(ctx, execution, nil)
}
if streamDisconnected {
m.logger.Info("agent stream already disconnected; cancel is complete",
zap.String("execution_id", executionID))
}
return nil
}
// Teardown may close the stream immediately before an explicit cancel (for
// example, archive followed by cancel). Treat it like an unacknowledged
// cancel: locally release any in-flight prompt and let orchestration perform
// its idempotent DB reconciliation.
if streamDisconnected {
m.logger.Warn("agent stream disconnected before cancel; escalating locally",
zap.String("execution_id", executionID),
zap.Error(cancelErr))
return m.escalateStuckCancel(ctx, execution, ch)
}
// The agent did not end the in-flight session/prompt RPC after cancel (e.g. it
// does not implement session/cancel). Escalate immediately so SendPrompt and the
// prompt gate are reconciled instead of waiting the full cancelWaitTimeout.
if errors.Is(cancelErr, agentctlclient.ErrTurnCancelNotAcknowledged) {
m.logger.Warn("agent cancel not acknowledged; escalating immediately",
zap.String("execution_id", executionID),
zap.Error(cancelErr))
return m.escalateStuckCancel(ctx, execution, ch)
}
m.logger.Info("agent cancel sent, waiting for turn completion",
zap.String("execution_id", executionID))
// Wait for the in-flight SendPrompt to finish processing the cancel completion.
// Without this, a follow-up PromptAgent races on promptDoneCh with two readers.
select {
case <-ch:
if execution.dispatchedPromptPending.Load() {
return m.escalateStuckCancel(ctx, execution, ch)
}
m.logger.Debug("in-flight prompt finished after cancel",
zap.String("execution_id", executionID))
return nil
case <-time.After(cancelWaitTimeout):
return m.escalateStuckCancel(ctx, execution, ch)
case <-ctx.Done():
return ctx.Err()
}
}
// escalateStuckCancel unblocks a SendPrompt that is stuck waiting for a completion
// event the agent will never emit. It injects a synthetic error onto promptDoneCh
// (non-blocking; the channel is buffered size 1), waits briefly for SendPrompt to
// exit and close promptFinished, and marks the execution ready so the workflow
// leaves the running state. Returns ErrCancelEscalated so callers know the ACP
// cancel did not get a clean acknowledgement.
//
// The ready-event publish below is dispatched asynchronously (asyncPublish=true
// on markReadyEventWithContext), not inline: every caller that can reach this
// escalation path (Service.CancelAgent, and cancelAgentSilent's callers —
// QueueAndInterruptForPeerMessage, retryClarificationAfterCancel,
// PauseForClarificationInput) does so while still holding sessionID's
// per-session cancelInFlightGuard, and the in-memory event bus delivers to
// the orchestrator's handleAgentReady (a queue subscriber for this event
// type) *synchronously*, on this same goroutine. An inline publish would
// have handleAgentReady try to re-acquire that same guard reentrantly and
// deadlock forever on the non-reentrant sync.Mutex. See
// markReadyEventWithContext's doc comment for the full explanation.
func (m *Manager) escalateStuckCancel(ctx context.Context, execution *AgentExecution, ch <-chan struct{}) error {
m.logger.Warn("timed out waiting for in-flight prompt to finish after cancel; escalating",
zap.String("execution_id", execution.ID),
zap.String("session_id", execution.SessionID))
// Clear the cancelled dispatch-only prompt before releasing its waiter. If
// the waiter immediately admits a successor, a later cleanup must not clear
// the successor's gate.
execution.dispatchedPromptPending.Store(false)
execution.signalPromptCompletionForStartupGeneration(
execution.startupAttemptSnapshot(),
PromptCompletionSignal{
IsError: true,
Error: "cancel escalated: agent did not complete turn within timeout",
PromptGeneration: execution.promptGenerationSnapshot(),
},
)
if ch != nil {
select {
case <-ch:
m.logger.Info("in-flight prompt released after cancel escalation",
zap.String("execution_id", execution.ID))
case <-time.After(cancelEscalationTimeout):
m.logger.Warn("in-flight prompt did not release after cancel escalation",
zap.String("execution_id", execution.ID))
case <-ctx.Done():
// Fall through to MarkReady/drain below — once the synthetic signal is
// queued, the cleanup must survive the caller's context cancellation
// or the execution leaks in the Running state and the stale signal
// breaks the next PromptAgent call.
}
}
if err := m.markReadyEventWithContext(context.Background(), execution.ID, events.AgentReady, true); err != nil {
m.logger.Warn("failed to mark execution ready after cancel escalation",
zap.String("execution_id", execution.ID),
zap.Error(err))
}
// Drain any stale signal that may have been left if the agent completed
// concurrently with the escalation timeout (Go's select is non-deterministic
// when both promptFinished and time.After are ready, so our send above may
// have landed on a channel that SendPrompt had just drained).
select {
case <-execution.promptDoneCh:
default:
}
if err := ctx.Err(); err != nil {
return err
}
return ErrCancelEscalated
}
// SetSessionMode changes the session mode for a running agent.
// Always uses the execution's current ACPSessionID rather than the caller's copy,
// which may be stale during startup/restart when the session is being re-initialized.
func (m *Manager) SetSessionMode(ctx context.Context, executionID, _ string, modeID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
operationRelease, err := execution.acquireContextResetOperation(ctx)
if err != nil {
return err
}
defer operationRelease()
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
if !execution.isSessionInitialized() || execution.ACPSessionID == "" {
return fmt.Errorf("execution %q ACP session is not ready", executionID)
}
return client.SetMode(ctx, execution.ACPSessionID, modeID)
}
// SetSessionModeBySessionID changes the session mode for a running agent by session ID.
func (m *Manager) SetSessionModeBySessionID(ctx context.Context, sessionID, modeID string) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent running for session %q", sessionID)
}
return m.SetSessionMode(ctx, execution.ID, execution.ACPSessionID, modeID)
}
// SetSessionModel changes the session model for a running agent. ACP agents
// swap the model in-place via their advertised model-selection mechanism.
// Passthrough (TUI) agents have no protocol channel — the model is a CLI flag
// baked into the launch — so the override is persisted on the execution and
// the PTY is relaunched so the next process picks up the new --model.
func (m *Manager) SetSessionModel(ctx context.Context, executionID, modelID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
operationRelease, err := execution.acquireContextResetOperation(ctx)
if err != nil {
return err
}
if execution.PassthroughProcessID != "" {
if err := m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
exec.setMetadataValue(MetadataKeyModelOverride, modelID)
}); err != nil {
operationRelease()
return fmt.Errorf("failed to persist model override for execution %q: %w", executionID, err)
}
operationRelease()
return m.RestartAgentProcess(ctx, executionID)
}
defer operationRelease()
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
return client.SetModel(ctx, modelID)
}
// SetSessionModelBySessionID changes the session model for a running agent by session ID.
func (m *Manager) SetSessionModelBySessionID(ctx context.Context, sessionID, modelID string) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent running for session %q", sessionID)
}
return m.SetSessionModel(ctx, execution.ID, modelID)
}
// SetSessionConfigOption changes an ACP session config option for a running agent.
func (m *Manager) SetSessionConfigOption(ctx context.Context, executionID, configID, value string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
operationRelease, err := execution.acquireContextResetOperation(ctx)
if err != nil {
return err
}
defer operationRelease()
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
if !execution.isSessionInitialized() || execution.ACPSessionID == "" {
return fmt.Errorf("execution %q ACP session is not ready", executionID)
}
return client.SetConfigOption(ctx, configID, value)
}
// SetSessionConfigOptionBySessionID changes an ACP session config option by task session ID.
func (m *Manager) SetSessionConfigOptionBySessionID(ctx context.Context, sessionID, configID, value string) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent running for session %q", sessionID)
}
return m.SetSessionConfigOption(ctx, execution.ID, configID, value)
}
// AuthenticateBySessionID triggers authentication for a given auth method on the agent.
func (m *Manager) AuthenticateBySessionID(ctx context.Context, sessionID, methodID string) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent running for session %q", sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", execution.ID)
}
return client.Authenticate(ctx, methodID)
}
// reapplySessionModeAfterReset re-applies the active session permission mode
// (e.g. auto / accept-edits) to a freshly (re)initialized ACP session so the
// user's choice survives a context reset instead of silently reverting to the
// agent's default.
//
// The mode to restore is resolved from the persisted session_mode in the DB
// (the authoritative, synchronously-written source — see persistSessionMode and
// the set_session_mode action), falling back to prev (the in-memory mode state
// captured before the reset) only when no provider is wired or nothing is
// persisted. Preferring the DB avoids re-applying a stale in-memory mode when a
// set_session_mode action persisted a newer mode in the same on_enter batch
// before its agent mode event updated modeState. A nil/empty resolved mode is a
// no-op. Addresses issue #1183.
func (m *Manager) reapplySessionModeAfterReset(ctx context.Context, execution *AgentExecution, newSessionID string, prev *CachedModeState) {
fallback := ""
if prev != nil {
fallback = prev.CurrentModeID
}
mode := m.effectiveSessionMode(ctx, execution, fallback)
if err := m.applySessionModeAfterReset(ctx, execution, newSessionID, mode); err != nil {
m.logger.Warn("failed to re-apply session mode after context reset",
zap.String("execution_id", execution.ID),
zap.String("mode", mode),
zap.Error(err))
}
}
func (m *Manager) applySessionModeAfterReset(
ctx context.Context,
execution *AgentExecution,
newSessionID, mode string,
) error {
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil || mode == "" {
return nil
}
if err := client.SetMode(ctx, newSessionID, mode); err != nil {
return fmt.Errorf("failed to restore session mode %q: %w", mode, err)
}
availableModes := []streams.SessionModeInfo(nil)
if current := execution.GetModeState(); current != nil {
availableModes = current.AvailableModes
}
// Restore the cache too: the fresh session would otherwise report the agent's
// default mode, leaving modeState stale relative to what we just re-applied.
execution.SetModeState(&CachedModeState{
CurrentModeID: mode,
AvailableModes: availableModes,
})
m.logger.Info("re-applied session mode after context reset",
zap.String("execution_id", execution.ID),
zap.String("session_id", execution.SessionID),
zap.String("mode", mode))
return nil
}
// reapplySessionModelAfterReset applies the executor-authoritative model
// decision to a freshly initialized ACP session.
func (m *Manager) reapplySessionModelAfterReset(
ctx context.Context,
execution *AgentExecution,
newSessionID, modelID string,
) error {
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
return m.reapplySessionModelAfterResetWithClient(ctx, execution, client, newSessionID, modelID)
}
func (m *Manager) reapplySessionModelAfterResetWithClient(
ctx context.Context,
execution *AgentExecution,
client *agentctlclient.Client,
newSessionID, modelID string,
) error {
return m.reapplySessionModelAfterResetWithClientAndState(
ctx, execution, client, execution.GetModelState(), newSessionID, modelID,
)
}
func (m *Manager) reapplySessionModelAfterResetWithClientAndState(
ctx context.Context,
execution *AgentExecution,
client *agentctlclient.Client,
modelState *CachedModelState,
newSessionID, modelID string,
) error {
if client == nil || modelID == "" {
return nil
}
policy := m.resolveStartModelPolicy(ctx, execution.AgentProfileID)
policy.Model = modelID
decision, err := applyStartModelPolicy(
ctx, m.logger, client, modelState, policy,
)
if err != nil {
m.logger.Warn("failed to re-apply session model after context reset",
zap.String("execution_id", execution.ID),
zap.String("model", modelID),
zap.Error(err))
return err
}
if decision.Warning && m.sessionManager != nil {
m.sessionManager.publishModelSelectionWarningEvent(execution, newSessionID, decision)
}
if decision.EffectiveModel != "" &&
(decision.Outcome == ModelSelectionOutcomeApplied ||
decision.Outcome == ModelSelectionOutcomeExplicitFallback) {
m.logger.Info("re-applied session model after context reset",
zap.String("execution_id", execution.ID),
zap.String("session_id", execution.SessionID),
zap.String("new_acp_session_id", newSessionID),
zap.String("model", decision.EffectiveModel),
zap.Bool("using_fallback", decision.Outcome == ModelSelectionOutcomeExplicitFallback))
}
return nil
}
func cacheFreshSessionModelState(execution *AgentExecution) bool {
if execution == nil {
return false
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
return cacheSessionModelStateFromClient(execution, client)
}
func cacheSessionModelStateFromClient(execution *AgentExecution, client *agentctlclient.Client) bool {
if execution == nil || client == nil {
return false
}
state, ready := sessionModelStateFromClient(client)
if !ready {
return false
}
execution.SetModelState(state)
return true
}
func sessionModelStateFromClient(client *agentctlclient.Client) (*CachedModelState, bool) {
if client == nil {
return nil, false
}
state := client.GetLastSessionModelState()
if state == nil {
return nil, false
}
cached := &CachedModelState{
CurrentModelID: state.CurrentModelID,
Models: state.Models,
ConfigOptions: state.ConfigOptions,
ConfigOptionsSettled: state.ConfigOptionsSettled,
}
if !freshSessionModelCatalogReady(cached) {
return nil, false
}
return cached, true
}
func waitForFreshSessionModelStateFromClient(
ctx context.Context,
log *logger.Logger,
execution *AgentExecution,
client *agentctlclient.Client,
) bool {
if execution == nil || client == nil {
return false
}
if cacheSessionModelStateFromClient(execution, client) {
return true
}
waitCtx, cancel := context.WithTimeout(ctx, freshSessionModelStateWait)
defer cancel()
ticker := time.NewTicker(freshSessionModelStatePoll)
defer ticker.Stop()
for {
select {
case <-waitCtx.Done():
if log != nil {
log.Debug("fresh session model catalog was not reported before policy evaluation",
zap.String("execution_id", execution.ID),
zap.Error(waitCtx.Err()))
}
return false
case <-ticker.C:
if cacheSessionModelStateFromClient(execution, client) {
return true
}
}
}
}
// waitForFreshSessionModelState gives the agent stream a chance to deliver the
// new session's model catalog before the reset/restart policy runs. ACP
// session/new returns before its session_models notification is dispatched to
// the lifecycle manager. This is the fallback for transports that do not
// include a synchronous snapshot in their session response. Reading the old
// nil/empty cache at that boundary would incorrectly treat an advertised model
// as unavailable and leave the agent on its provider default. A missing
// notification still follows the executor-authoritative default path after the
// bounded wait.
func waitForFreshSessionModelState(ctx context.Context, log *logger.Logger, execution *AgentExecution) bool {
if execution == nil {
return false
}
if freshSessionModelCatalogReady(execution.GetModelState()) {
return true
}
waitCtx, cancel := context.WithTimeout(ctx, freshSessionModelStateWait)
defer cancel()
ticker := time.NewTicker(freshSessionModelStatePoll)
defer ticker.Stop()
for {
select {
case <-waitCtx.Done():
if log != nil {
log.Debug("fresh session model catalog was not reported before policy evaluation",
zap.String("execution_id", execution.ID),
zap.Error(waitCtx.Err()))
}
return false
case <-ticker.C:
if freshSessionModelCatalogReady(execution.GetModelState()) {
return true
}
}
}
}
func freshSessionModelCatalogReady(state *CachedModelState) bool {
return state != nil && (len(state.Models) > 0 || len(state.ConfigOptions) > 0 || state.ConfigOptionsSettled)
}
func (m *Manager) effectiveSessionModelForReset(ctx context.Context, execution *AgentExecution) string {
return m.captureSessionRuntimeConfigForReset(ctx, execution).Model
}
func (m *Manager) captureSessionRuntimeConfigForReset(
ctx context.Context,
execution *AgentExecution,
) models.SessionRuntimeConfig {
profileModel, profileMode, profileOptions := m.resolveProfileSessionConfig(ctx, execution.AgentProfileID)
modelState := execution.GetModelState()
liveModel := profileModel
if modelState != nil && strings.TrimSpace(modelState.CurrentModelID) != "" {
liveModel = modelState.CurrentModelID
}
liveMode := profileMode
if modeState := execution.GetModeState(); modeState != nil && strings.TrimSpace(modeState.CurrentModeID) != "" {
liveMode = modeState.CurrentModeID
}
liveOptions := maps.Clone(profileOptions)
if liveStateOptions, liveOptionsKnown := liveSessionRuntimeConfigOptions(modelState); liveOptionsKnown {
// A settled provider snapshot is the live layer, including an empty
// snapshot. Keep an allocated empty map so the effective-config helper
// preserves that presence while persisted state can still overlay it.
liveOptions = make(map[string]string, len(liveStateOptions))
for id, value := range liveStateOptions {
liveOptions[id] = value
}
}
model, mode, options, optionsSet := m.effectiveSessionRuntimeConfigWithPresence(
ctx, execution, liveModel, liveMode, liveOptions,
)
if !optionsSet {
options = selectedRuntimeConfigOptions(modelState)
}
return models.SessionRuntimeConfig{
Model: model,
Mode: mode,
ConfigOptions: sanitizeRuntimeConfigOptionsWithCatalog(options, modelState),
}
}
func liveSessionRuntimeConfigOptions(state *CachedModelState) (map[string]string, bool) {
if state == nil || (!state.ConfigOptionsSettled && len(state.ConfigOptions) == 0) {
return nil, false
}
return selectedRuntimeConfigOptions(state), true
}
func selectedRuntimeConfigOptions(state *CachedModelState) map[string]string {
if state == nil {
return nil
}
options := make(map[string]string, len(state.ConfigOptions))
for _, option := range state.ConfigOptions {
id := strings.TrimSpace(option.ID)
value := strings.TrimSpace(option.CurrentValue)
if isRestorableRuntimeConfigOption(id, option.Category, value) {
options[id] = value
}
}
if len(options) == 0 {
return nil
}
return options
}
func sanitizeRuntimeConfigOptions(options map[string]string) map[string]string {
return sanitizeRuntimeConfigOptionsWithCatalog(options, nil)
}
func sanitizeRuntimeConfigOptionsWithCatalog(options map[string]string, state *CachedModelState) map[string]string {
if options == nil {
return nil
}
catalog, catalogKnown := capturedRuntimeConfigOptionCatalog(state)
cleaned := make(map[string]string, len(options))
for id, value := range options {
trimmedID := strings.TrimSpace(id)
category := ""
if catalogKnown {
var ok bool
category, ok = catalog[trimmedID]
if !ok {
continue
}
}
if isRestorableRuntimeConfigOption(trimmedID, category, value) {
cleaned[trimmedID] = strings.TrimSpace(value)
}
}
return cleaned
}
func capturedRuntimeConfigOptionCatalog(state *CachedModelState) (map[string]string, bool) {
if state == nil || (!state.ConfigOptionsSettled && len(state.ConfigOptions) == 0) {
return nil, false
}
catalog := make(map[string]string, len(state.ConfigOptions))
for _, option := range state.ConfigOptions {
id := strings.TrimSpace(option.ID)
if id == "" {
continue
}
category := strings.TrimSpace(option.Category)
if previous, exists := catalog[id]; exists && previous != "" {
continue
}
catalog[id] = category
}
return catalog, true
}
func isRestorableRuntimeConfigOption(id, category, value string) bool {
id = strings.TrimSpace(id)
value = strings.TrimSpace(value)
if id == "" || value == "" {
return false
}
if strings.EqualFold(id, "model") || strings.EqualFold(id, "mode") {
return false
}
return !strings.EqualFold(category, "model") && !strings.EqualFold(category, "mode")
}
func (m *Manager) restoreSessionRuntimeConfig(
ctx context.Context,
execution *AgentExecution,
sessionID string,
config models.SessionRuntimeConfig,
) error {
if execution == nil {
return fmt.Errorf("cannot restore session runtime configuration without an agentctl client")
}
client, releaseClient := execution.AcquireAgentCtlClient()
releaseClient()
if client == nil {
return fmt.Errorf("cannot restore session runtime configuration without an agentctl client")
}
if err := m.reapplySessionModelAfterReset(ctx, execution, sessionID, config.Model); err != nil {
return fmt.Errorf("restore session runtime model: %w", err)
}
if err := m.applySessionModeAfterReset(ctx, execution, sessionID, config.Mode); err != nil {
return fmt.Errorf("restore session runtime mode: %w", err)
}
options := sanitizeRuntimeConfigOptionsWithCatalog(config.ConfigOptions, execution.GetModelState())
optionIDs := make([]string, 0, len(options))
for id := range options {
optionIDs = append(optionIDs, id)
}
sort.Strings(optionIDs)
for _, id := range optionIDs {
value := options[id]
client, releaseClient := execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("restore session runtime option %q: agentctl client is unavailable", id)
}
err := client.SetConfigOption(ctx, id, value)
releaseClient()
if err != nil {
return fmt.Errorf("restore session runtime option %q: %w", id, err)
}
}
return nil
}
// ResetAgentContext resets the agent's conversation context. For ACP agents that support
// session reset, this creates a new session on the existing connection (fast, no process restart).
// For all other agents, this falls back to RestartAgentProcess (full subprocess restart).
func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
// Passthrough agents always need full restart
if execution.PassthroughProcessID != "" {
return m.RestartAgentProcess(ctx, executionID)
}
if err := execution.contextResetAdmissionError(); err != nil {
return err
}
execution.remoteInstanceLifecycleMu.Lock()
defer execution.remoteInstanceLifecycleMu.Unlock()
if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
if err := execution.beginContextReset(); err != nil {
return err
}
operationRelease, err := execution.acquireContextResetExclusive(ctx)
if err != nil {
execution.failContextReset(false, err.Error())
return err
}
defer operationRelease()
client, releaseClient := execution.AcquireAgentCtlClient()
if client == nil {
execution.failContextReset(false, "agentctl client is unavailable")
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
// Capture the complete effective session configuration before the reset. The
// fresh ACP session starts at provider defaults, so asynchronous events from
// it must not replace the task's pre-reset restoration intent.
runtimeConfig := m.captureSessionRuntimeConfigForReset(ctx, execution)
// Clear the old catalog before creating the fresh session. The new
// session_models event is delivered asynchronously and must be the only
// catalog consulted by the reset decision.
_ = m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
exec.SetModelState(nil)
})
// Resolve agent config and MCP servers for session reset
agentConfig, err := m.getAgentConfigForExecution(execution)
if err != nil {
releaseClient()
execution.failContextReset(false, err.Error())
m.logger.Info("cannot resolve agent config for session reset, falling back to process restart",
zap.String("execution_id", executionID), zap.Error(err))
if restartErr := m.restartAgentProcess(ctx, executionID, &runtimeConfig); restartErr != nil {
execution.failContextReset(true, restartErr.Error())
return restartErr
}
return nil
}
mcpServers, err := m.resolveMcpServers(ctx, execution, agentConfig)
if err != nil {
releaseClient()
execution.failContextReset(false, err.Error())
m.logger.Warn("cannot resolve MCP servers for session reset, falling back to process restart",
zap.String("execution_id", executionID), zap.Error(err))
if restartErr := m.restartAgentProcess(ctx, executionID, &runtimeConfig); restartErr != nil {
execution.failContextReset(true, restartErr.Error())
return restartErr
}
return nil
}
// Try session-level reset (only ACP adapters support this). The provider
// request has its own bound so an unanswered response cannot hold the
// execution lifecycle lock indefinitely.
resetCtx, cancelReset := context.WithTimeout(ctx, resetAgentContextRequestTimeout)
newSessionID, err := client.ResetSession(resetCtx, execution.WorkspacePath, mcpServers)
resetErr := resetCtx.Err()
cancelReset()
releaseClient()
// A successful session/new response is authoritative even if the request
// context becomes done while the adapter is returning. In particular, the
// adapter may still be doing detached superseded-session cleanup after it
// has committed the replacement. Only an unsuccessful RPC can turn the
// request deadline into an uncertain reset that needs fencing.
if err != nil {
if resetErr != nil {
resetFailure := fmt.Errorf("agent session reset request ended with %w", resetErr)
if err != nil {
resetFailure = fmt.Errorf("%w: %v", resetFailure, err)
}
execution.failContextReset(true, resetFailure.Error())
return resetFailure
}
execution.failContextReset(false, err.Error())
m.logger.Info("session reset not supported, falling back to process restart",
zap.String("execution_id", executionID), zap.Error(err))
if restartErr := m.restartAgentProcess(ctx, executionID, &runtimeConfig); restartErr != nil {
execution.failContextReset(true, restartErr.Error())
return restartErr
}
return nil
}
if newSessionID == "" {
err := errors.New("agent session reset returned an empty session ID")
execution.failContextReset(true, err.Error())
return err
}
// Success — update execution state without restarting process
if err := m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
exec.ACPSessionID = newSessionID
exec.needsResumeContext = false
exec.resumeContextInjected = false
m.resetStreamingStateWithHistory(exec)
// Drain any stale prompt completion signal
select {
case <-exec.promptDoneCh:
default:
}
exec.dispatchedPromptPending.Store(false)
}); err != nil {
execution.failContextReset(true, err.Error())
return fmt.Errorf("commit reset session state: %w", err)
}
for _, event := range execution.drainContextResetEvents(newSessionID) {
m.handleAgentEventAfterContextReset(execution, event)
}
reconcileCtx, cancelReconcile := context.WithTimeout(
context.WithoutCancel(ctx), resetAgentContextRequestTimeout,
)
defer cancelReconcile()
// Restore the complete captured configuration. A strict-mode model that is
// rejected by the fresh session fails the reset explicitly instead of
// silently dropping to the provider default.
if !cacheFreshSessionModelState(execution) &&
(runtimeConfig.Model != "" || len(runtimeConfig.ConfigOptions) > 0) {
waitForFreshSessionModelState(reconcileCtx, m.logger, execution)
}
if err := m.restoreSessionRuntimeConfig(reconcileCtx, execution, newSessionID, runtimeConfig); err != nil {
restoreErr := fmt.Errorf("failed to restore session runtime configuration after reset: %w", err)
execution.failContextReset(true, restoreErr.Error())
m.updateExecutionError(executionID, restoreErr.Error())
m.persistExecutorRunning(context.WithoutCancel(reconcileCtx), execution)
return restoreErr
}
for _, event := range execution.drainContextResetEvents(newSessionID) {
m.handleAgentEventAfterContextReset(execution, event)
}
if err := m.updateStatusAndPersist(reconcileCtx, executionID, v1.AgentStatusReady); err != nil {
resetErr := fmt.Errorf("failed to mark reset agent ready: %w", err)
execution.failContextReset(true, resetErr.Error())
m.updateExecutionError(executionID, "failed to mark reset agent ready: "+err.Error())
m.persistExecutorRunning(context.WithoutCancel(reconcileCtx), execution)
return resetErr
}
m.logger.Info("agent context reset via session (no process restart)",
zap.String("execution_id", executionID),
zap.String("session_id", execution.SessionID),
zap.String("new_acp_session_id", newSessionID))
m.eventPublisher.PublishAgentEvent(reconcileCtx, events.AgentContextReset, execution)
// Boot-equivalent signal: a fresh ACP session is alive but no turn has run yet.
// Use AgentBootReady so the orchestrator routes this to handleAgentBootReady
// (idle/WAITING transition) rather than handleAgentReady (turn-end transition,
// which would fire on_turn_complete against the current step — the original
// boot-vs-turn ambiguity bug).
m.eventPublisher.PublishAgentEvent(reconcileCtx, events.AgentBootReady, execution)
for _, event := range execution.finishContextReset(newSessionID) {
m.handleAgentEventAfterContextReset(execution, event)
}
return nil
}
// ErrNoExecutionForSession is returned when no live execution is tracked for a session,
// typically because the agent process has crashed, exited, or the session state is stuck
// (for example after a backend restart that did not re-register the execution).
//
// Callers should treat this as a "there is nothing to cancel/stop" signal and still reconcile
// the session's state in the database, otherwise the session will appear stuck as RUNNING
// with no agent to drive it.
var ErrNoExecutionForSession = errors.New("no execution for session")
// ErrCancelEscalated is returned by CancelAgent when the agent subprocess accepted
// the ACP cancel but did not publish a completion event before the timeout. The
// lifecycle manager has locally unblocked the in-flight SendPrompt and marked the
// execution ready, but the agent never acknowledged the cancel. Callers should
// still reconcile session-level state (e.g. transition the task session to
// WAITING_FOR_INPUT) — the user's intent was unambiguous.
var ErrCancelEscalated = errors.New("cancel escalated: agent did not acknowledge within timeout")
// CancelAgentBySessionID cancels the current agent turn for a specific session.
// Returns ErrNoExecutionForSession (wrapped) when no execution is tracked for the session.
//
// Passthrough sessions don't speak ACP — Ctrl-C (0x03) written to PTY stdin is
// the only stop signal a TUI CLI understands. A write failure is non-fatal:
// the caller still reconciles DB state so the UI unsticks even if the PTY is
// already gone.
func (m *Manager) CancelAgentBySessionID(ctx context.Context, sessionID string) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("session %q: %w", sessionID, ErrNoExecutionForSession)
}
if execution.PassthroughProcessID != "" {
if err := m.WritePassthroughStdin(ctx, sessionID, "\x03"); err != nil {
m.logger.Warn("failed to write Ctrl-C to passthrough stdin",
zap.String("session_id", sessionID),
zap.Error(err))
}
return nil
}
return m.CancelAgent(ctx, execution.ID)
}
// StopAgent stops an agent execution
func (m *Manager) StopAgent(ctx context.Context, executionID string, force bool) error {
return m.StopAgentWithReason(ctx, executionID, "", force)
}
// StopAgentWithReason stops an agent execution and passes a semantic reason to runtime teardown.
func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, reason string, force bool) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
if handled, err := m.stopPersistedKubernetesExecution(ctx, executionID, reason, force); handled {
return err
}
if handled, err := m.stopPersistedSSHExecution(ctx, executionID, reason, force); handled {
return err
}
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
execution.remoteInstanceLifecycleMu.Lock()
defer execution.remoteInstanceLifecycleMu.Unlock()
if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
backendForce := force
stopCtx := ctx
if shouldPreserveFailedKubernetesResume(execution, reason) {
backendForce = false
var cancelStop context.CancelFunc
stopCtx, cancelStop = kubernetesDurableContext(ctx)
defer cancelStop()
}
activityLease, err := m.acquireActivity(stopCtx, activity.KindExecutionStopping)
if err != nil {
return err
}
defer activityLease.Release()
// Keep the execution's running activity lease until backend teardown
// succeeds. A failed stop remains retryable, so maintenance must not treat
// a potentially live runtime as idle. RemoveExecution releases the lease on
// the successful path.
// AC-EXECUTORS-SURVIVAL kill-path #4 (design 01 "Kill paths that must
// change together", design 02 "Shutdown"): a graceful backend shutdown
// must not terminate the instance when the capability is enabled. Every
// other stop reason, including force, keeps the terminating path below.
// The capability covers only the standalone (worktree/local) runtime --
// every other runtime's StopAllAgents call must still terminate normally
// even while the capability is globally enabled for the installation.
// A passthrough session is excluded too (AC-EXECUTORS-SURVIVAL-005.3,
// design 02 "Passthrough scope"): its agent runs on a terminal this
// backend process owns, so it dies with the backend regardless, and
// detaching would leave an executors_running row claiming a live agent
// with no agent.stopped published. See isPassthroughExecution.
if m.agentSurvivalEnabled && reason == StopReasonBackendShutdown &&
execution.RuntimeName == executor.NameStandalone && !isPassthroughExecution(execution) {
return m.detachAgentExecution(executionID, execution)
}
m.logger.Info("stopping agent",
zap.String("execution_id", executionID),
zap.String("reason", reason),
zap.Bool("force", force),
zap.Bool("runtime_force", backendForce),
zap.Stringer("runtime", execution.RuntimeName))
// Try to gracefully stop via agentctl first, then always close connections.
// A retained Kubernetes resume gets a bounded non-cancelled opportunity to
// stop the failed process before its Pod is preserved for another retry.
agentStopFailed := m.stopExecutionAgentctl(stopCtx, executionID, execution, backendForce)
// Stop the agent execution via the runtime that created it. A failed stop
// must remain tracked: removing it here would turn a retryable cleanup into
// an unobservable orphan process.
if err := m.stopAgentViaBackend(stopCtx, executionID, execution, reason, backendForce, agentStopFailed); err != nil {
return fmt.Errorf("stop runtime for execution %q: %w", executionID, err)
}
if execution.RuntimeName == executor.NameKubernetes && (backendForce || shouldRunExecutorCleanup(reason)) {
cleanupCtx, cancelCleanup := kubernetesDurableContext(ctx)
err := m.deleteKubernetesRuntimeSecrets(cleanupCtx, execution.MetadataSnapshot())
cancelCleanup()
if err != nil {
return fmt.Errorf("delete runtime secrets for execution %s: %w", executionID, err)
}
}
// The runtime is now terminal. Detach agentctl under its own short critical
// section before any cleanup or event snapshot can acquire prompt state.
// Failed runtime stops intentionally retain the closed-but-reusable client so
// a later retry can issue the agentctl stop request again.
execution.agentctlLifecycleMu.Lock()
execution.detachAgentctlClient()
execution.agentctlLifecycleMu.Unlock()
// Update execution status and remove from tracking
_ = m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
exec.Status = v1.AgentStatusStopped
now := time.Now()
exec.FinishedAt = &now
})
// End session trace span
execution.EndSessionSpan()
m.RemoveExecution(executionID)
m.clearRemoteStatus(execution.SessionID)
m.logger.Info("agent stopped and removed from tracking",
zap.String("execution_id", executionID),
zap.String("task_id", execution.TaskID))
// Publish stopped event
m.eventPublisher.PublishAgentEvent(ctx, events.AgentStopped, execution)
return nil
}
func shouldPreserveFailedKubernetesResume(execution *AgentExecution, reason string) bool {
return execution != nil && execution.RuntimeName == executor.NameKubernetes &&
execution.isResumedSession && reason == StopReasonAgentBootstrapFailed
}
// detachAgentExecution implements the AC-EXECUTORS-SURVIVAL survivable-detach
// branch of StopAgentWithReason (design 01/02 kill-path #4): when the
// agent-survival capability is enabled and the stop reason is graceful
// backend shutdown, this backend releases its own hold on the execution --
// its agentctl stream subscription and in-memory tracking -- without
// stopping the agentctl-side instance or calling the runtime backend's
// StopInstance. The instance is left running, unowned by this backend, so
// the adoption path (Layer 6.1) or a fresh recovery pass on the next startup
// can find and re-track it.
//
// Deliberately does NOT: call client.Stop, call stopAgentViaBackend, set
// execution.Status to stopped, or publish events.AgentStopped. The execution
// is not stopped -- publishing that event would run the orchestrator's
// handleAgentStopped, which persists a terminal task-session state for a
// session whose agent is still running, corrupting the very state recovery
// depends on. The executors_running row is intentionally left untouched so
// AC-EXECUTORS-SURVIVAL-002's recovery-inventory read still finds it live.
//
// Also deliberately does NOT call client.Close(): closing the client tears
// down the agentctl updates stream, which makes any in-flight SendPrompt's
// stream-reader goroutine observe a disconnect and report it through the
// same error path a genuine agent crash uses (handleInitialPromptFailure /
// handleErrorEvent) -- persisting the executors_running row as failed out
// from under the very read this function's contract promises to leave
// untouched, and racing RemoveExecution below to do it. Simply dropping the
// local reference lets this backend's process exit (moments later, in the
// same shutdown) reclaim the socket without disturbing the still-running
// agentctl instance or its live stream.
func (m *Manager) detachAgentExecution(executionID string, execution *AgentExecution) error {
execution.agentctlLifecycleMu.Lock()
execution.detachAgentctlClient()
execution.agentctlLifecycleMu.Unlock()
execution.EndSessionSpan()
m.RemoveExecution(executionID)
m.clearRemoteStatus(execution.SessionID)
m.logger.Info("detached agent execution for survivable backend shutdown; instance left running",
zap.String("execution_id", executionID),
zap.String("session_id", execution.SessionID),
zap.String("task_id", execution.TaskID),
zap.Stringer("runtime", execution.RuntimeName))
return nil
}
func (m *Manager) stopExecutionAgentctl(
ctx context.Context,
executionID string,
execution *AgentExecution,
force bool,
) bool {
execution.agentctlLifecycleMu.Lock()
defer execution.agentctlLifecycleMu.Unlock()
client := execution.currentAgentCtlClient() // protected by agentctlLifecycleMu
if client == nil {
return false
}
defer client.Close()
if force {
return false
}
if err := client.Stop(ctx); err != nil {
// During shutdown the instance may already be stopping through another
// lifecycle path, so a failed HTTP call is expected.
if m.IsShuttingDown() {
m.logger.Debug("failed to stop agent via agentctl",
zap.String("execution_id", executionID), zap.Error(err))
} else {
m.logger.Warn("failed to stop agent via agentctl",
zap.String("execution_id", executionID), zap.Error(err))
}
return true
}
return false
}
// StopBySessionID stops the agent for a specific session
func (m *Manager) StopBySessionID(ctx context.Context, sessionID string, force bool) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent running for session %q", sessionID)
}
return m.StopAgent(ctx, execution.ID, force)
}
// RestartAgentProcess stops the agent subprocess and starts a fresh one, clearing the agent's
// conversation context. For ACP agents this restarts via agentctl with a new ACP session.
// For passthrough (TUI) agents this kills the PTY process and relaunches without --resume.
func (m *Manager) RestartAgentProcess(ctx context.Context, executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
execution.remoteInstanceLifecycleMu.Lock()
defer execution.remoteInstanceLifecycleMu.Unlock()
if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
operationRelease, err := execution.acquireContextResetExclusive(ctx)
if err != nil {
return err
}
defer operationRelease()
err = m.restartAgentProcess(ctx, executionID, nil)
if err != nil {
execution.finishContextResetRecovery(false, err.Error())
return err
}
execution.finishContextResetRecovery(true, "")
return nil
}
// restartAgentProcess restarts an ACP process. When runtimeConfigOverride is
// supplied, it is the immutable pre-reset snapshot and must be used instead of
// reading state again after ResetAgentContext has cleared the old catalog.
func (m *Manager) restartAgentProcess(
ctx context.Context,
executionID string,
runtimeConfigOverride *models.SessionRuntimeConfig,
) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
// Passthrough agents: kill PTY and relaunch fresh (no --resume).
if execution.PassthroughProcessID != "" {
return m.restartPassthroughProcess(ctx, execution)
}
client, releaseClient := execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
releaseClient()
preparation, err := m.prepareAgentRestart(ctx, execution, runtimeConfigOverride)
if err != nil {
return err
}
// 1. Close WebSocket streams (updates + workspace). Use per-stream Close
// methods rather than client.Close — the latter is a terminal drain
// barrier that flips the client into a closed state and would block
// every StreamUpdates/StreamWorkspace call that this same restart path
// makes a few lines below.
client, releaseClient = execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
client.CloseUpdatesStream()
client.CloseWorkspaceStream()
releaseClient()
// 2. Stop the agent subprocess via agentctl (keeps agentctl server alive)
m.stopAgentProcessForRestart(ctx, execution)
// 3. Reset execution state after the replacement command has been validated.
m.resetAgentRestartState(executionID, preparation.commands)
// 4. Wait for agentctl to be ready (it should still be running)
client, releaseClient = execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
err = client.WaitForReady(ctx, 30*time.Second)
releaseClient()
if err != nil {
m.updateExecutionError(executionID, "agentctl not ready after restart: "+err.Error())
return fmt.Errorf("agentctl not ready after restart: %w", err)
}
// 5. Reconfigure and start new agent subprocess
approvalPolicy, _ := m.resolveApprovalPolicyAndDisplayName(ctx, execution)
if _, err := m.configureAndStartAgent(ctx, execution, approvalPolicy); err != nil {
m.updateExecutionError(executionID, "failed to restart agent: "+err.Error())
return fmt.Errorf("failed to restart agent: %w", err)
}
// 6. Wait for agent process to initialize
client, releaseClient = execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("execution %q has no agentctl client", executionID)
}
err = client.WaitForReady(ctx, 10*time.Second)
releaseClient()
if err != nil {
m.logger.Warn("agent process slow to initialize after restart, continuing",
zap.String("execution_id", executionID),
zap.Error(err))
}
mcpServers, err := m.resolveMcpServers(ctx, execution, preparation.agentConfig)
if err != nil {
return fmt.Errorf("failed to resolve MCP config for restart: %w", err)
}
if err := m.initializeACPSessionForRestart(ctx, execution, preparation.agentConfig, mcpServers); err != nil {
m.updateExecutionError(executionID, "failed to initialize ACP session after restart: "+err.Error())
return fmt.Errorf("failed to initialize ACP session after restart: %w", err)
}
if !cacheFreshSessionModelState(execution) &&
(preparation.runtimeConfig.Model != "" || len(preparation.runtimeConfig.ConfigOptions) > 0) {
waitForFreshSessionModelState(ctx, m.logger, execution)
}
if err := m.restoreSessionRuntimeConfig(ctx, execution, execution.ACPSessionID, preparation.runtimeConfig); err != nil {
m.updateExecutionError(executionID, "failed to restore session runtime configuration after restart: "+err.Error())
return fmt.Errorf("failed to restore session runtime configuration after restart: %w", err)
}
if err := m.updateStatusAndPersist(ctx, execution.ID, v1.AgentStatusReady); err != nil {
m.updateExecutionError(executionID, "failed to mark restarted agent ready: "+err.Error())
return fmt.Errorf("failed to mark restarted agent ready: %w", err)
}
m.eventPublisher.PublishAgentEvent(ctx, events.AgentBootReady, execution)
m.logger.Info("agent process restarted with fresh context",
zap.String("execution_id", executionID),
zap.String("session_id", execution.SessionID),
zap.String("new_acp_session_id", execution.ACPSessionID))
m.eventPublisher.PublishAgentEvent(ctx, events.AgentContextReset, execution)
return nil
}
func (m *Manager) stopAgentProcessForRestart(ctx context.Context, execution *AgentExecution) {
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return
}
if err := client.Stop(ctx); err != nil {
m.logger.Warn("failed to stop agent subprocess during restart",
zap.String("execution_id", execution.ID),
zap.Error(err))
// Continue — the process may already be stopped
}
}
type agentRestartPreparation struct {
agentConfig agents.Agent
commands agentCommands
runtimeConfig models.SessionRuntimeConfig
}
// prepareAgentRestart builds and validates the replacement before touching the current process.
func (m *Manager) prepareAgentRestart(
ctx context.Context,
execution *AgentExecution,
runtimeConfigOverride *models.SessionRuntimeConfig,
) (agentRestartPreparation, error) {
m.logger.Info("restarting agent process for context reset",
zap.String("execution_id", execution.ID),
zap.String("task_id", execution.TaskID),
zap.String("session_id", execution.SessionID))
agentConfig, err := m.getAgentConfigForExecution(execution)
if err != nil {
return agentRestartPreparation{}, fmt.Errorf("failed to get agent config for restart: %w", err)
}
commands, err := m.buildFreshAgentCommand(ctx, execution, agentConfig)
if err != nil {
return agentRestartPreparation{}, fmt.Errorf("failed to rebuild agent command for restart: %w", err)
}
var runtimeConfig models.SessionRuntimeConfig
if runtimeConfigOverride == nil {
runtimeConfig = m.captureSessionRuntimeConfigForReset(ctx, execution)
} else {
runtimeConfig = cloneSessionRuntimeConfig(*runtimeConfigOverride)
}
return agentRestartPreparation{
agentConfig: agentConfig,
commands: commands,
runtimeConfig: runtimeConfig,
}, nil
}
func cloneSessionRuntimeConfig(config models.SessionRuntimeConfig) models.SessionRuntimeConfig {
return models.SessionRuntimeConfig{
Model: config.Model,
Mode: config.Mode,
ConfigOptions: maps.Clone(config.ConfigOptions),
}
}
func (m *Manager) resetAgentRestartState(executionID string, commands agentCommands) {
if execution, exists := m.executionStore.Get(executionID); exists && execution.contextResetFencedState() {
// The old reset outcome is fenced until this explicit restart owns the
// replacement. Streams are closed before this state reset, so setup
// events from the new process may be admitted while the fence remains in
// place; finishContextResetRecovery clears it only after restart success.
execution.beginContextResetRecovery()
}
_ = m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
exec.ACPSessionID = ""
exec.Status = v1.AgentStatusStarting
exec.ErrorMessage = ""
exec.needsResumeContext = false
exec.resumeContextInjected = false
exec.setSessionInitialized(false)
exec.SetModelState(nil)
exec.AgentCommand = commands.initial
exec.ContinueCommand = commands.continue_
exec.AgentArgs = commands.args
exec.ContinueArgs = commands.continueArgs
m.resetStreamingStateWithHistory(exec)
select {
case <-exec.promptDoneCh:
default:
}
exec.dispatchedPromptPending.Store(false)
})
}
// initializeACPSessionForRestart connects streams and creates a new ACP session without
// sending an initial prompt. The caller (workflow processOnEnter) handles prompting separately.
func (m *Manager) initializeACPSessionForRestart(
ctx context.Context,
execution *AgentExecution,
agentConfig agents.Agent,
mcpServers []agentctltypes.McpServer,
) error {
// Connect WebSocket streams
if m.streamManager != nil {
updatesReady := make(chan struct{})
m.streamManager.ConnectAll(execution, updatesReady)
select {
case <-updatesReady:
m.logger.Debug("updates stream ready after restart")
case <-time.After(10 * time.Second):
return fmt.Errorf("timeout waiting for agent stream to connect after restart")
}
}
// Initialize ACP session (always session/new since ACPSessionID was cleared)
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("agentctl client is unavailable")
}
result, err := m.sessionManager.InitializeSession(
ctx,
client,
agentConfig,
"", // empty — force session/new
execution.WorkspacePath,
mcpServers,
)
if err != nil {
return fmt.Errorf("ACP session initialization failed: %w", err)
}
execution.ACPSessionID = result.SessionID
if m.sessionManager.eventPublisher != nil {
m.sessionManager.eventPublisher.PublishACPSessionCreatedWithAttempt(
execution, result.SessionID, ResumeAttemptIDFromContext(ctx),
)
}
return nil
}
// GetExecution returns an agent execution by ID.
//
// Returns (execution, true) if found, or (nil, false) if not found.
// The returned execution pointer should not be modified directly - use the Manager's
// methods to update execution state (MarkReady, MarkCompleted, UpdateStatus).
//
// Thread-safe: Can be called concurrently from multiple goroutines.
func (m *Manager) GetExecution(executionID string) (*AgentExecution, bool) {
return m.executionStore.Get(executionID)
}
// GetExecutionBySessionID returns the agent execution for a session from the in-memory store only.
//
// Returns (execution, true) if found, or (nil, false) if not found.
// A session can have at most one active execution at a time. If a session exists
// but has no active execution, this returns (nil, false).
//
// For workspace-oriented operations that need restart recovery,
// use GetOrEnsureExecution instead.
//
// Thread-safe: Can be called concurrently from multiple goroutines.
func (m *Manager) GetExecutionBySessionID(sessionID string) (*AgentExecution, bool) {
return m.executionStore.GetBySessionID(sessionID)
}
// ResolveTaskEnvironmentID returns the task environment ID for a session.
// User shell resources must be environment-scoped; missing mappings are
// lifecycle errors and must not be converted into session-scoped shell state.
func (m *Manager) ResolveTaskEnvironmentID(ctx context.Context, sessionID string) (string, error) {
if sessionID == "" {
return "", fmt.Errorf("session_id is required")
}
if exec, ok := m.executionStore.GetBySessionID(sessionID); ok {
if exec.TaskEnvironmentID != "" {
return exec.TaskEnvironmentID, nil
}
return "", fmt.Errorf("session %s has no task environment ID", sessionID)
}
if m.workspaceInfoProvider == nil {
return "", fmt.Errorf("workspace info provider not configured")
}
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, "", sessionID)
if err != nil {
return "", fmt.Errorf("failed to resolve task environment for session %s: %w", sessionID, err)
}
if info == nil || info.TaskEnvironmentID == "" {
return "", fmt.Errorf("session %s has no task environment ID", sessionID)
}
return info.TaskEnvironmentID, nil
}
// IsRemoteSession checks whether a session is associated with a remote executor
// (e.g., sprites). It first checks the in-memory execution store, then falls back
// to the database via WorkspaceInfoProvider. This is useful when the execution
// hasn't been recreated yet after a backend restart.
func (m *Manager) IsRemoteSession(ctx context.Context, sessionID string) bool {
// Check in-memory execution first (fast path).
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
if execution.RuntimeName == executor.NameSprites {
return true
}
return execution.metadataBool(MetadataKeyIsRemote)
}
// Fall back to database records (post-restart, execution not yet recreated).
if m.workspaceInfoProvider == nil {
return false
}
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, "", sessionID)
if err != nil || info == nil {
return false
}
if models.IsRemoteExecutorType(models.ExecutorType(info.ExecutorType)) {
return true
}
// Backwards compatibility: old records may only have RuntimeName set.
return info.RuntimeName == executor.NameSprites || info.RuntimeName == executor.NameRemoteDocker
}
// ShouldUseContainerShell checks whether a session's shell should run inside a container/sandbox
// (via agentctl) rather than on the host. This is true for Docker, Sprites, and remote executors.
// It first checks the in-memory execution store, then falls back to the database.
func (m *Manager) ShouldUseContainerShell(ctx context.Context, sessionID string) bool {
// Check in-memory execution first (fast path).
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
// Docker and Sprites executors run shells inside the container/sandbox
if execution.RuntimeName == executor.NameDocker ||
execution.RuntimeName == executor.NameSprites {
return true
}
return execution.metadataBool(MetadataKeyIsRemote)
}
// Fall back to database records (post-restart, execution not yet recreated).
if m.workspaceInfoProvider == nil {
return false
}
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, "", sessionID)
if err != nil || info == nil {
return false
}
if models.IsContainerizedExecutorType(models.ExecutorType(info.ExecutorType)) {
return true
}
// Backwards compatibility: old records may only have RuntimeName set.
return info.RuntimeName == executor.NameDocker ||
info.RuntimeName == executor.NameSprites ||
info.RuntimeName == executor.NameRemoteDocker
}
// GetAvailableCommandsForSession returns the available slash commands for a session.
// Returns nil if the session doesn't exist or has no commands stored.
//
// Thread-safe: Can be called concurrently from multiple goroutines.
func (m *Manager) GetAvailableCommandsForSession(sessionID string) []streams.AvailableCommand {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil
}
return execution.GetAvailableCommands()
}
// GetModeStateForSession returns the cached session mode state.
// Returns nil if the session has no execution or no mode state cached.
func (m *Manager) GetModeStateForSession(sessionID string) *CachedModeState {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil
}
return execution.GetModeState()
}
// GetModelStateForSession returns the cached session model state.
// Returns nil if the session has no execution or no model state cached.
func (m *Manager) GetModelStateForSession(sessionID string) *CachedModelState {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil
}
return execution.GetModelState()
}
// ListExecutions returns all currently tracked agent executions.
//
// Returns a snapshot of all executions in memory at the time of call. The returned slice
// contains pointers to execution objects that may be modified by other goroutines after
// this method returns. Do not modify execution state directly - use Manager methods instead.
//
// The list includes executions in all states:
// - Starting (process launching, agentctl initializing)
// - Running (actively processing prompts)
// - Ready (waiting for user input)
// - Completed/Failed (finished but not yet removed)
//
// Thread-safe: Can be called concurrently. Returns a new slice on each call.
//
// Typical usage: Status endpoints, debugging, cleanup loops.
func (m *Manager) ListExecutions() []*AgentExecution {
return m.executionStore.List()
}
const agentStartupLivenessGrace = 75 * time.Second
// ProbeAgentRunningForSession checks if an agent process is running or
// starting for a session and reports probe failures to the caller.
//
// For passthrough sessions (direct PTY mode), it checks whether the PTY process is alive
// in the InteractiveRunner. For ACP sessions, it probes agentctl's status endpoint.
//
// A successful probe returns true if:
// - Passthrough process is alive in the InteractiveRunner
// - Agent status is "running" (actively processing prompts)
// - Agent status is "starting" (process launched but not yet ready)
//
// A successful probe returns false if:
// - No execution exists for this session
// - Passthrough process ID is set but process is not alive
// - Agent is in any other state (stopped, failed, etc.)
//
// An unavailable agentctl client, interactive runner, or status endpoint is
// returned as an error rather than as a false result.
func (m *Manager) ProbeAgentRunningForSession(ctx context.Context, sessionID string) (bool, error) {
// First check if we have an execution tracked for this session
execution, exists := m.GetExecutionBySessionID(sessionID)
if !exists {
return false, nil
}
// Launch registers the execution before agentctl is guaranteed to answer
// status requests. During that bounded boot window, a failed status probe
// means "not ready yet", not "stale". Treat the execution-store STARTING
// state as live long enough to cover StartAgentProcess's 60-second agentctl
// readiness deadline; after the grace expires, normal probing allows stale
// starting entries to be reaped.
if execution.Status == v1.AgentStatusStarting &&
!execution.StartedAt.IsZero() &&
time.Since(execution.StartedAt) <= agentStartupLivenessGrace {
return true, nil
}
// Passthrough sessions run as direct PTY processes via InteractiveRunner,
// bypassing agentctl's ACP protocol. Check the process directly.
if execution.PassthroughProcessID != "" {
if runner := m.GetInteractiveRunner(); runner != nil {
return runner.IsProcessReadyOrPending(execution.PassthroughProcessID), nil
}
return false, fmt.Errorf("interactive runner is unavailable")
}
// Probe agentctl status to verify the agent process is running
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return false, fmt.Errorf("agentctl client is unavailable")
}
status, err := client.GetStatus(ctx)
if err != nil {
m.logger.Debug("failed to get agentctl status",
zap.String("session_id", sessionID),
zap.Error(err))
return false, err
}
return status.IsAgentRunning(), nil
}
// IsAgentRunningForSession preserves the legacy boolean probe for callers
// that do not need to distinguish a dead process from an unavailable probe.
func (m *Manager) IsAgentRunningForSession(ctx context.Context, sessionID string) bool {
running, _ := m.ProbeAgentRunningForSession(ctx, sessionID)
return running
}
// IsAgentReadyForPrompt reports whether the session can accept an ACP prompt
// immediately. A process can be "running" while its update stream is still
// reconnecting after resume; PromptAgent needs the stream-backed request path.
func (m *Manager) IsAgentReadyForPrompt(ctx context.Context, sessionID string) bool {
execution, exists := m.GetExecutionBySessionID(sessionID)
if !exists {
return false
}
if execution.PassthroughProcessID != "" || execution.IsPassthrough {
return m.IsAgentRunningForSession(ctx, sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if execution.Status != v1.AgentStatusReady || client == nil {
return false
}
if !execution.isSessionInitialized() || execution.ACPSessionID == "" {
return false
}
return client.HasAgentStream()
}
func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string) error {
execution, exists := m.GetExecutionBySessionID(sessionID)
if !exists {
return fmt.Errorf("session %q has no execution: %w", sessionID, ErrExecutionNotFound)
}
client, releaseClient := execution.AcquireAgentCtlClient()
if execution.PassthroughProcessID != "" || execution.IsPassthrough || client == nil {
releaseClient()
return nil
}
// InitializeAndPrompt owns the first updates stream. Starting a recovery
// stream before ACP initialization finishes creates competing consumers and
// can split one prompt's events across them.
if !execution.isSessionInitialized() || execution.ACPSessionID == "" {
releaseClient()
return nil
}
if client.HasAgentStream() {
releaseClient()
return nil
}
releaseClient()
if m.streamManager == nil {
return fmt.Errorf("stream manager is not configured")
}
ready := make(chan struct{})
// Prompt recovery is reached through the per-session prompt path. Avoid a
// broader reconnect registry here; HasAgentStream above covers steady state.
m.streamManager.connectUpdatesStreamAsync(execution, ready)
select {
case <-ready:
case <-ctx.Done():
return ctx.Err()
}
client, releaseClient = execution.AcquireAgentCtlClient()
if client == nil {
return fmt.Errorf("agentctl client is unavailable")
}
hasAgentStream := client.HasAgentStream()
releaseClient()
if !hasAgentStream {
return fmt.Errorf("agent stream not connected")
}
if execution.Status == v1.AgentStatusFailed && execution.isSessionInitialized() && execution.ACPSessionID != "" {
return m.restoreRecoveredFailedExecution(ctx, execution)
}
return nil
}
func (m *Manager) restoreRecoveredFailedExecution(ctx context.Context, execution *AgentExecution) error {
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("agentctl client is unavailable")
}
status, err := client.GetStatus(ctx)
if err != nil {
return fmt.Errorf("failed to verify agent status after stream recovery: %w", err)
}
if !status.IsAgentRunning() {
return fmt.Errorf("agent process is not running after stream recovery: %s", status.AgentStatus)
}
if err := m.markBootReadyFromFailed(ctx, execution.ID); err != nil {
return err
}
return nil
}
// UpdateStatus updates the status of an execution
func (m *Manager) UpdateStatus(executionID string, status v1.AgentStatus) error {
return m.updateStatusAndPersist(context.Background(), executionID, status)
}
func (m *Manager) updateStatusAndPersist(ctx context.Context, executionID string, status v1.AgentStatus) error {
var updated *AgentExecution
if err := m.executionStore.WithLock(executionID, func(execution *AgentExecution) {
execution.Status = status
updated = execution
}); err != nil {
if errors.Is(err, ErrExecutionNotFound) {
return fmt.Errorf("execution %q not found", executionID)
}
return err
}
m.logger.Debug("updated execution status",
zap.String("execution_id", executionID),
zap.String("status", string(status)))
if updated != nil {
m.persistExecutorRunning(context.WithoutCancel(ctx), updated)
}
return nil
}
// BeginPrompt advances prompt ownership and marks the execution running before dispatch.
func (m *Manager) BeginPrompt(executionID string) (uint64, error) {
generation, err := m.executionStore.BeginPrompt(executionID)
if err != nil {
if errors.Is(err, ErrExecutionNotFound) {
return 0, fmt.Errorf("execution %q not found", executionID)
}
return 0, err
}
if updated, exists := m.executionStore.Get(executionID); exists {
m.persistExecutorRunning(context.Background(), updated)
}
return generation, nil
}
// OwnsPromptGeneration reports whether a ready event's immutable execution and
// prompt generation still identify the lifecycle prompt active for sessionID.
func (m *Manager) OwnsPromptGeneration(sessionID, executionID string, generation uint64) bool {
return m.executionStore.OwnsPromptGeneration(sessionID, executionID, generation)
}
// OwnsPromptActivity reports whether a stall snapshot still belongs to the
// active prompt and no genuine event arrived after the snapshot was captured.
func (m *Manager) OwnsPromptActivity(
sessionID, executionID string,
generation, activityEpoch uint64,
) bool {
return m.executionStore.OwnsPromptActivity(sessionID, executionID, generation, activityEpoch)
}
// GetPromptGenerationForSession returns the generation currently owned by the
// active prompt for sessionID. Orchestrator cancellation uses this together
// with the execution and turn IDs to reject stale terminal events while the
// lifecycle manager waits for an acknowledged cancel to settle.
func (m *Manager) GetPromptGenerationForSession(_ context.Context, sessionID string) (uint64, error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return 0, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
}
return execution.promptGenerationSnapshot(), nil
}
// GetPromptActivityForSession returns the execution ID, prompt generation,
// activity epoch, and last-activity timestamp currently owned by sessionID's
// active prompt. Unlike OwnsPromptGeneration/OwnsPromptActivity (which check
// a value someone else already captured), this is the capture step itself —
// for watchdogs that scan for stuck sessions and must snapshot "is anything
// happening right now" on their own, rather than waiting for a stall event to
// carry the values in its payload. lastActivityAt in particular is what lets
// a caller gate on real elapsed inactivity (time.Since(lastActivityAt))
// instead of an epoch comparison across its own scan window, which only
// catches activity that lands during that window and sails through a live
// agent that is simply between events. Returns ErrNoExecutionForSession
// (wrapped) when no execution is tracked for the session.
func (m *Manager) GetPromptActivityForSession(_ context.Context, sessionID string) (executionID string, generation, activityEpoch uint64, lastActivityAt time.Time, err error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return "", 0, 0, time.Time{}, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
}
lastActivityAt, _, activityEpoch = execution.promptActivitySnapshot()
return execution.ID, execution.promptGenerationSnapshot(), activityEpoch, lastActivityAt, nil
}
// MarkReady marks an execution as ready for follow-up prompts AFTER A TURN.
// Use MarkBootReady instead when the agent has just initialized and hasn't yet
// processed a turn — orchestrator subscribers rely on the distinction.
//
// This transitions the execution to the "ready" state, indicating the agent has finished
// processing the current prompt and is waiting for user input. Called when:
// - Agent finishes processing a prompt (via stream completion event)
// - User cancels an agent turn (to allow new prompts)
//
// State Machine Transitions:
//
// Running -> Ready (after prompt completion)
// Any -> Ready (after cancel)
//
// Publishes events.AgentReady. Returns error if execution not found.
func (m *Manager) MarkReady(executionID string) error {
return m.markReadyEvent(executionID, events.AgentReady)
}
// MarkBootReady marks a freshly-initialized execution as ready for its first
// prompt. Distinct from MarkReady so the orchestrator can disambiguate boot
// signals from turn-end signals without race-prone flag tracking. Use this
// from session/init paths and post-context-reset; use MarkReady from the
// turn-completion path.
//
// State Machine Transitions:
//
// Starting -> Ready (after initialization)
//
// Publishes events.AgentBootReady. Returns error if execution not found.
func (m *Manager) MarkBootReady(executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
err := m.markBootReadyForStartup(
context.Background(), executionID, execution.startupAttemptSnapshot(),
)
return err
}
func (m *Manager) markBootReadyForStartup(
ctx context.Context,
executionID string,
startupGeneration uint64,
) error {
err := m.markReadyEventWithStartupGeneration(
ctx, executionID, events.AgentBootReady, false, startupGeneration,
func(execution *AgentExecution) {
m.finalWorkspaceRefresh(execution, "startup_grace")
},
func(execution *AgentExecution) {
m.setRuntimeInterest(execution.SessionID, false)
m.releaseActivity(executionActivityKey(execution.ID))
},
)
return err
}
// markReadyEvent is the shared body of MarkReady / MarkBootReady — both flip
// the execution to the Ready status and publish their respective event type.
func (m *Manager) markReadyEvent(executionID, eventType string) error {
return m.markReadyEventWithContext(context.Background(), executionID, eventType, false)
}
func (m *Manager) markBootReadyFromFailed(ctx context.Context, executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
if execution.Status != v1.AgentStatusFailed {
return nil
}
startupGeneration := execution.startupAttemptSnapshot()
if attemptID := ResumeAttemptIDFromContext(ctx); attemptID != "" {
var found bool
startupGeneration, found = execution.startupGenerationForAttemptID(attemptID)
if !found {
return fmt.Errorf("execution %q has no startup generation for resume attempt %q", executionID, attemptID)
}
}
return m.markReadyEventWithStartupGeneration(
ctx, executionID, events.AgentBootReady, false, startupGeneration, nil, nil,
)
}
// markReadyEventWithContext flips executionID to Ready and publishes
// eventType. When asyncPublish is false (MarkReady, MarkBootReady, and
// markBootReadyFromFailed all pass false — unchanged behavior), the publish
// happens inline before this returns, matching the event bus's synchronous,
// order-preserving delivery to registered subscribers.
//
// When asyncPublish is true (escalateStuckCancel only — see its own doc
// comment), the publish is dispatched on its own goroutine instead. That
// caller is reached from Service.CancelAgent (and cancelAgentSilent's other
// guard-holding callers) while still holding sessionID's per-session
// cancelInFlightGuard: the in-memory event bus delivers to a queue
// subscription's handler *synchronously*, on the publishing goroutine, and
// the orchestrator's handleAgentReady is registered on exactly such a queue
// subscription for this event type. An inline publish here would have
// handleAgentReady try to re-acquire that same guard from the very same
// goroutine that is already holding it — sync.Mutex is not reentrant, so
// that blocks forever. Deferring the publish lets the guard-holding caller
// finish (and release the guard) first. The detached publish receives the
// immutable payload captured while Ready was set, so handleAgentReady can
// reject it if another prompt generation starts before delivery.
func (m *Manager) markReadyEventWithContext(ctx context.Context, executionID, eventType string, asyncPublish bool) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
return m.markReadyEventWithStartupGeneration(
ctx, executionID, eventType, asyncPublish, execution.startupAttemptSnapshot(), nil, nil,
)
}
func (m *Manager) markReadyEventWithStartupGeneration(
ctx context.Context,
executionID, eventType string,
asyncPublish bool,
startupGeneration uint64,
before func(*AgentExecution),
after func(*AgentExecution),
) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
var readyErr error
accepted := execution.withStartupAttempt(startupGeneration, func(attemptID string) {
if before != nil {
before(execution)
}
readyErr = m.markReadyEventForExecution(
ctx, execution, eventType, asyncPublish, attemptID,
)
if readyErr == nil && after != nil {
after(execution)
}
})
if !accepted {
return fmt.Errorf("execution %q startup generation %d is stale", executionID, startupGeneration)
}
return readyErr
}
func (m *Manager) markReadyEventForExecution(
ctx context.Context,
execution *AgentExecution,
eventType string,
asyncPublish bool,
attemptID string,
) error {
executionID := execution.ID
var payload AgentEventPayload
var updated *AgentExecution
var alreadyReady bool
if err := m.executionStore.WithLock(executionID, func(execution *AgentExecution) {
if execution.Status == v1.AgentStatusReady {
alreadyReady = true
return
}
execution.Status = v1.AgentStatusReady
payload = newAgentEventPayload(execution)
payload.AttemptID = attemptID
updated = execution
}); err != nil {
if errors.Is(err, ErrExecutionNotFound) {
return fmt.Errorf("execution %q not found", executionID)
}
return err
}
if alreadyReady {
return nil
}
m.persistExecutorRunning(context.WithoutCancel(ctx), updated)
m.logger.Info("execution ready",
zap.String("execution_id", executionID),
zap.String("event_type", eventType))
if asyncPublish {
go m.eventPublisher.publishAgentEventPayload(context.Background(), eventType, payload)
} else {
m.eventPublisher.publishAgentEventPayload(ctx, eventType, payload)
}
return nil
}
// MarkCompleted marks an execution as completed or failed.
//
// This is called when the agent process terminates, either successfully or with an error.
// The final status is determined by exit code and error message:
//
// - exitCode == 0 && errorMessage == "" → AgentStatusCompleted (success)
// - Otherwise → AgentStatusFailed (failure)
//
// Parameters:
// - executionID: The execution to mark as completed
// - exitCode: Process exit code (0 = success, non-zero = failure)
// - errorMessage: Human-readable error description (empty string if no error)
//
// State Machine:
//
// This is a terminal state transition - no further state changes are expected after this.
// Typical flow: Starting -> Running -> Ready -> ... -> Completed/Failed
//
// Publishes either AgentCompleted or AgentFailed event depending on final status.
//
// Returns error if execution not found.
func (m *Manager) MarkCompleted(executionID string, exitCode int, errorMessage string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
return m.markCompletedWithTurnID(
executionID,
exitCode,
errorMessage,
execution.promptTurnIDSnapshot(),
nil,
)
}
func (m *Manager) markCompletedWithTurnID(
executionID string,
exitCode int,
errorMessage, turnID string,
failureEvidence *PromptAttemptEvidence,
) error {
return m.markCompletedWithTurnIDAndAttempt(
executionID, exitCode, errorMessage, turnID, failureEvidence, "",
)
}
func (m *Manager) markCompletedWithTurnIDAndAttempt(
executionID string,
exitCode int,
errorMessage, turnID string,
failureEvidence *PromptAttemptEvidence,
attemptID string,
) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found", executionID)
}
// Guard against duplicate completion (e.g. ACP prompt error + process exit error).
// MarkCompleted is a terminal transition — once in Completed/Failed/Stopped, skip
// re-publishing. Stopped is a terminal state too (shutdown-race abort, StopAgent),
// so a second completion must not overwrite it or re-emit an event.
if isTerminalStatus(execution.Status) {
m.logger.Warn("ignoring duplicate MarkCompleted for already-terminal execution",
zap.String("execution_id", executionID),
zap.String("current_status", string(execution.Status)),
zap.Int("exit_code", exitCode))
return nil
}
// A turn aborted because backend graceful shutdown killed the agent
// subprocess is not an agent failure. Treat the terminal error as a benign
// stop so the session stays resumable and the UI shows no red error banner.
if (exitCode != 0 || errorMessage != "") && m.IsShuttingDown() {
return m.markStoppedDuringShutdown(execution, exitCode, errorMessage, turnID)
}
if (exitCode != 0 || errorMessage != "") && isUninitializedStartupExecution(execution) {
m.logger.Debug("deferring uninitialized startup process exit to startup owner",
zap.String("execution_id", execution.ID),
zap.Int("exit_code", exitCode))
return nil
}
failureEvidence = ensureCompletionFailureEvidence(execution, exitCode, errorMessage, failureEvidence)
_ = m.executionStore.WithLock(executionID, func(exec *AgentExecution) {
now := time.Now()
exec.FinishedAt = &now
exec.ExitCode = &exitCode
exec.ErrorMessage = errorMessage
if exitCode == 0 && errorMessage == "" {
exec.Status = v1.AgentStatusCompleted
} else {
exec.Status = v1.AgentStatusFailed
}
})
// End session trace span
execution.EndSessionSpan()
// Persist the terminal status to executors_running. Unlike the Ready/Running
// transitions (which flow through updateStatusAndPersist), MarkCompleted is
// the process-exit/crash boundary and historically skipped persistence — so a
// row kept claiming a `running`/`starting` process after it had exited
// (#1597). Re-stamping here leaves the row truthful (terminal
// status + fresh last_seen_at) the moment the process is gone.
m.persistExecutorRunning(context.Background(), execution)
m.releaseActivity(executionActivityKey(executionID))
m.logger.Info("execution completed",
zap.String("execution_id", executionID),
zap.Int("exit_code", exitCode),
zap.String("status", string(execution.Status)))
// Publish completion event
eventType := events.AgentCompleted
if execution.Status == v1.AgentStatusFailed {
eventType = events.AgentFailed
m.classifyAndMaybeRemediate(execution, exitCode, errorMessage)
}
if eventType == events.AgentFailed {
m.eventPublisher.publishAgentEventWithTurnIDAndEvidence(
WithResumeAttemptID(context.Background(), attemptID), eventType, execution, turnID, failureEvidence,
)
return nil
}
m.eventPublisher.publishAgentEventWithTurnID(
WithResumeAttemptID(context.Background(), attemptID), eventType, execution, turnID,
)
return nil
}
func ensureCompletionFailureEvidence(
execution *AgentExecution,
exitCode int,
errorMessage string,
failureEvidence *PromptAttemptEvidence,
) *PromptAttemptEvidence {
if (exitCode == 0 && errorMessage == "") || failureEvidence != nil {
return failureEvidence
}
evidence := execution.promptAttemptEvidenceSnapshot()
return &evidence
}
// isTerminalStatus reports whether a status is a final execution state that
// MarkCompleted / markStoppedDuringShutdown must not overwrite or re-publish.
func isTerminalStatus(status v1.AgentStatus) bool {
return status == v1.AgentStatusCompleted ||
status == v1.AgentStatusFailed ||
status == v1.AgentStatusStopped
}
// markStoppedDuringShutdown records a terminal error completion that arrived
// while backend graceful shutdown was in progress as a benign STOPPED outcome
// instead of a FAILED one. It stamps the terminal fields, runs the same teardown
// as the failed branch, and publishes events.AgentStopped — mirroring the
// StopReasonBackendShutdown teardown so the orchestrator treats it as resumable.
// It deliberately does not remove the execution or run classifyAndMaybeRemediate.
//
// Both terminal chokepoints (handleCompleteEventMarkState and MarkCompleted) can
// reach this during shutdown for the same execution (ACP error event + process
// exit). The transition is therefore guarded under the store lock: if the
// execution is already terminal, or was removed by StopAgentWithReason, this is a
// stale/duplicate callback — skip teardown and the AgentStopped publish so the
// orchestrator does not process the same stop twice.
func (m *Manager) markStoppedDuringShutdown(
execution *AgentExecution, exitCode int, errorMessage string, turnIDs ...string,
) error {
applied := false
err := m.executionStore.WithLock(execution.ID, func(exec *AgentExecution) {
if isTerminalStatus(exec.Status) {
return
}
now := time.Now()
exec.FinishedAt = &now
exec.ExitCode = &exitCode
exec.ErrorMessage = errorMessage
exec.Status = v1.AgentStatusStopped
applied = true
})
if err != nil || !applied {
m.logger.Debug("ignoring stale/duplicate shutdown completion",
zap.String("execution_id", execution.ID),
zap.String("current_status", string(execution.Status)),
zap.Error(err))
return nil
}
m.logger.Warn("error completion during shutdown, treating as cancellation",
zap.String("execution_id", execution.ID),
zap.String("task_id", execution.TaskID),
zap.Int("exit_code", exitCode),
zap.String("error", errorMessage))
execution.EndSessionSpan()
m.persistExecutorRunning(context.Background(), execution)
m.releaseActivity(executionActivityKey(execution.ID))
turnID := ""
if len(turnIDs) > 0 {
turnID = turnIDs[0]
}
m.eventPublisher.publishAgentEventWithTurnID(context.Background(), events.AgentStopped, execution, turnID)
return nil
}
// classifyAndMaybeRemediate runs routingerr.Classify at the failure
// boundary, logs the structured result, and - for codes that carry a
// safe remediation (currently CodeNpxCacheCorrupted) - fires a
// best-effort cleanup goroutine so the next launch attempt (Office
// scheduler retry or Kanban "Resume") finds a clean environment.
//
// The remediation hook is injectable via Manager.remediateNpxCache;
// production wiring points it at routingerr.RemediateNpxCache. Tests
// stub it to avoid touching the real filesystem.
func (m *Manager) classifyAndMaybeRemediate(execution *AgentExecution, exitCode int, errorMessage string) {
phase := routingerr.PhaseSessionInit
if execution.isSessionInitialized() {
phase = routingerr.PhasePromptSend
}
var exitPtr *int
if exitCode != 0 {
ec := exitCode
exitPtr = &ec
}
e := routingerr.Classify(routingerr.Input{
Phase: phase,
ProviderID: execution.AgentID,
ExitCode: exitPtr,
Stderr: errorMessage,
})
if e == nil {
return
}
m.logger.Info("agent failure classified for provider routing",
zap.String("execution_id", execution.ID),
zap.String("task_id", execution.TaskID),
zap.String("session_id", execution.SessionID),
zap.String("provider_id", execution.AgentID),
zap.String("phase", string(e.Phase)),
zap.String("routing_code", string(e.Code)),
zap.String("confidence", string(e.Confidence)),
zap.String("classifier_rule", e.ClassifierRule),
zap.Bool("fallback_allowed", e.FallbackAllowed),
zap.Bool("auto_retryable", e.AutoRetryable),
zap.Bool("user_action", e.UserAction),
zap.String("remediation_path", e.RemediationPath))
if e.Code != routingerr.CodeNpxCacheCorrupted || e.RemediationPath == "" {
return
}
remediate := m.remediateNpxCache
path := e.RemediationPath
execID := execution.ID
zapLog := m.logger.Zap().With(
zap.String("execution_id", execID),
zap.String("path", path),
)
go func() {
if err := remediate(path, zapLog); err != nil {
m.logger.Warn("npx cache remediation failed",
zap.String("execution_id", execID),
zap.String("path", path),
zap.Error(err))
}
}()
}
// RespondToPermission sends a response to an agent's permission request.
//
// When an agent requests permission (e.g., to run a bash command, modify files, etc.),
// it pauses execution and waits for user approval. This method sends the user's response.
//
// Parameters:
// - executionID: The agent execution waiting for permission
// - pendingID: Unique ID of the permission request (from permission request event)
// - optionID: The user-selected option ID (from the permission request's options array)
// - cancelled: If true, indicates user cancelled/rejected the permission request.
// When cancelled=true, optionID is ignored.
//
// Response Semantics:
// - cancelled=false, optionID="approve" → User approved the action
// - cancelled=false, optionID="deny" → User explicitly denied the action
// - cancelled=true, optionID="" → User cancelled/closed the dialog
//
// After receiving the response, the agent will either:
// - Continue executing (if approved)
// - Skip the action and report failure (if denied/cancelled)
//
// Timeout: 30 seconds for agentctl to acknowledge the response.
//
// Returns error if execution not found, agentctl unavailable, or communication fails.
func (m *Manager) RespondToPermission(executionID, pendingID, optionID string, cancelled bool) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("agent execution not found: %s", executionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return fmt.Errorf("agent execution has no agentctl client: %s", executionID)
}
m.logger.Info("responding to permission request",
zap.String("execution_id", executionID),
zap.String("pending_id", pendingID),
zap.String("option_id", optionID),
zap.Bool("cancelled", cancelled))
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
return client.RespondToPermission(ctx, pendingID, optionID, cancelled)
}
// RespondToPermissionBySessionID sends a response to a permission request using session ID.
//
// Convenience method that looks up the execution by session ID and delegates to RespondToPermission.
// See RespondToPermission for parameter semantics and behavior.
func (m *Manager) RespondToPermissionBySessionID(sessionID, pendingID, optionID string, cancelled bool) error {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return fmt.Errorf("no agent execution found for session: %s", sessionID)
}
return m.RespondToPermission(execution.ID, pendingID, optionID, cancelled)
}
// ListPendingPermissionsBySessionID reads safe live snapshots from the current
// execution for a task session. It never starts or resumes an execution.
func (m *Manager) ListPendingPermissionsBySessionID(ctx context.Context, sessionID string) ([]streams.PendingAgentPermission, error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID)
}
requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return client.ListPendingPermissions(requestCtx)
}
// ResolvePermissionBySessionID selects one exact option on one exact request
// generation in the current execution.
func (m *Manager) ResolvePermissionBySessionID(ctx context.Context, sessionID, requestID, pendingID, optionID string) (*streams.PermissionResolveResponse, error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID)
}
requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return client.ResolvePermission(requestCtx, requestID, pendingID, optionID)
}
func (m *Manager) CancelPermissionBySessionID(ctx context.Context, sessionID, requestID, pendingID string) (*streams.PermissionCancelResponse, error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID)
}
requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return client.CancelPermission(requestCtx, requestID, pendingID)
}
// ProbeBackgroundWorkloadsBySessionID samples the agent execution owning
// sessionID for background-workload liveness (spec
// docs/specs/disambiguate-waiting/spec.md, §"Probe transport"). Unlike
// RespondToPermission, no timeout is applied here — the caller wraps ctx
// with the KANDEV_PARKED_PROBE_BUDGET timeout (D2) before calling this.
func (m *Manager) ProbeBackgroundWorkloadsBySessionID(ctx context.Context, sessionID string) (agentctlclient.ProbeResult, error) {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return agentctlclient.ProbeResultUnknown, fmt.Errorf("no agent execution found for session: %s", sessionID)
}
client, releaseClient := execution.AcquireAgentCtlClient()
defer releaseClient()
if client == nil {
return agentctlclient.ProbeResultUnknown, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID)
}
return client.ProbeBackgroundWorkloads(ctx, sessionID)
}
// stopAgentViaBackend stops the agent execution via the runtime that created it.
func (m *Manager) stopAgentViaBackend(ctx context.Context, executionID string, execution *AgentExecution, reason string, force bool, agentStopFailed bool) error {
if execution.RuntimeName == "" || m.executorRegistry == nil {
return nil
}
rt, err := m.executorRegistry.GetBackend(execution.RuntimeName)
if err != nil {
m.logger.Warn("failed to get runtime for stopping execution",
zap.String("execution_id", executionID),
zap.Stringer("runtime", execution.RuntimeName),
zap.Error(err))
return fmt.Errorf("get runtime %s: %w", execution.RuntimeName, err)
}
m.stopPassthroughProcess(ctx, executionID, execution, rt)
runtimeInstance := &ExecutorInstance{
InstanceID: execution.ID,
TaskID: execution.TaskID,
SessionID: execution.SessionID,
ContainerID: execution.ContainerID,
StandaloneInstanceID: execution.standaloneInstanceID,
StandalonePort: execution.standalonePort,
Metadata: execution.MetadataSnapshot(),
StopReason: reason,
AgentStopFailed: agentStopFailed,
}
if execution.RuntimeName == executor.NameKubernetes {
metadata, resolveErr := m.currentKubernetesConnectionMetadata(ctx, runtimeInstance.Metadata)
if resolveErr != nil {
return fmt.Errorf("resolve current Kubernetes cleanup connection: %w", resolveErr)
}
runtimeInstance.Metadata = metadata
}
if err := rt.StopInstance(ctx, runtimeInstance, force); err != nil {
// During shutdown the runtime instance may already be stopping or
// absent. Only surface this at WARN outside shutdown.
if m.IsShuttingDown() {
m.logger.Debug("failed to stop runtime instance",
zap.String("execution_id", executionID),
zap.Error(err))
} else {
m.logger.Warn("failed to stop runtime instance",
zap.String("execution_id", executionID),
zap.Error(err))
}
return err
}
return nil
}
// stopPassthroughProcess stops the passthrough interactive process if one is running.
func (m *Manager) stopPassthroughProcess(ctx context.Context, executionID string, execution *AgentExecution, rt ExecutorBackend) {
if execution.PassthroughProcessID == "" {
return
}
interactiveRunner := rt.GetInteractiveRunner()
if interactiveRunner == nil {
return
}
if err := interactiveRunner.Stop(ctx, execution.PassthroughProcessID); err != nil {
m.logger.Warn("failed to stop passthrough process",
zap.String("execution_id", executionID),
zap.String("process_id", execution.PassthroughProcessID),
zap.Error(err))
return
}
m.logger.Info("passthrough process stopped",
zap.String("execution_id", executionID),
zap.String("process_id", execution.PassthroughProcessID))
}
// buildFreshAgentCommand rebuilds the agent command without resume flags by going through
// the standard command-building pipeline with an empty SessionID. This works for all
// agent types because each agent's BuildCommand respects SessionID="" to skip resume flags.
//
// It fails closed: if the profile cannot be resolved, or a configured
// command_prefix cannot be tokenised, it returns an error so a context reset
// never relaunches a configured agent without its wrapper.
func (m *Manager) buildFreshAgentCommand(ctx context.Context, execution *AgentExecution, agentConfig agents.Agent) (agentCommands, error) {
var profileInfo *AgentProfileInfo
if execution.AgentProfileID != "" && m.profileResolver != nil {
pi, resolveErr := m.profileResolver.ResolveProfile(ctx, execution.AgentProfileID)
if resolveErr != nil {
// A profile was expected but could not be resolved. We cannot tell
// whether it carried a sandbox prefix, so refuse to relaunch rather
// than risk running unwrapped.
return agentCommands{}, fmt.Errorf("resolve profile %s for restart: %w", execution.AgentProfileID, resolveErr)
}
profileInfo = pi
}
model := ""
autoApprove := false
permissionValues := make(map[string]bool)
if profileInfo != nil {
model = profileInfo.Model
autoApprove = profileInfo.AutoApprove
permissionValues[agents.PermissionKeyAutoApprove] = profileInfo.AutoApprove
permissionValues["allow_indexing"] = profileInfo.AllowIndexing
permissionValues["dangerously_skip_permissions"] = profileInfo.DangerouslySkipPermissions
}
if override := execution.metadataString(MetadataKeyModelOverride); override != "" {
model = override
}
// Preserve the profile's cli_flags and command_prefix across restarts. A
// context reset that dropped the launcher prefix would relaunch a
// configured agent unwrapped.
cliFlagTokens, commandPrefixTokens, err := m.resolveProfileLaunchTokens(profileInfo)
if err != nil {
return agentCommands{}, err
}
managedRuntimeVersion, err := m.resolveManagedRuntimeVersion(ctx, execution.RuntimeName, agentConfig)
if err != nil {
return agentCommands{}, err
}
opts := agents.CommandOptions{
Model: model,
SessionID: "", // Fresh start — no resume flags
AutoApprove: autoApprove,
PermissionValues: permissionValues,
CLIFlagTokens: cliFlagTokens,
CommandPrefixTokens: commandPrefixTokens,
// Runtime is "standalone" / "docker" / "sprites" — MockAgent
// reads this to pick a bare name (container PATH lookup) vs.
// an absolute host path.
Runtime: execution.RuntimeName,
ManagedRuntimeVersion: managedRuntimeVersion,
}
args := m.commandBuilder.BuildCommandArgs(agentConfig, opts)
continueArgs := m.commandBuilder.BuildContinueCommandArgs(agentConfig, opts)
if err := validateBuiltAgentCommands(args, continueArgs); err != nil {
return agentCommands{}, err
}
return newAgentCommands(args, continueArgs), nil
}