Files
kandev/apps/backend/internal/orchestrator/dynamic_launch.go
T

1328 lines
48 KiB
Go

package orchestrator
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/kandev/kandev/internal/agent/planinjection"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
dynamicruntime "github.com/kandev/kandev/internal/agent/runtime/dynamic"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
"github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/orchestrator/watcher"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
"go.uber.org/zap"
)
const dynamicRouteStatusWaiting = "waiting"
const dynamicRouteStatusActive = "active"
const dynamicRouteStatusActionRequired = "action_required"
// dynamicSuccessorDetachedTimeout bounds one detached fallback launch: the
// predecessor stop plus the successor launch. Without it the repository calls
// inside the launch could hold the session's cancel guard indefinitely and
// block every later stop or message for that session.
const dynamicSuccessorDetachedTimeout = 2 * time.Minute
// dynamicTaskDownstream adapts the task executor to the provider-neutral
// conductor. The callback updates the task-session attribution before every
// concrete launch, including a cross-profile fallback.
type dynamicTaskDownstream struct {
service *Service
task *v1.Task
sessionID string
options executor.LaunchOptions
execution *executor.TaskExecution
}
func (d *dynamicTaskDownstream) Launch(
ctx context.Context,
launch dynamicruntime.DownstreamLaunch,
) (dynamicruntime.DownstreamExecution, error) {
if err := d.service.persistDynamicLaunchDecision(ctx, d.sessionID, launch.Decision); err != nil {
return dynamicruntime.DownstreamExecution{}, err
}
options := d.options
options.AgentProfileID = launch.ExecutionProfileID
options.Prompt = launch.Prompt
options.PriorACPSession = launch.PriorACPSession
if d.task != nil {
if err := d.service.admitCeilingDispatch(ctx, d.task.ID); err != nil {
return dynamicruntime.DownstreamExecution{}, err
}
}
d.service.beginDynamicAttempt(d.sessionID)
taskID := ""
if d.task != nil {
taskID = d.task.ID
}
dispatchCtx, releaseDispatchCommit, err := d.service.commitCeilingEntryDispatch(
ctx, taskID, ceilingEntryBindingFromContext(ctx),
)
if err != nil {
return dynamicruntime.DownstreamExecution{}, err
}
defer releaseDispatchCommit()
execution, err := d.service.executor.LaunchPreparedSession(dispatchCtx, d.task, d.sessionID, options)
if err != nil {
var classified *routingerr.Error
if errors.As(err, &classified) {
return dynamicruntime.DownstreamExecution{}, err
}
classified = routingerr.Classify(routingerr.Input{
Phase: routingerr.PhaseProcessStart,
ProviderID: launch.ExecutionProfileID,
Stderr: err.Error(),
})
// Unknown low-confidence launch failures are workspace/runtime errors,
// not provider failures. Let the ordinary launch recovery own them.
if classified.Confidence == routingerr.ConfLow {
return dynamicruntime.DownstreamExecution{}, err
}
return dynamicruntime.DownstreamExecution{}, fmt.Errorf("%w: %v", classified, err)
}
d.service.bindDynamicAttemptExecution(d.sessionID, execution.AgentExecutionID)
d.service.bindPromptAttemptToExecution(dispatchCtx, d.sessionID, execution.AgentExecutionID)
d.execution = execution
acpSessionID := ""
if session, sessionErr := d.service.repo.GetTaskSession(ctx, d.sessionID); sessionErr == nil && session != nil {
acpSessionID = session.DownstreamACPSessionID
}
if acpSessionID != "" {
if err := d.service.persistDynamicACPSession(ctx, d.sessionID, launch.Decision, acpSessionID); err != nil {
return dynamicruntime.DownstreamExecution{}, err
}
}
return dynamicruntime.DownstreamExecution{
ID: execution.AgentExecutionID,
ExecutionProfileID: launch.ExecutionProfileID,
ACPSessionID: acpSessionID,
}, nil
}
func (d *dynamicTaskDownstream) LoadExecution(
ctx context.Context,
sessionID string,
) (dynamicruntime.DownstreamExecution, bool, error) {
if sessionID == "" {
return dynamicruntime.DownstreamExecution{}, false, nil
}
session, err := d.service.repo.GetTaskSession(ctx, sessionID)
if err != nil {
return dynamicruntime.DownstreamExecution{}, false, err
}
if session == nil || session.AgentExecutionID == "" {
return dynamicruntime.DownstreamExecution{}, false, nil
}
return dynamicruntime.DownstreamExecution{
ID: session.AgentExecutionID,
ExecutionProfileID: session.ExecutionProfileID,
ACPSessionID: session.DownstreamACPSessionID,
}, true, nil
}
func (d *dynamicTaskDownstream) Resume(context.Context, string, string) error {
return errors.New("dynamic task downstream resume is owned by the orchestrator")
}
func (d *dynamicTaskDownstream) Stop(context.Context, string, string) error {
return nil
}
func (s *Service) persistDynamicLaunchDecision(
ctx context.Context,
sessionID string,
decision dynamicruntime.RouteDecision,
) error {
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil {
return err
}
if session.ExecutionProfileID == decision.ExecutionProfileID &&
session.RouteGeneration == decision.Generation &&
decision.Status == "" {
return nil
}
previousExecutionProfileID := session.ExecutionProfileID
session.ExecutionProfileID = decision.ExecutionProfileID
session.RouteGeneration = decision.Generation
session.RouteState = decision.Status
if session.RouteState == "" {
session.RouteState = "starting"
}
session.RouteReason = decision.Reason
applyDynamicRouteDecisionProjection(session, decision)
// The executor marks a failed provider launch terminal before the
// conductor can claim the next candidate. Re-open the logical session for
// that immediate retry so the second launch can persist STARTING state.
if session.State == models.TaskSessionStateFailed {
session.State = models.TaskSessionStateCreated
session.ErrorMessage = ""
}
if previousExecutionProfileID != decision.ExecutionProfileID {
session.DownstreamACPSessionID = ""
}
return s.repo.UpdateTaskSession(ctx, session)
}
func (s *Service) persistDynamicACPSession(
ctx context.Context,
sessionID string,
decision dynamicruntime.RouteDecision,
acpSessionID string,
) error {
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil {
return err
}
if session == nil || session.RouteGeneration != decision.Generation ||
session.ExecutionProfileID != decision.ExecutionProfileID {
return dynamicruntime.ErrStaleGeneration
}
if session.DownstreamACPSessionID == acpSessionID {
return nil
}
session.DownstreamACPSessionID = acpSessionID
return s.repo.UpdateTaskSession(ctx, session)
}
// markDynamicRouteActive completes the durable "starting" -> "active"
// transition after a concrete launch has actually succeeded, and mirrors the
// status onto the task-session projection so both durable stores agree.
// Best-effort: a launch that already succeeded is not undone by a failure to
// record this side transition.
func (s *Service) markDynamicRouteActive(ctx context.Context, sessionID string, generation int64) {
if s.profileExecutionResolver == nil || sessionID == "" || generation <= 0 {
return
}
if err := s.profileExecutionResolver.MarkRouteActive(ctx, sessionID, generation); err != nil {
if !errors.Is(err, dynamicruntime.ErrStaleGeneration) && !errors.Is(err, dynamicruntime.ErrRouteStateNotFound) {
s.logger.Warn("failed to mark dynamic route active",
zap.String("session_id", sessionID), zap.Error(err))
}
return
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || session == nil || session.RouteGeneration != generation {
return
}
if loader, ok := s.repo.(dynamicRouteStateLoader); ok {
state, loadErr := loader.LoadRouteState(ctx, sessionID)
if loadErr != nil || state == nil || state.Generation != generation || state.Status != dynamicRouteStatusActive {
return
}
}
s.mirrorDynamicRouteProjection(ctx, session, generation, dynamicRouteStatusActive, session.RouteReason)
}
// markDynamicRouteActionRequired transitions a claimed dynamic route to
// durable action_required and mirrors the status onto the task-session
// projection. It is the catch-all recovery marker every declined or failed
// dynamic launch path falls back to, so a claimed route is never left
// silently stuck at "starting" or "retrying" with no recovery affordance.
// The underlying engine call only transitions a route that is still
// "starting" or "retrying" (see Engine.MarkActionRequired), so calling this
// on a route a launch already carried to "active" is a safe no-op: neither
// store is touched, and an unrelated later failure cannot overwrite a
// healthy route's reason.
func (s *Service) markDynamicRouteActionRequired(ctx context.Context, sessionID string, generation int64, reason string) {
if s.profileExecutionResolver == nil || sessionID == "" || generation <= 0 {
return
}
decision, err := s.profileExecutionResolver.MarkRouteActionRequired(ctx, sessionID, generation, reason)
if err != nil {
if !errors.Is(err, dynamicruntime.ErrStaleGeneration) && !errors.Is(err, dynamicruntime.ErrRouteStateNotFound) {
s.logger.Warn("failed to mark dynamic route action_required",
zap.String("session_id", sessionID), zap.Error(err))
}
return
}
if decision.Status != dynamicRouteStatusActionRequired {
return
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || session == nil || session.RouteGeneration != generation {
return
}
s.mirrorDynamicRouteProjection(ctx, session, generation, decision.Status, reason)
}
type dynamicRouteSessionProjector interface {
UpdateTaskSessionDynamicRouteIfCurrent(
context.Context,
string,
int64,
string,
string,
string,
) (bool, time.Time, error)
}
func (s *Service) mirrorDynamicRouteProjection(
ctx context.Context,
session *models.TaskSession,
generation int64,
status, reason string,
) {
if session == nil || session.RouteGeneration != generation || status == "" {
return
}
if session.RouteState == status && session.RouteReason == reason {
return
}
oldState := session.State
if projector, ok := s.repo.(dynamicRouteSessionProjector); ok {
changed, updatedAt, err := projector.UpdateTaskSessionDynamicRouteIfCurrent(
ctx, session.ID, generation, session.RouteState, status, reason,
)
if err != nil {
s.logger.Warn("failed to mirror dynamic route state to task session",
zap.String("session_id", session.ID), zap.Error(err))
return
}
if !changed {
return
}
session.RouteState = status
session.RouteReason = reason
session.UpdatedAt = updatedAt
s.publishTaskSessionStateChanged(ctx, session.TaskID, session.ID, oldState, session.State, session.ErrorMessage, &updatedAt, session)
return
}
// Legacy repositories do not expose the narrow route projection update. Keep
// the fallback for tests and older adapters, while production SQLite uses the
// generation/status-guarded method above.
session.RouteState = status
session.RouteReason = reason
if err := s.repo.UpdateTaskSession(ctx, session); err != nil {
s.logger.Warn("failed to mirror dynamic route state to task session",
zap.String("session_id", session.ID), zap.Error(err))
return
}
updatedAt := session.UpdatedAt
s.publishTaskSessionStateChanged(ctx, session.TaskID, session.ID, oldState, session.State, session.ErrorMessage, &updatedAt, session)
}
// handleAgentProcessStarted settles a dynamic route only after the asynchronous
// process start succeeds. LaunchPreparedSession returns before this point.
func (s *Service) handleAgentProcessStarted(
ctx context.Context,
taskID, sessionID, agentExecutionID string,
) {
if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) {
return
}
s.retireWorkflowStartPromptAttempt(ctx, taskID, sessionID, agentExecutionID)
// AC-52's acceptance edge, composed first and unconditionally on
// sessionID alone (AC-56a): profileExecutionResolver is a dynamic-routing
// precondition, not a launch one, so an instance without it configured
// must still confirm the reservation on every ordinary launch.
s.confirmCeilingReservation(sessionID)
if s.profileExecutionResolver == nil || sessionID == "" {
return
}
// The executor callback preserves the launch context's attempt value. An
// ordinary launch has no recovery identity and must be rejected while a
// newer recovery attempt is active; borrowing that newer identity here
// would let a delayed finalizeLaunch callback mutate the replacement route.
if !s.resumeAttemptAllowsExecution(sessionID, agentExecutionID, executor.ResumeAttemptIDFromContext(ctx)) {
return
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
return
}
if session.AgentExecutionID != "" && agentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
return
}
if session.State != models.TaskSessionStateStarting && session.State != models.TaskSessionStateRunning {
return
}
s.markDynamicRouteActive(ctx, sessionID, session.RouteGeneration)
}
// handleAgentProcessStartFailed settles a claimed dynamic route when process
// startup fails after LaunchPreparedSession already returned successfully.
func (s *Service) handleAgentProcessStartFailed(
ctx context.Context,
_, sessionID, agentExecutionID string,
_ error,
) {
if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) {
return
}
// AC-52's failure edge, composed first and unconditionally on sessionID
// alone (AC-56a) — see handleAgentProcessStarted's acceptance edge.
s.releaseCeilingReservation(sessionID)
if s.profileExecutionResolver == nil || sessionID == "" {
return
}
// See handleAgentProcessStarted: missing origin is intentionally fail-closed
// during an active recovery attempt rather than being relabelled as the
// current attempt.
if !s.resumeAttemptAllowsExecution(sessionID, agentExecutionID, executor.ResumeAttemptIDFromContext(ctx)) {
return
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
return
}
if session.AgentExecutionID != "" && agentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
return
}
if session.State == models.TaskSessionStateCancelled || session.State == models.TaskSessionStateCompleted {
return
}
s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, "agent_process_start_failed")
}
func applyDynamicRouteDecisionProjection(
session *models.TaskSession,
decision dynamicruntime.RouteDecision,
) {
if session == nil {
return
}
session.RouteErrorCode = string(decision.ErrorCode)
session.RouteErrorClass = string(decision.ErrorClass)
session.RouteCatalogueVersion = decision.CatalogueVersion
session.RouteRetryOrdinal = decision.RetryOrdinal
session.RoutePendingOutcome = string(decision.PendingOutcome)
session.RouteDeadline = nil
if decision.Deadline != nil {
deadline := decision.Deadline.UTC()
session.RouteDeadline = &deadline
}
}
// launchPreparedSessionWithDynamicFallback keeps the logical session stable
// while delegating classified pre-result provider fallback to dynamic.Conductor.
// Concrete profiles continue through the ordinary executor path.
func (s *Service) launchPreparedSessionWithDynamicFallback(
ctx context.Context,
task *v1.Task,
sessionID string,
options executor.LaunchOptions,
) (*executor.TaskExecution, error) {
return s.launchPreparedSessionWithDynamicFallbackWithContinuation(ctx, task, sessionID, options, nil)
}
func (s *Service) launchPreparedSessionWithDynamicFallbackWithContinuation(
ctx context.Context,
task *v1.Task,
sessionID string,
options executor.LaunchOptions,
continuationInput *dynamicruntime.ContinuationInput,
) (*executor.TaskExecution, error) {
if task != nil {
if err := s.validateContextCeilingEntry(ctx, task.ID); err != nil {
return nil, err
}
}
if s.profileExecutionResolver == nil {
return s.launchConcretePreparedSession(ctx, task, sessionID, options)
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil {
return nil, err
}
decision, dynamic, err := s.dynamicLaunchDecision(ctx, session)
if err != nil {
return nil, err
}
if !dynamic {
return s.launchConcretePreparedSession(ctx, task, sessionID, options)
}
downstream := &dynamicTaskDownstream{
service: s, task: task, sessionID: sessionID, options: options,
}
conductor := s.profileExecutionResolver.NewConductor(downstream)
prebuiltContinuation, continuationInput, err := s.dynamicContinuationForLaunch(
ctx, task, sessionID, options.Prompt, continuationInput,
)
if err != nil {
return nil, err
}
selected := dynamicruntime.ConductorSelectedLaunch{
SessionID: session.ID, LogicalProfileID: session.AgentProfileID,
Decision: decision, Prompt: options.Prompt,
PriorACPSession: session.DownstreamACPSessionID,
}
if prebuiltContinuation != nil {
selected.PrebuiltContinuation = prebuiltContinuation
} else {
selected.Continuation = *continuationInput
}
if task != nil {
if err := s.validateContextCeilingEntry(ctx, task.ID); err != nil {
return nil, err
}
}
result, err := conductor.LaunchSelected(ctx, selected)
if err != nil {
return nil, err
}
if downstream.execution == nil {
return nil, errors.New("dynamic conductor returned no task execution")
}
if result.Execution.ExecutionProfileID == "" {
result.Execution.ExecutionProfileID = session.ExecutionProfileID
}
return downstream.execution, nil
}
func (s *Service) launchConcretePreparedSession(
ctx context.Context,
task *v1.Task,
sessionID string,
options executor.LaunchOptions,
) (*executor.TaskExecution, error) {
if task != nil {
if err := s.admitCeilingDispatch(ctx, task.ID); err != nil {
return nil, err
}
}
taskID := ""
if task != nil {
taskID = task.ID
}
dispatchCtx, releaseDispatchCommit, err := s.commitCeilingEntryDispatch(
ctx, taskID, ceilingEntryBindingFromContext(ctx),
)
if err != nil {
return nil, err
}
defer releaseDispatchCommit()
if options.StartAgent && (options.Prompt != "" || len(options.Attachments) > 0) {
s.beginInitialPromptAttempt(sessionID, false)
}
execution, err := s.executor.LaunchPreparedSession(dispatchCtx, task, sessionID, options)
if execution != nil {
s.bindPromptAttemptToExecution(dispatchCtx, sessionID, execution.AgentExecutionID)
}
return execution, err
}
func (s *Service) dynamicContinuationForLaunch(
ctx context.Context,
task *v1.Task,
sessionID, prompt string,
continuationInput *dynamicruntime.ContinuationInput,
) (*dynamicruntime.Continuation, *dynamicruntime.ContinuationInput, error) {
if loader, ok := s.repo.(dynamicRouteStateLoader); ok {
state, err := loader.LoadRouteState(ctx, sessionID)
if err != nil {
return nil, nil, err
}
if state != nil && state.ContinuationJSON != "" {
var continuation dynamicruntime.Continuation
if err := json.Unmarshal([]byte(state.ContinuationJSON), &continuation); err != nil {
return nil, nil, fmt.Errorf("decode dynamic continuation: %w", err)
}
return &continuation, nil, nil
}
}
if continuationInput != nil {
return nil, continuationInput, nil
}
input, err := s.buildDynamicContinuation(ctx, task, sessionID, prompt, "")
if err != nil {
return nil, nil, err
}
return nil, &input, nil
}
func (s *Service) buildDynamicContinuation(
ctx context.Context,
task *v1.Task,
sessionID, prompt, failureReason string,
) (dynamicruntime.ContinuationInput, error) {
if task == nil || sessionID == "" {
return dynamicruntime.ContinuationInput{}, errors.New("dynamic continuation requires task and session")
}
input := dynamicruntime.ContinuationInput{
TaskDescription: task.Description,
FailureReason: failureReason,
}
if strings.TrimSpace(prompt) != "" {
input.UserMessages = append(input.UserMessages, prompt)
}
if err := s.addDynamicTaskMetadata(ctx, task, &input); err != nil {
return dynamicruntime.ContinuationInput{}, err
}
if err := s.addDynamicConversation(ctx, sessionID, &input); err != nil {
return dynamicruntime.ContinuationInput{}, err
}
if err := s.addDynamicPlan(ctx, task.ID, &input); err != nil {
return dynamicruntime.ContinuationInput{}, err
}
input.RepositorySummary = dynamicRepositorySummary(task)
return input, nil
}
func (s *Service) addDynamicTaskMetadata(ctx context.Context, task *v1.Task, input *dynamicruntime.ContinuationInput) error {
dbTask, err := s.repo.GetTask(ctx, task.ID)
if err != nil {
return fmt.Errorf("load task for dynamic continuation: %w", err)
}
if dbTask != nil {
input.WorkflowStep = dbTask.WorkflowStepID
}
return nil
}
func (s *Service) addDynamicConversation(ctx context.Context, sessionID string, input *dynamicruntime.ContinuationInput) error {
messages, err := s.repo.ListMessages(ctx, sessionID)
if err != nil {
return fmt.Errorf("load conversation for dynamic continuation: %w", err)
}
conversation := make([]string, 0, len(messages))
toolSummary := make([]string, 0)
for _, message := range messages {
if message == nil || strings.TrimSpace(message.Content) == "" {
continue
}
content := strings.TrimSpace(message.Content)
if message.AuthorType == models.MessageAuthorUser {
input.UserMessages = append(input.UserMessages, content)
continue
}
switch message.Type {
case models.MessageTypeToolCall, models.MessageTypeToolEdit,
models.MessageTypeToolRead, models.MessageTypeToolExecute:
toolSummary = append(toolSummary, string(message.Type)+": "+content)
default:
conversation = append(conversation, string(message.AuthorType)+": "+content)
}
}
input.Conversation = strings.Join(conversation, "\n")
input.ToolSummary = strings.Join(toolSummary, "\n")
return nil
}
func (s *Service) addDynamicPlan(ctx context.Context, taskID string, input *dynamicruntime.ContinuationInput) error {
plan, err := s.repo.GetTaskPlan(ctx, taskID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil
}
return fmt.Errorf("load plan for dynamic continuation: %w", err)
}
if plan == nil {
return nil
}
composed := strings.TrimSpace(plan.Title + "\n" + plan.Content)
// ContainTags is not called here: PlanSummary lands in the plain-text
// continuation package (ContinuationPrompt), not in a <kandev-system>
// block. If that ever changes, add ContainTags(composed) before Reduce.
reducedPlan, reduced, omitted := planinjection.Reduce(composed, planinjection.DynamicBudget)
input.PlanSummary = reducedPlan
if reduced {
s.logger.Info("reducing dynamic continuation plan",
zap.String("site", "dynamic_continuation"),
zap.String("task_id", taskID),
zap.Int("plan_input_bytes", len(composed)),
zap.Int("plan_output_bytes", len(reducedPlan)),
zap.Int("plan_sections_omitted", omitted),
)
}
return nil
}
func dynamicRepositorySummary(task *v1.Task) string {
repositories := make([]string, 0, len(task.Repositories)+len(task.WorkspaceFolders))
for _, repository := range task.Repositories {
repositories = append(repositories, repository.RepositoryID+" @ "+repository.BaseBranch)
}
for _, folder := range task.WorkspaceFolders {
repositories = append(repositories, "folder: "+folder.DisplayName+" @ "+folder.LocalPath)
}
return strings.Join(repositories, "\n")
}
func (s *Service) dynamicLaunchDecision(
ctx context.Context,
session *models.TaskSession,
) (dynamicruntime.RouteDecision, bool, error) {
decision := dynamicruntime.RouteDecision{
SessionID: session.ID,
LogicalProfileID: session.AgentProfileID,
ExecutionProfileID: session.ExecutionProfileID,
Generation: session.RouteGeneration,
Reason: session.RouteReason,
}
dynamic := decision.Generation > 0 && decision.ExecutionProfileID != ""
loader, ok := s.repo.(dynamicRouteStateLoader)
if !ok {
return decision, dynamic, nil
}
state, err := loader.LoadRouteState(ctx, session.ID)
if err != nil {
return dynamicruntime.RouteDecision{}, false, err
}
if state == nil || state.LogicalProfileID != session.AgentProfileID || state.Generation <= 0 {
return decision, dynamic, nil
}
if state.ExecutionProfileID == "" {
if state.Status != dynamicRouteStatusWaiting {
return dynamicruntime.RouteDecision{}, false, &dynamicruntime.NoEligibleCandidateError{
SessionID: session.ID, LogicalProfile: session.AgentProfileID,
Generation: state.Generation,
}
}
resolved, err := s.profileExecutionResolver.Resolve(
ctx, session.ID, session.AgentProfileID, state.Generation, "",
)
if err != nil {
return dynamicruntime.RouteDecision{}, false, err
}
decision = resolved.Decision
if err := s.persistDynamicLaunchDecision(ctx, session.ID, decision); err != nil {
return dynamicruntime.RouteDecision{}, false, fmt.Errorf("persist dynamic route attribution: %w", err)
}
return decision, true, nil
}
resolved, err := s.profileExecutionResolver.ResolveExisting(
ctx, session.ID, session.AgentProfileID, state.ExecutionProfileID,
state.Generation, state.ProfileVersion, "durable_route_state",
)
if err != nil {
return dynamicruntime.RouteDecision{}, false, err
}
if session.ExecutionProfileID == resolved.ExecutionProfileID &&
session.RouteGeneration == resolved.Generation {
return resolved.Decision, true, nil
}
applyResolvedExecution(session, resolved)
if err := s.repo.UpdateTaskSession(ctx, session); err != nil {
return dynamicruntime.RouteDecision{}, false, fmt.Errorf("persist dynamic route attribution: %w", err)
}
return resolved.Decision, true, nil
}
// routeDynamicAgentFailure applies the configured action for a classified
// task failure. It is intentionally limited to dynamic sessions and provider
// errors that explicitly allow fallback; user/runtime errors keep the normal
// recovery surface.
func (s *Service) routeDynamicAgentFailure(
ctx context.Context,
data watcher.AgentEventData,
classified *routingerr.Error,
) bool {
data = s.withDynamicAttemptEvidence(data)
session, ok := s.dynamicFailureSession(ctx, data)
if !ok {
return false
}
reason := "dynamic_recovery_declined"
if classified != nil {
reason = classified.Error()
}
// The route already holds a claimed generation the moment
// dynamicFailureSession succeeds. Every return below is a decline, so the
// deferred guard covers every early return uniformly (including any added
// later) instead of relying on each branch to remember its own
// transition. `generation` is updated in place once RouteAfterFailure
// claims a new one so the guard marks the generation actually holding the
// failed attempt, not the one this call started from. The guard is safe
// to arm this early even for a failure the classifier forbids fallback
// for: markDynamicRouteActionRequired only transitions a route that is
// still "starting" or "retrying", so it is a no-op once this generation's
// launch has already reached "active" (an ordinary runtime failure on a
// healthy session), and only fires for a route genuinely stranded
// mid-launch.
generation := session.RouteGeneration
handled := false
defer func() {
if !handled {
s.markDynamicRouteActionRequired(ctx, session.ID, generation, reason)
}
}()
if classified == nil || !classified.FallbackAllowed {
return false
}
if !dynamicPreResultSafe(data) {
// A dynamic route is never allowed to guess that a failed turn was
// pre-result. Missing evidence is as unsafe as observed output or a
// tool effect because the replacement could repeat a side effect.
return false
}
conductor := s.profileExecutionResolver.NewConductor(nil)
task, err := s.scheduler.GetTask(ctx, data.TaskID)
if err != nil {
return false
}
continuationInput, err := s.buildDynamicContinuation(
ctx, task, session.ID, "", classified.Error(),
)
if err != nil {
return false
}
decision, err := conductor.RouteAfterFailure(
ctx, session.ID, session.AgentProfileID, session.ExecutionProfileID,
session.RouteGeneration, classified,
)
if err != nil {
handled = s.persistPendingDynamicRecovery(ctx, session, decision, err)
return handled
}
generation = decision.Generation
handled = s.launchDynamicSuccessorAfterFailure(ctx, data, session, conductor, decision, continuationInput)
return handled
}
func (s *Service) persistPendingDynamicRecovery(
ctx context.Context,
session *models.TaskSession,
decision dynamicruntime.RouteDecision,
err error,
) bool {
if !errors.Is(err, dynamicruntime.ErrRecoveryPending) {
return false
}
// The evaluator durably owns the retry/reset deadline. Keep the logical
// session available for the authoritative recovery surface; no successor
// launch is allowed until a due/manual action claims the same generation.
session.RouteState = decision.Status
session.RouteReason = decision.Reason
applyDynamicRouteDecisionProjection(session, decision)
session.State = models.TaskSessionStateWaitingForInput
session.DownstreamACPSessionID = ""
if updateErr := s.repo.UpdateTaskSession(ctx, session); updateErr != nil {
s.logger.Warn("failed to persist pending dynamic recovery",
zap.String("session_id", session.ID), zap.Error(updateErr))
return false
}
s.schedulePersistedDynamicRecovery(ctx, session.ID, decision.Generation)
return true
}
func (s *Service) schedulePersistedDynamicRecovery(ctx context.Context, sessionID string, generation int64) {
loader, ok := s.repo.(dynamicRouteStateLoader)
if !ok {
return
}
state, err := loader.LoadRouteState(ctx, sessionID)
if err != nil || state == nil {
return
}
s.scheduleDynamicPolicyRecovery(sessionID, generation, state.PolicyStateJSON)
}
func (s *Service) launchDynamicSuccessorAfterFailure(
ctx context.Context,
data watcher.AgentEventData,
session *models.TaskSession,
conductor *dynamicruntime.Conductor,
decision dynamicruntime.RouteDecision,
continuationInput dynamicruntime.ContinuationInput,
) bool {
continuation, err := conductor.BuildContinuation(ctx, continuationInput)
if err != nil {
return false
}
if err := conductor.PersistContinuation(ctx, decision, continuation); err != nil {
return false
}
next, err := s.profileExecutionResolver.ResolveExisting(
ctx, session.ID, session.AgentProfileID, decision.ExecutionProfileID,
decision.Generation, decision.ProfileVersion, decision.Reason,
)
if err != nil || next.ExecutionProfileID == "" {
return false
}
if err := s.persistDynamicLaunchDecision(ctx, session.ID, next.Decision); err != nil {
s.logger.Warn("failed to persist dynamic fallback attribution",
zap.String("session_id", session.ID), zap.Error(err))
return false
}
return s.launchDynamicSuccessorDetached(ctx, data, next.ExecutionProfileID)
}
// launchDynamicSuccessorDetached runs the predecessor stop and successor
// launch outside the agent.failed dispatch that decided the route.
//
// The failure event is published synchronously by the lifecycle completion
// handler while it still holds the execution's prompt lifecycle lock, and the
// in-memory event bus delivers it on the publisher's goroutine. Stopping the
// predecessor from that goroutine publishes agent.stopped for the same
// execution, which needs the same lock to snapshot the prompt turn: the
// dispatch deadlocks, the successor never starts, and the session stays
// RUNNING without a process. Leaving the dispatch first lets the completion
// handler release the lock before the stop runs.
// The worker runs under the service-owned dynamicSuccessorCtx rather than
// context.WithoutCancel(ctx): the launch must survive the dispatch that
// scheduled it, but it must not outlive Stop and keep mutating session state
// after shutdown has begun. Returning false means no worker was scheduled, so
// the caller falls through to the ordinary terminal-failure path instead of
// reporting a route that will never be taken.
func (s *Service) launchDynamicSuccessorDetached(
_ context.Context,
data watcher.AgentEventData,
executionProfileID string,
) bool {
s.dynamicSuccessorMu.Lock()
if s.dynamicSuccessorStopped {
s.dynamicSuccessorMu.Unlock()
s.logger.Warn("dynamic successor launch skipped; service is stopping",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("execution_profile_id", executionProfileID))
return false
}
if s.dynamicSuccessorCtx == nil {
s.dynamicSuccessorCtx, s.dynamicSuccessorCancel = context.WithCancel(context.Background())
}
workerCtx := s.dynamicSuccessorCtx
s.dynamicSuccessorWorkers.Add(1)
s.dynamicSuccessorMu.Unlock()
go func() {
defer s.dynamicSuccessorWorkers.Done()
launchCtx, cancel := context.WithTimeout(workerCtx, dynamicSuccessorDetachedTimeout)
defer cancel()
s.runDetachedDynamicSuccessorLaunch(launchCtx, data, executionProfileID)
}()
return true
}
// runDetachedDynamicSuccessorLaunch serializes with the session's cancel guard
// like every other session-backed failure decision, then launches the
// successor. A launch that cannot complete falls back to the ordinary
// recoverable-failure surface instead of leaving the session RUNNING. A
// ceiling deferral is different: the durable replay record owns the retry, so
// the automation run must remain open until that replay either succeeds or
// fails for a non-ceiling reason.
func (s *Service) runDetachedDynamicSuccessorLaunch(
ctx context.Context,
data watcher.AgentEventData,
executionProfileID string,
) {
lock, release := s.acquireCancelInFlightGuard(data.SessionID)
defer release()
lock.Lock()
defer lock.Unlock()
if ctx.Err() != nil {
s.logger.Warn("dynamic successor launch abandoned before validation",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("execution_profile_id", executionProfileID),
zap.Error(ctx.Err()))
return
}
// A coordinator stop can win the guard between the route decision and this
// goroutine: it persists the session as cancelled and releases the guard
// before the launch acquires it. Relaunching from the stale event would
// reset that session back to CREATED and resurrect work after an
// acknowledged stop, so re-read the current state under the guard and
// abort unless the session is still nonterminal and still owns this
// execution.
if drop, _ := s.shouldDropSessionFailure(ctx, data, "dynamic.successor.detached", true); drop {
s.logger.Debug("dropping detached dynamic successor launch for stale session",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("agent_execution_id", data.AgentExecutionID),
zap.String("execution_profile_id", executionProfileID))
return
}
if ctx.Err() != nil {
s.logger.Warn("dynamic successor launch abandoned before start",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("execution_profile_id", executionProfileID),
zap.Error(ctx.Err()))
return
}
switch s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, launchOriginAutomatic) {
case dynamicRelaunchSucceeded:
return
case dynamicRelaunchDeferred:
s.logger.Debug("dynamic successor launch deferred by session ceiling",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("execution_profile_id", executionProfileID))
return
}
if errors.Is(ctx.Err(), context.Canceled) {
s.logger.Warn("dynamic successor launch abandoned after cancellation",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("execution_profile_id", executionProfileID),
zap.Error(ctx.Err()))
return
}
s.logger.Warn("dynamic successor launch failed; surfacing recoverable failure",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("agent_execution_id", data.AgentExecutionID),
zap.String("execution_profile_id", executionProfileID))
// The synchronous failure path finalized the automation run before
// surfacing the recoverable failure. handleAgentFailedLocked already
// returned here because the route was accepted, so this branch owns that
// finalization: without it an automation run stays nonterminal and holds a
// max_concurrent_runs slot and its worktree forever.
failureCtx := context.WithoutCancel(ctx)
s.retireExecutionActivityAndPublish(failureCtx, data.TaskID, data.SessionID, data.AgentExecutionID)
errMsg := data.ErrorMessage
if errMsg == "" {
errMsg = "dynamic successor launch failed"
}
s.finalizeAutomationRun(failureCtx, data.TaskID, false, errMsg)
if dispatch := s.handleRecoverableFailureLockedState(failureCtx, data, agentruntime.StopReasonRecoverableAgentFailure); dispatch != nil {
s.startAgentFailureRecovery(dispatch)
}
}
func (s *Service) resetDynamicSuccessorWorkers() {
s.dynamicSuccessorMu.Lock()
defer s.dynamicSuccessorMu.Unlock()
s.dynamicSuccessorStopped = false
s.dynamicSuccessorCtx, s.dynamicSuccessorCancel = context.WithCancel(context.Background())
}
func (s *Service) stopDynamicSuccessorWorkers() {
s.dynamicSuccessorMu.Lock()
s.dynamicSuccessorStopped = true
cancel := s.dynamicSuccessorCancel
s.dynamicSuccessorMu.Unlock()
if cancel != nil {
cancel()
}
done := make(chan struct{})
go func() {
s.dynamicSuccessorWorkers.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(sendNowClaimRecoveryTimeout):
s.logger.Warn("timed out waiting for dynamic successor workers during shutdown")
}
}
// LaunchDynamicRouteAction completes a manual retry/try-next operation after
// the backend route-action handler has claimed the successor generation. It
// owns the predecessor shutdown, continuation persistence, successor launch,
// and final launch error surface so the UI never has to issue a second launch
// request.
func (s *Service) LaunchDynamicRouteAction(ctx context.Context, sessionID string) error {
return s.launchDynamicRouteAction(ctx, sessionID, launchOriginManual)
}
// LaunchDynamicRouteActionForRecovery is the unattended counterpart used by
// the policy recovery timer sweep: same successor-launch transaction, but the
// session was never routed by a human action in this call.
func (s *Service) LaunchDynamicRouteActionForRecovery(ctx context.Context, sessionID string) error {
return s.launchDynamicRouteAction(ctx, sessionID, launchOriginAutomatic)
}
func (s *Service) launchDynamicRouteAction(ctx context.Context, sessionID string, origin launchOrigin) error {
if s.profileExecutionResolver == nil || sessionID == "" {
return errors.New("dynamic route action launch is not configured")
}
if !s.profileExecutionResolver.Enabled() {
return agentruntime.ErrDynamicRoutingDisabled
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || session == nil {
if err != nil {
return err
}
return errors.New("dynamic route action session not found")
}
// The backend route-action handler has already claimed session.RouteGeneration
// as "starting" before calling this. Every error return below leaves that
// claim without an owner unless the deferred guard marks it action_required.
succeeded := false
defer func() {
if !succeeded {
s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, string(origin)+" dynamic route action failed")
}
}()
task, err := s.scheduler.GetTask(ctx, session.TaskID)
if err != nil {
return err
}
reason := string(origin) + " dynamic route action"
input, err := s.buildDynamicContinuation(ctx, task, sessionID, "", reason)
if err != nil {
return err
}
conductor := s.profileExecutionResolver.NewConductor(nil)
continuation, err := conductor.BuildContinuation(ctx, input)
if err != nil {
return err
}
decision := dynamicruntime.RouteDecision{
SessionID: session.ID,
LogicalProfileID: session.AgentProfileID,
ExecutionProfileID: session.ExecutionProfileID,
Generation: session.RouteGeneration,
Reason: session.RouteReason,
}
if err := conductor.PersistContinuation(ctx, decision, continuation); err != nil {
return err
}
data := watcher.AgentEventData{
TaskID: task.ID,
SessionID: session.ID,
AgentExecutionID: session.AgentExecutionID,
AgentProfileID: session.AgentProfileID,
ExecutionProfileID: session.ExecutionProfileID,
ErrorMessage: reason,
}
if !s.relaunchDynamicTaskAfterFailure(ctx, data, session.ExecutionProfileID, origin) {
return errors.New("dynamic route action successor launch failed")
}
succeeded = true
return nil
}
func (s *Service) dynamicFailureSession(
ctx context.Context,
data watcher.AgentEventData,
) (*models.TaskSession, bool) {
if s.profileExecutionResolver == nil || data.SessionID == "" {
return nil, false
}
session, err := s.repo.GetTaskSession(ctx, data.SessionID)
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
return nil, false
}
if session.AgentExecutionID != "" && data.AgentExecutionID != "" &&
session.AgentExecutionID != data.AgentExecutionID {
return nil, false
}
return session, true
}
type dynamicRelaunchOutcome uint8
const (
dynamicRelaunchFailed dynamicRelaunchOutcome = iota
dynamicRelaunchDeferred
dynamicRelaunchSucceeded
dynamicRelaunchSuperseded
)
// relaunchDynamicTaskAfterFailure preserves the historical boolean API for
// synchronous route actions. Callers that own terminalization use the richer
// outcome so a durable ceiling deferral is not mistaken for a failed launch.
func (s *Service) relaunchDynamicTaskAfterFailure(
ctx context.Context,
data watcher.AgentEventData,
executionProfileID string,
origin launchOrigin,
) bool {
return s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, origin) == dynamicRelaunchSucceeded
}
func (s *Service) relaunchDynamicTaskAfterFailureOutcome(
ctx context.Context,
data watcher.AgentEventData,
executionProfileID string,
origin launchOrigin,
) (outcome dynamicRelaunchOutcome) {
return s.relaunchDynamicTaskAfterFailureOutcomeWithBinding(ctx, data, executionProfileID, origin, ceilingEntryBindingFromContext(ctx))
}
//nolint:cyclop // Dynamic relaunch keeps admission, failure recovery, and generation fencing in one boundary.
func (s *Service) relaunchDynamicTaskAfterFailureOutcomeWithBinding(
ctx context.Context,
data watcher.AgentEventData,
executionProfileID string,
origin launchOrigin,
binding *models.CeilingWorkflowEntryBinding,
) (outcome dynamicRelaunchOutcome) {
if binding != nil {
if err := s.validateClaimedCeilingBinding(ctx, data.TaskID, binding); err != nil {
if errors.Is(err, ErrCeilingLaunchSuperseded) {
return dynamicRelaunchSuperseded
}
return dynamicRelaunchFailed
}
ctx = withCeilingEntryBinding(ctx, binding)
}
seam5Res, deferredLaunch, err := s.admitOrDeferSeam5WithBinding(ctx, data.TaskID, origin, seam5DynamicRelaunchPayloadWithBinding(data, executionProfileID, binding), binding)
if err != nil {
s.logger.Zap().Error("could not persist a ceiling deferral; the dynamic relaunch could not be admitted or recorded",
zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.Error(err))
return dynamicRelaunchFailed
}
if deferredLaunch {
return dynamicRelaunchDeferred
}
defer seam5Res.releaseIfNotConsumed()
s.recordManualOverrideIfAdmitted(ctx, data.TaskID, data.SessionID, seam5Res.manualOverride, seam5Res.population, seam5Res.populationKnown, seam5Res.ceiling)
defer func() {
if outcome == dynamicRelaunchSucceeded || s.profileExecutionResolver == nil || data.SessionID == "" {
return
}
// This helper is used by both the synchronous route-failure path and
// the detached successor worker. The caller's deferred guard covers
// the former, but the detached path reports handled=true before the
// worker completes, so this helper must settle the claimed generation
// when the relaunch fails.
markCtx := context.WithoutCancel(ctx)
session, err := s.repo.GetTaskSession(markCtx, data.SessionID)
if err != nil || session == nil || session.RouteGeneration <= 0 {
return
}
s.markDynamicRouteActionRequired(
markCtx,
data.SessionID,
session.RouteGeneration,
"dynamic_successor_launch_failed",
)
}()
task, session, prompt, ok := s.prepareDynamicRelaunchAfterFailure(ctx, data)
if !ok {
return dynamicRelaunchFailed
}
if binding != nil {
if err := s.validateClaimedCeilingBinding(ctx, data.TaskID, binding); err != nil {
if errors.Is(err, ErrCeilingLaunchSuperseded) {
return dynamicRelaunchSuperseded
}
return dynamicRelaunchFailed
}
}
return s.launchPreparedDynamicRelaunch(ctx, data, task, session, prompt, executionProfileID, seam5Res)
}
func (s *Service) prepareDynamicRelaunchAfterFailure(
ctx context.Context,
data watcher.AgentEventData,
) (*v1.Task, *models.TaskSession, capturedPrompt, bool) {
prompt, ok := s.dynamicRelaunchPrompt(ctx, data.SessionID)
if !ok {
return nil, nil, capturedPrompt{}, false
}
task, err := s.scheduler.GetTask(ctx, data.TaskID)
if err != nil {
return nil, nil, capturedPrompt{}, false
}
session, err := s.repo.GetTaskSession(ctx, data.SessionID)
if err != nil || session == nil {
return nil, nil, capturedPrompt{}, false
}
if !s.stopDynamicRelaunchPredecessor(ctx, data.AgentExecutionID) {
return nil, nil, capturedPrompt{}, false
}
if !s.resetDynamicRelaunchSession(ctx, data.SessionID) {
return nil, nil, capturedPrompt{}, false
}
s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "")
s.completeTurnForSession(ctx, data.SessionID)
s.retireExecutionActivityAndPublish(ctx, data.TaskID, data.SessionID, data.AgentExecutionID)
return task, session, prompt, true
}
func (s *Service) stopDynamicRelaunchPredecessor(ctx context.Context, agentExecutionID string) bool {
if s.lspLeases != nil {
s.lspLeases.StopLSPLeasesForExecution(agentExecutionID)
}
err := s.executor.StopExecution(ctx, agentExecutionID, "dynamic route fallback", true)
if err == nil {
return true
}
if errors.Is(err, agentruntime.ErrNotFound) {
s.logger.Debug("dynamic fallback predecessor is already absent", zap.Error(err))
return true
}
s.logger.Debug("failed to stop dynamic fallback predecessor", zap.Error(err))
return false
}
func (s *Service) resetDynamicRelaunchSession(ctx context.Context, sessionID string) bool {
// Reload immediately before the reset: StopExecution is I/O and a
// coordinator stop can commit a terminal state while it runs. A stale
// pre-stop snapshot would let this write resurrect a session the user
// already cancelled, so refuse on a reloaded terminal state and CAS the
// write on that same reload to catch a cancellation landing after it.
preResetState, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil || preResetState == nil {
return false
}
if isTerminalSessionState(preResetState.State) {
return false
}
changed, _, err := s.repo.UpdateTaskSessionStateIfCurrent(
ctx, sessionID, preResetState.State, models.TaskSessionStateCreated, "",
)
return err == nil && changed
}
func (s *Service) launchPreparedDynamicRelaunch(
ctx context.Context,
data watcher.AgentEventData,
task *v1.Task,
session *models.TaskSession,
prompt capturedPrompt,
executionProfileID string,
seam5Res *sessionKeyedCeilingReservation,
) dynamicRelaunchOutcome {
officeAgentProfileID := data.AgentProfileID
if officeAgentProfileID == "" {
officeAgentProfileID = session.AgentProfileID
}
isOfficeTask, officeErr := s.lookupOfficeTask(ctx, data.TaskID)
if officeErr == nil && !isOfficeTask {
_, err := s.StartCreatedSession(
ctx, data.TaskID, data.SessionID, session.AgentProfileID,
prompt.text, true, prompt.planMode, true, prompt.attachments, nil,
)
if err != nil {
return dynamicRelaunchFailed
}
seam5Res.consume()
return dynamicRelaunchSucceeded
}
_, err := s.launchPreparedSessionWithDynamicFallback(ctx, task, data.SessionID, executor.LaunchOptions{
AgentProfileID: executionProfileID,
OfficeAgentProfileID: officeAgentProfileID,
ExecutorID: "",
Prompt: prompt.text,
StartAgent: true,
McpMode: executor.McpModeOffice,
})
if err != nil {
return dynamicRelaunchFailed
}
seam5Res.consume()
return dynamicRelaunchSucceeded
}
// dynamicRelaunchPrompt prefers the in-memory prompt cache for an automatic
// fallback, but reconstructs the latest durable user prompt for a manual route
// action or a backend restart. A route action must not strand a claimed route
// merely because the process-local retry cache was lost.
func (s *Service) dynamicRelaunchPrompt(ctx context.Context, sessionID string) (capturedPrompt, bool) {
if value, ok := s.lastTurnPrompt.Load(sessionID); ok {
if prompt, ok := value.(capturedPrompt); ok && strings.TrimSpace(prompt.text) != "" {
return prompt, true
}
}
messages, err := s.repo.ListMessages(ctx, sessionID)
if err != nil {
return capturedPrompt{}, false
}
for index := len(messages) - 1; index >= 0; index-- {
message := messages[index]
if message == nil || message.AuthorType != models.MessageAuthorUser ||
strings.TrimSpace(message.Content) == "" {
continue
}
return capturedPrompt{text: message.Content}, true
}
return capturedPrompt{}, false
}