1328 lines
48 KiB
Go
1328 lines
48 KiB
Go
package orchestrator
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/kandev/kandev/internal/agent/planinjection"
|
|
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
|
dynamicruntime "github.com/kandev/kandev/internal/agent/runtime/dynamic"
|
|
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
|
|
"github.com/kandev/kandev/internal/orchestrator/executor"
|
|
"github.com/kandev/kandev/internal/orchestrator/watcher"
|
|
"github.com/kandev/kandev/internal/task/models"
|
|
v1 "github.com/kandev/kandev/pkg/api/v1"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
const dynamicRouteStatusWaiting = "waiting"
|
|
const dynamicRouteStatusActive = "active"
|
|
const dynamicRouteStatusActionRequired = "action_required"
|
|
|
|
// dynamicSuccessorDetachedTimeout bounds one detached fallback launch: the
|
|
// predecessor stop plus the successor launch. Without it the repository calls
|
|
// inside the launch could hold the session's cancel guard indefinitely and
|
|
// block every later stop or message for that session.
|
|
const dynamicSuccessorDetachedTimeout = 2 * time.Minute
|
|
|
|
// dynamicTaskDownstream adapts the task executor to the provider-neutral
|
|
// conductor. The callback updates the task-session attribution before every
|
|
// concrete launch, including a cross-profile fallback.
|
|
type dynamicTaskDownstream struct {
|
|
service *Service
|
|
task *v1.Task
|
|
sessionID string
|
|
options executor.LaunchOptions
|
|
execution *executor.TaskExecution
|
|
}
|
|
|
|
func (d *dynamicTaskDownstream) Launch(
|
|
ctx context.Context,
|
|
launch dynamicruntime.DownstreamLaunch,
|
|
) (dynamicruntime.DownstreamExecution, error) {
|
|
if err := d.service.persistDynamicLaunchDecision(ctx, d.sessionID, launch.Decision); err != nil {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
options := d.options
|
|
options.AgentProfileID = launch.ExecutionProfileID
|
|
options.Prompt = launch.Prompt
|
|
options.PriorACPSession = launch.PriorACPSession
|
|
if d.task != nil {
|
|
if err := d.service.admitCeilingDispatch(ctx, d.task.ID); err != nil {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
}
|
|
d.service.beginDynamicAttempt(d.sessionID)
|
|
taskID := ""
|
|
if d.task != nil {
|
|
taskID = d.task.ID
|
|
}
|
|
dispatchCtx, releaseDispatchCommit, err := d.service.commitCeilingEntryDispatch(
|
|
ctx, taskID, ceilingEntryBindingFromContext(ctx),
|
|
)
|
|
if err != nil {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
defer releaseDispatchCommit()
|
|
execution, err := d.service.executor.LaunchPreparedSession(dispatchCtx, d.task, d.sessionID, options)
|
|
if err != nil {
|
|
var classified *routingerr.Error
|
|
if errors.As(err, &classified) {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
classified = routingerr.Classify(routingerr.Input{
|
|
Phase: routingerr.PhaseProcessStart,
|
|
ProviderID: launch.ExecutionProfileID,
|
|
Stderr: err.Error(),
|
|
})
|
|
// Unknown low-confidence launch failures are workspace/runtime errors,
|
|
// not provider failures. Let the ordinary launch recovery own them.
|
|
if classified.Confidence == routingerr.ConfLow {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
return dynamicruntime.DownstreamExecution{}, fmt.Errorf("%w: %v", classified, err)
|
|
}
|
|
d.service.bindDynamicAttemptExecution(d.sessionID, execution.AgentExecutionID)
|
|
d.service.bindPromptAttemptToExecution(dispatchCtx, d.sessionID, execution.AgentExecutionID)
|
|
d.execution = execution
|
|
acpSessionID := ""
|
|
if session, sessionErr := d.service.repo.GetTaskSession(ctx, d.sessionID); sessionErr == nil && session != nil {
|
|
acpSessionID = session.DownstreamACPSessionID
|
|
}
|
|
if acpSessionID != "" {
|
|
if err := d.service.persistDynamicACPSession(ctx, d.sessionID, launch.Decision, acpSessionID); err != nil {
|
|
return dynamicruntime.DownstreamExecution{}, err
|
|
}
|
|
}
|
|
return dynamicruntime.DownstreamExecution{
|
|
ID: execution.AgentExecutionID,
|
|
ExecutionProfileID: launch.ExecutionProfileID,
|
|
ACPSessionID: acpSessionID,
|
|
}, nil
|
|
}
|
|
|
|
func (d *dynamicTaskDownstream) LoadExecution(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
) (dynamicruntime.DownstreamExecution, bool, error) {
|
|
if sessionID == "" {
|
|
return dynamicruntime.DownstreamExecution{}, false, nil
|
|
}
|
|
session, err := d.service.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return dynamicruntime.DownstreamExecution{}, false, err
|
|
}
|
|
if session == nil || session.AgentExecutionID == "" {
|
|
return dynamicruntime.DownstreamExecution{}, false, nil
|
|
}
|
|
return dynamicruntime.DownstreamExecution{
|
|
ID: session.AgentExecutionID,
|
|
ExecutionProfileID: session.ExecutionProfileID,
|
|
ACPSessionID: session.DownstreamACPSessionID,
|
|
}, true, nil
|
|
}
|
|
|
|
func (d *dynamicTaskDownstream) Resume(context.Context, string, string) error {
|
|
return errors.New("dynamic task downstream resume is owned by the orchestrator")
|
|
}
|
|
|
|
func (d *dynamicTaskDownstream) Stop(context.Context, string, string) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) persistDynamicLaunchDecision(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
decision dynamicruntime.RouteDecision,
|
|
) error {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if session.ExecutionProfileID == decision.ExecutionProfileID &&
|
|
session.RouteGeneration == decision.Generation &&
|
|
decision.Status == "" {
|
|
return nil
|
|
}
|
|
previousExecutionProfileID := session.ExecutionProfileID
|
|
session.ExecutionProfileID = decision.ExecutionProfileID
|
|
session.RouteGeneration = decision.Generation
|
|
session.RouteState = decision.Status
|
|
if session.RouteState == "" {
|
|
session.RouteState = "starting"
|
|
}
|
|
session.RouteReason = decision.Reason
|
|
applyDynamicRouteDecisionProjection(session, decision)
|
|
// The executor marks a failed provider launch terminal before the
|
|
// conductor can claim the next candidate. Re-open the logical session for
|
|
// that immediate retry so the second launch can persist STARTING state.
|
|
if session.State == models.TaskSessionStateFailed {
|
|
session.State = models.TaskSessionStateCreated
|
|
session.ErrorMessage = ""
|
|
}
|
|
if previousExecutionProfileID != decision.ExecutionProfileID {
|
|
session.DownstreamACPSessionID = ""
|
|
}
|
|
return s.repo.UpdateTaskSession(ctx, session)
|
|
}
|
|
|
|
func (s *Service) persistDynamicACPSession(
|
|
ctx context.Context,
|
|
sessionID string,
|
|
decision dynamicruntime.RouteDecision,
|
|
acpSessionID string,
|
|
) error {
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if session == nil || session.RouteGeneration != decision.Generation ||
|
|
session.ExecutionProfileID != decision.ExecutionProfileID {
|
|
return dynamicruntime.ErrStaleGeneration
|
|
}
|
|
if session.DownstreamACPSessionID == acpSessionID {
|
|
return nil
|
|
}
|
|
session.DownstreamACPSessionID = acpSessionID
|
|
return s.repo.UpdateTaskSession(ctx, session)
|
|
}
|
|
|
|
// markDynamicRouteActive completes the durable "starting" -> "active"
|
|
// transition after a concrete launch has actually succeeded, and mirrors the
|
|
// status onto the task-session projection so both durable stores agree.
|
|
// Best-effort: a launch that already succeeded is not undone by a failure to
|
|
// record this side transition.
|
|
func (s *Service) markDynamicRouteActive(ctx context.Context, sessionID string, generation int64) {
|
|
if s.profileExecutionResolver == nil || sessionID == "" || generation <= 0 {
|
|
return
|
|
}
|
|
if err := s.profileExecutionResolver.MarkRouteActive(ctx, sessionID, generation); err != nil {
|
|
if !errors.Is(err, dynamicruntime.ErrStaleGeneration) && !errors.Is(err, dynamicruntime.ErrRouteStateNotFound) {
|
|
s.logger.Warn("failed to mark dynamic route active",
|
|
zap.String("session_id", sessionID), zap.Error(err))
|
|
}
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || session == nil || session.RouteGeneration != generation {
|
|
return
|
|
}
|
|
if loader, ok := s.repo.(dynamicRouteStateLoader); ok {
|
|
state, loadErr := loader.LoadRouteState(ctx, sessionID)
|
|
if loadErr != nil || state == nil || state.Generation != generation || state.Status != dynamicRouteStatusActive {
|
|
return
|
|
}
|
|
}
|
|
s.mirrorDynamicRouteProjection(ctx, session, generation, dynamicRouteStatusActive, session.RouteReason)
|
|
}
|
|
|
|
// markDynamicRouteActionRequired transitions a claimed dynamic route to
|
|
// durable action_required and mirrors the status onto the task-session
|
|
// projection. It is the catch-all recovery marker every declined or failed
|
|
// dynamic launch path falls back to, so a claimed route is never left
|
|
// silently stuck at "starting" or "retrying" with no recovery affordance.
|
|
// The underlying engine call only transitions a route that is still
|
|
// "starting" or "retrying" (see Engine.MarkActionRequired), so calling this
|
|
// on a route a launch already carried to "active" is a safe no-op: neither
|
|
// store is touched, and an unrelated later failure cannot overwrite a
|
|
// healthy route's reason.
|
|
func (s *Service) markDynamicRouteActionRequired(ctx context.Context, sessionID string, generation int64, reason string) {
|
|
if s.profileExecutionResolver == nil || sessionID == "" || generation <= 0 {
|
|
return
|
|
}
|
|
decision, err := s.profileExecutionResolver.MarkRouteActionRequired(ctx, sessionID, generation, reason)
|
|
if err != nil {
|
|
if !errors.Is(err, dynamicruntime.ErrStaleGeneration) && !errors.Is(err, dynamicruntime.ErrRouteStateNotFound) {
|
|
s.logger.Warn("failed to mark dynamic route action_required",
|
|
zap.String("session_id", sessionID), zap.Error(err))
|
|
}
|
|
return
|
|
}
|
|
if decision.Status != dynamicRouteStatusActionRequired {
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || session == nil || session.RouteGeneration != generation {
|
|
return
|
|
}
|
|
s.mirrorDynamicRouteProjection(ctx, session, generation, decision.Status, reason)
|
|
}
|
|
|
|
type dynamicRouteSessionProjector interface {
|
|
UpdateTaskSessionDynamicRouteIfCurrent(
|
|
context.Context,
|
|
string,
|
|
int64,
|
|
string,
|
|
string,
|
|
string,
|
|
) (bool, time.Time, error)
|
|
}
|
|
|
|
func (s *Service) mirrorDynamicRouteProjection(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
generation int64,
|
|
status, reason string,
|
|
) {
|
|
if session == nil || session.RouteGeneration != generation || status == "" {
|
|
return
|
|
}
|
|
if session.RouteState == status && session.RouteReason == reason {
|
|
return
|
|
}
|
|
oldState := session.State
|
|
if projector, ok := s.repo.(dynamicRouteSessionProjector); ok {
|
|
changed, updatedAt, err := projector.UpdateTaskSessionDynamicRouteIfCurrent(
|
|
ctx, session.ID, generation, session.RouteState, status, reason,
|
|
)
|
|
if err != nil {
|
|
s.logger.Warn("failed to mirror dynamic route state to task session",
|
|
zap.String("session_id", session.ID), zap.Error(err))
|
|
return
|
|
}
|
|
if !changed {
|
|
return
|
|
}
|
|
session.RouteState = status
|
|
session.RouteReason = reason
|
|
session.UpdatedAt = updatedAt
|
|
s.publishTaskSessionStateChanged(ctx, session.TaskID, session.ID, oldState, session.State, session.ErrorMessage, &updatedAt, session)
|
|
return
|
|
}
|
|
// Legacy repositories do not expose the narrow route projection update. Keep
|
|
// the fallback for tests and older adapters, while production SQLite uses the
|
|
// generation/status-guarded method above.
|
|
session.RouteState = status
|
|
session.RouteReason = reason
|
|
if err := s.repo.UpdateTaskSession(ctx, session); err != nil {
|
|
s.logger.Warn("failed to mirror dynamic route state to task session",
|
|
zap.String("session_id", session.ID), zap.Error(err))
|
|
return
|
|
}
|
|
updatedAt := session.UpdatedAt
|
|
s.publishTaskSessionStateChanged(ctx, session.TaskID, session.ID, oldState, session.State, session.ErrorMessage, &updatedAt, session)
|
|
}
|
|
|
|
// handleAgentProcessStarted settles a dynamic route only after the asynchronous
|
|
// process start succeeds. LaunchPreparedSession returns before this point.
|
|
func (s *Service) handleAgentProcessStarted(
|
|
ctx context.Context,
|
|
taskID, sessionID, agentExecutionID string,
|
|
) {
|
|
if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) {
|
|
return
|
|
}
|
|
s.retireWorkflowStartPromptAttempt(ctx, taskID, sessionID, agentExecutionID)
|
|
// AC-52's acceptance edge, composed first and unconditionally on
|
|
// sessionID alone (AC-56a): profileExecutionResolver is a dynamic-routing
|
|
// precondition, not a launch one, so an instance without it configured
|
|
// must still confirm the reservation on every ordinary launch.
|
|
s.confirmCeilingReservation(sessionID)
|
|
if s.profileExecutionResolver == nil || sessionID == "" {
|
|
return
|
|
}
|
|
// The executor callback preserves the launch context's attempt value. An
|
|
// ordinary launch has no recovery identity and must be rejected while a
|
|
// newer recovery attempt is active; borrowing that newer identity here
|
|
// would let a delayed finalizeLaunch callback mutate the replacement route.
|
|
if !s.resumeAttemptAllowsExecution(sessionID, agentExecutionID, executor.ResumeAttemptIDFromContext(ctx)) {
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
|
|
return
|
|
}
|
|
if session.AgentExecutionID != "" && agentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
|
|
return
|
|
}
|
|
if session.State != models.TaskSessionStateStarting && session.State != models.TaskSessionStateRunning {
|
|
return
|
|
}
|
|
s.markDynamicRouteActive(ctx, sessionID, session.RouteGeneration)
|
|
}
|
|
|
|
// handleAgentProcessStartFailed settles a claimed dynamic route when process
|
|
// startup fails after LaunchPreparedSession already returned successfully.
|
|
func (s *Service) handleAgentProcessStartFailed(
|
|
ctx context.Context,
|
|
_, sessionID, agentExecutionID string,
|
|
_ error,
|
|
) {
|
|
if !s.ceilingCallbackOwnsSession(ctx, sessionID, agentExecutionID) {
|
|
return
|
|
}
|
|
// AC-52's failure edge, composed first and unconditionally on sessionID
|
|
// alone (AC-56a) — see handleAgentProcessStarted's acceptance edge.
|
|
s.releaseCeilingReservation(sessionID)
|
|
if s.profileExecutionResolver == nil || sessionID == "" {
|
|
return
|
|
}
|
|
// See handleAgentProcessStarted: missing origin is intentionally fail-closed
|
|
// during an active recovery attempt rather than being relabelled as the
|
|
// current attempt.
|
|
if !s.resumeAttemptAllowsExecution(sessionID, agentExecutionID, executor.ResumeAttemptIDFromContext(ctx)) {
|
|
return
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
|
|
return
|
|
}
|
|
if session.AgentExecutionID != "" && agentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
|
|
return
|
|
}
|
|
if session.State == models.TaskSessionStateCancelled || session.State == models.TaskSessionStateCompleted {
|
|
return
|
|
}
|
|
s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, "agent_process_start_failed")
|
|
}
|
|
|
|
func applyDynamicRouteDecisionProjection(
|
|
session *models.TaskSession,
|
|
decision dynamicruntime.RouteDecision,
|
|
) {
|
|
if session == nil {
|
|
return
|
|
}
|
|
session.RouteErrorCode = string(decision.ErrorCode)
|
|
session.RouteErrorClass = string(decision.ErrorClass)
|
|
session.RouteCatalogueVersion = decision.CatalogueVersion
|
|
session.RouteRetryOrdinal = decision.RetryOrdinal
|
|
session.RoutePendingOutcome = string(decision.PendingOutcome)
|
|
session.RouteDeadline = nil
|
|
if decision.Deadline != nil {
|
|
deadline := decision.Deadline.UTC()
|
|
session.RouteDeadline = &deadline
|
|
}
|
|
}
|
|
|
|
// launchPreparedSessionWithDynamicFallback keeps the logical session stable
|
|
// while delegating classified pre-result provider fallback to dynamic.Conductor.
|
|
// Concrete profiles continue through the ordinary executor path.
|
|
func (s *Service) launchPreparedSessionWithDynamicFallback(
|
|
ctx context.Context,
|
|
task *v1.Task,
|
|
sessionID string,
|
|
options executor.LaunchOptions,
|
|
) (*executor.TaskExecution, error) {
|
|
return s.launchPreparedSessionWithDynamicFallbackWithContinuation(ctx, task, sessionID, options, nil)
|
|
}
|
|
|
|
func (s *Service) launchPreparedSessionWithDynamicFallbackWithContinuation(
|
|
ctx context.Context,
|
|
task *v1.Task,
|
|
sessionID string,
|
|
options executor.LaunchOptions,
|
|
continuationInput *dynamicruntime.ContinuationInput,
|
|
) (*executor.TaskExecution, error) {
|
|
if task != nil {
|
|
if err := s.validateContextCeilingEntry(ctx, task.ID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if s.profileExecutionResolver == nil {
|
|
return s.launchConcretePreparedSession(ctx, task, sessionID, options)
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
decision, dynamic, err := s.dynamicLaunchDecision(ctx, session)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !dynamic {
|
|
return s.launchConcretePreparedSession(ctx, task, sessionID, options)
|
|
}
|
|
downstream := &dynamicTaskDownstream{
|
|
service: s, task: task, sessionID: sessionID, options: options,
|
|
}
|
|
conductor := s.profileExecutionResolver.NewConductor(downstream)
|
|
prebuiltContinuation, continuationInput, err := s.dynamicContinuationForLaunch(
|
|
ctx, task, sessionID, options.Prompt, continuationInput,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
selected := dynamicruntime.ConductorSelectedLaunch{
|
|
SessionID: session.ID, LogicalProfileID: session.AgentProfileID,
|
|
Decision: decision, Prompt: options.Prompt,
|
|
PriorACPSession: session.DownstreamACPSessionID,
|
|
}
|
|
if prebuiltContinuation != nil {
|
|
selected.PrebuiltContinuation = prebuiltContinuation
|
|
} else {
|
|
selected.Continuation = *continuationInput
|
|
}
|
|
if task != nil {
|
|
if err := s.validateContextCeilingEntry(ctx, task.ID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
result, err := conductor.LaunchSelected(ctx, selected)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if downstream.execution == nil {
|
|
return nil, errors.New("dynamic conductor returned no task execution")
|
|
}
|
|
if result.Execution.ExecutionProfileID == "" {
|
|
result.Execution.ExecutionProfileID = session.ExecutionProfileID
|
|
}
|
|
return downstream.execution, nil
|
|
}
|
|
|
|
func (s *Service) launchConcretePreparedSession(
|
|
ctx context.Context,
|
|
task *v1.Task,
|
|
sessionID string,
|
|
options executor.LaunchOptions,
|
|
) (*executor.TaskExecution, error) {
|
|
if task != nil {
|
|
if err := s.admitCeilingDispatch(ctx, task.ID); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
taskID := ""
|
|
if task != nil {
|
|
taskID = task.ID
|
|
}
|
|
dispatchCtx, releaseDispatchCommit, err := s.commitCeilingEntryDispatch(
|
|
ctx, taskID, ceilingEntryBindingFromContext(ctx),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer releaseDispatchCommit()
|
|
if options.StartAgent && (options.Prompt != "" || len(options.Attachments) > 0) {
|
|
s.beginInitialPromptAttempt(sessionID, false)
|
|
}
|
|
execution, err := s.executor.LaunchPreparedSession(dispatchCtx, task, sessionID, options)
|
|
if execution != nil {
|
|
s.bindPromptAttemptToExecution(dispatchCtx, sessionID, execution.AgentExecutionID)
|
|
}
|
|
return execution, err
|
|
}
|
|
|
|
func (s *Service) dynamicContinuationForLaunch(
|
|
ctx context.Context,
|
|
task *v1.Task,
|
|
sessionID, prompt string,
|
|
continuationInput *dynamicruntime.ContinuationInput,
|
|
) (*dynamicruntime.Continuation, *dynamicruntime.ContinuationInput, error) {
|
|
if loader, ok := s.repo.(dynamicRouteStateLoader); ok {
|
|
state, err := loader.LoadRouteState(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if state != nil && state.ContinuationJSON != "" {
|
|
var continuation dynamicruntime.Continuation
|
|
if err := json.Unmarshal([]byte(state.ContinuationJSON), &continuation); err != nil {
|
|
return nil, nil, fmt.Errorf("decode dynamic continuation: %w", err)
|
|
}
|
|
return &continuation, nil, nil
|
|
}
|
|
}
|
|
if continuationInput != nil {
|
|
return nil, continuationInput, nil
|
|
}
|
|
input, err := s.buildDynamicContinuation(ctx, task, sessionID, prompt, "")
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return nil, &input, nil
|
|
}
|
|
|
|
func (s *Service) buildDynamicContinuation(
|
|
ctx context.Context,
|
|
task *v1.Task,
|
|
sessionID, prompt, failureReason string,
|
|
) (dynamicruntime.ContinuationInput, error) {
|
|
if task == nil || sessionID == "" {
|
|
return dynamicruntime.ContinuationInput{}, errors.New("dynamic continuation requires task and session")
|
|
}
|
|
input := dynamicruntime.ContinuationInput{
|
|
TaskDescription: task.Description,
|
|
FailureReason: failureReason,
|
|
}
|
|
if strings.TrimSpace(prompt) != "" {
|
|
input.UserMessages = append(input.UserMessages, prompt)
|
|
}
|
|
if err := s.addDynamicTaskMetadata(ctx, task, &input); err != nil {
|
|
return dynamicruntime.ContinuationInput{}, err
|
|
}
|
|
if err := s.addDynamicConversation(ctx, sessionID, &input); err != nil {
|
|
return dynamicruntime.ContinuationInput{}, err
|
|
}
|
|
if err := s.addDynamicPlan(ctx, task.ID, &input); err != nil {
|
|
return dynamicruntime.ContinuationInput{}, err
|
|
}
|
|
input.RepositorySummary = dynamicRepositorySummary(task)
|
|
return input, nil
|
|
}
|
|
|
|
func (s *Service) addDynamicTaskMetadata(ctx context.Context, task *v1.Task, input *dynamicruntime.ContinuationInput) error {
|
|
dbTask, err := s.repo.GetTask(ctx, task.ID)
|
|
if err != nil {
|
|
return fmt.Errorf("load task for dynamic continuation: %w", err)
|
|
}
|
|
if dbTask != nil {
|
|
input.WorkflowStep = dbTask.WorkflowStepID
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) addDynamicConversation(ctx context.Context, sessionID string, input *dynamicruntime.ContinuationInput) error {
|
|
messages, err := s.repo.ListMessages(ctx, sessionID)
|
|
if err != nil {
|
|
return fmt.Errorf("load conversation for dynamic continuation: %w", err)
|
|
}
|
|
conversation := make([]string, 0, len(messages))
|
|
toolSummary := make([]string, 0)
|
|
for _, message := range messages {
|
|
if message == nil || strings.TrimSpace(message.Content) == "" {
|
|
continue
|
|
}
|
|
content := strings.TrimSpace(message.Content)
|
|
if message.AuthorType == models.MessageAuthorUser {
|
|
input.UserMessages = append(input.UserMessages, content)
|
|
continue
|
|
}
|
|
switch message.Type {
|
|
case models.MessageTypeToolCall, models.MessageTypeToolEdit,
|
|
models.MessageTypeToolRead, models.MessageTypeToolExecute:
|
|
toolSummary = append(toolSummary, string(message.Type)+": "+content)
|
|
default:
|
|
conversation = append(conversation, string(message.AuthorType)+": "+content)
|
|
}
|
|
}
|
|
input.Conversation = strings.Join(conversation, "\n")
|
|
input.ToolSummary = strings.Join(toolSummary, "\n")
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) addDynamicPlan(ctx context.Context, taskID string, input *dynamicruntime.ContinuationInput) error {
|
|
plan, err := s.repo.GetTaskPlan(ctx, taskID)
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("load plan for dynamic continuation: %w", err)
|
|
}
|
|
if plan == nil {
|
|
return nil
|
|
}
|
|
|
|
composed := strings.TrimSpace(plan.Title + "\n" + plan.Content)
|
|
// ContainTags is not called here: PlanSummary lands in the plain-text
|
|
// continuation package (ContinuationPrompt), not in a <kandev-system>
|
|
// block. If that ever changes, add ContainTags(composed) before Reduce.
|
|
reducedPlan, reduced, omitted := planinjection.Reduce(composed, planinjection.DynamicBudget)
|
|
input.PlanSummary = reducedPlan
|
|
if reduced {
|
|
s.logger.Info("reducing dynamic continuation plan",
|
|
zap.String("site", "dynamic_continuation"),
|
|
zap.String("task_id", taskID),
|
|
zap.Int("plan_input_bytes", len(composed)),
|
|
zap.Int("plan_output_bytes", len(reducedPlan)),
|
|
zap.Int("plan_sections_omitted", omitted),
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func dynamicRepositorySummary(task *v1.Task) string {
|
|
repositories := make([]string, 0, len(task.Repositories)+len(task.WorkspaceFolders))
|
|
for _, repository := range task.Repositories {
|
|
repositories = append(repositories, repository.RepositoryID+" @ "+repository.BaseBranch)
|
|
}
|
|
for _, folder := range task.WorkspaceFolders {
|
|
repositories = append(repositories, "folder: "+folder.DisplayName+" @ "+folder.LocalPath)
|
|
}
|
|
return strings.Join(repositories, "\n")
|
|
}
|
|
|
|
func (s *Service) dynamicLaunchDecision(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
) (dynamicruntime.RouteDecision, bool, error) {
|
|
decision := dynamicruntime.RouteDecision{
|
|
SessionID: session.ID,
|
|
LogicalProfileID: session.AgentProfileID,
|
|
ExecutionProfileID: session.ExecutionProfileID,
|
|
Generation: session.RouteGeneration,
|
|
Reason: session.RouteReason,
|
|
}
|
|
dynamic := decision.Generation > 0 && decision.ExecutionProfileID != ""
|
|
loader, ok := s.repo.(dynamicRouteStateLoader)
|
|
if !ok {
|
|
return decision, dynamic, nil
|
|
}
|
|
state, err := loader.LoadRouteState(ctx, session.ID)
|
|
if err != nil {
|
|
return dynamicruntime.RouteDecision{}, false, err
|
|
}
|
|
if state == nil || state.LogicalProfileID != session.AgentProfileID || state.Generation <= 0 {
|
|
return decision, dynamic, nil
|
|
}
|
|
if state.ExecutionProfileID == "" {
|
|
if state.Status != dynamicRouteStatusWaiting {
|
|
return dynamicruntime.RouteDecision{}, false, &dynamicruntime.NoEligibleCandidateError{
|
|
SessionID: session.ID, LogicalProfile: session.AgentProfileID,
|
|
Generation: state.Generation,
|
|
}
|
|
}
|
|
resolved, err := s.profileExecutionResolver.Resolve(
|
|
ctx, session.ID, session.AgentProfileID, state.Generation, "",
|
|
)
|
|
if err != nil {
|
|
return dynamicruntime.RouteDecision{}, false, err
|
|
}
|
|
decision = resolved.Decision
|
|
if err := s.persistDynamicLaunchDecision(ctx, session.ID, decision); err != nil {
|
|
return dynamicruntime.RouteDecision{}, false, fmt.Errorf("persist dynamic route attribution: %w", err)
|
|
}
|
|
return decision, true, nil
|
|
}
|
|
resolved, err := s.profileExecutionResolver.ResolveExisting(
|
|
ctx, session.ID, session.AgentProfileID, state.ExecutionProfileID,
|
|
state.Generation, state.ProfileVersion, "durable_route_state",
|
|
)
|
|
if err != nil {
|
|
return dynamicruntime.RouteDecision{}, false, err
|
|
}
|
|
if session.ExecutionProfileID == resolved.ExecutionProfileID &&
|
|
session.RouteGeneration == resolved.Generation {
|
|
return resolved.Decision, true, nil
|
|
}
|
|
applyResolvedExecution(session, resolved)
|
|
if err := s.repo.UpdateTaskSession(ctx, session); err != nil {
|
|
return dynamicruntime.RouteDecision{}, false, fmt.Errorf("persist dynamic route attribution: %w", err)
|
|
}
|
|
return resolved.Decision, true, nil
|
|
}
|
|
|
|
// routeDynamicAgentFailure applies the configured action for a classified
|
|
// task failure. It is intentionally limited to dynamic sessions and provider
|
|
// errors that explicitly allow fallback; user/runtime errors keep the normal
|
|
// recovery surface.
|
|
func (s *Service) routeDynamicAgentFailure(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
classified *routingerr.Error,
|
|
) bool {
|
|
data = s.withDynamicAttemptEvidence(data)
|
|
session, ok := s.dynamicFailureSession(ctx, data)
|
|
if !ok {
|
|
return false
|
|
}
|
|
reason := "dynamic_recovery_declined"
|
|
if classified != nil {
|
|
reason = classified.Error()
|
|
}
|
|
// The route already holds a claimed generation the moment
|
|
// dynamicFailureSession succeeds. Every return below is a decline, so the
|
|
// deferred guard covers every early return uniformly (including any added
|
|
// later) instead of relying on each branch to remember its own
|
|
// transition. `generation` is updated in place once RouteAfterFailure
|
|
// claims a new one so the guard marks the generation actually holding the
|
|
// failed attempt, not the one this call started from. The guard is safe
|
|
// to arm this early even for a failure the classifier forbids fallback
|
|
// for: markDynamicRouteActionRequired only transitions a route that is
|
|
// still "starting" or "retrying", so it is a no-op once this generation's
|
|
// launch has already reached "active" (an ordinary runtime failure on a
|
|
// healthy session), and only fires for a route genuinely stranded
|
|
// mid-launch.
|
|
generation := session.RouteGeneration
|
|
handled := false
|
|
defer func() {
|
|
if !handled {
|
|
s.markDynamicRouteActionRequired(ctx, session.ID, generation, reason)
|
|
}
|
|
}()
|
|
if classified == nil || !classified.FallbackAllowed {
|
|
return false
|
|
}
|
|
if !dynamicPreResultSafe(data) {
|
|
// A dynamic route is never allowed to guess that a failed turn was
|
|
// pre-result. Missing evidence is as unsafe as observed output or a
|
|
// tool effect because the replacement could repeat a side effect.
|
|
return false
|
|
}
|
|
conductor := s.profileExecutionResolver.NewConductor(nil)
|
|
task, err := s.scheduler.GetTask(ctx, data.TaskID)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
continuationInput, err := s.buildDynamicContinuation(
|
|
ctx, task, session.ID, "", classified.Error(),
|
|
)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
decision, err := conductor.RouteAfterFailure(
|
|
ctx, session.ID, session.AgentProfileID, session.ExecutionProfileID,
|
|
session.RouteGeneration, classified,
|
|
)
|
|
if err != nil {
|
|
handled = s.persistPendingDynamicRecovery(ctx, session, decision, err)
|
|
return handled
|
|
}
|
|
generation = decision.Generation
|
|
handled = s.launchDynamicSuccessorAfterFailure(ctx, data, session, conductor, decision, continuationInput)
|
|
return handled
|
|
}
|
|
|
|
func (s *Service) persistPendingDynamicRecovery(
|
|
ctx context.Context,
|
|
session *models.TaskSession,
|
|
decision dynamicruntime.RouteDecision,
|
|
err error,
|
|
) bool {
|
|
if !errors.Is(err, dynamicruntime.ErrRecoveryPending) {
|
|
return false
|
|
}
|
|
// The evaluator durably owns the retry/reset deadline. Keep the logical
|
|
// session available for the authoritative recovery surface; no successor
|
|
// launch is allowed until a due/manual action claims the same generation.
|
|
session.RouteState = decision.Status
|
|
session.RouteReason = decision.Reason
|
|
applyDynamicRouteDecisionProjection(session, decision)
|
|
session.State = models.TaskSessionStateWaitingForInput
|
|
session.DownstreamACPSessionID = ""
|
|
if updateErr := s.repo.UpdateTaskSession(ctx, session); updateErr != nil {
|
|
s.logger.Warn("failed to persist pending dynamic recovery",
|
|
zap.String("session_id", session.ID), zap.Error(updateErr))
|
|
return false
|
|
}
|
|
s.schedulePersistedDynamicRecovery(ctx, session.ID, decision.Generation)
|
|
return true
|
|
}
|
|
|
|
func (s *Service) schedulePersistedDynamicRecovery(ctx context.Context, sessionID string, generation int64) {
|
|
loader, ok := s.repo.(dynamicRouteStateLoader)
|
|
if !ok {
|
|
return
|
|
}
|
|
state, err := loader.LoadRouteState(ctx, sessionID)
|
|
if err != nil || state == nil {
|
|
return
|
|
}
|
|
s.scheduleDynamicPolicyRecovery(sessionID, generation, state.PolicyStateJSON)
|
|
}
|
|
|
|
func (s *Service) launchDynamicSuccessorAfterFailure(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
session *models.TaskSession,
|
|
conductor *dynamicruntime.Conductor,
|
|
decision dynamicruntime.RouteDecision,
|
|
continuationInput dynamicruntime.ContinuationInput,
|
|
) bool {
|
|
continuation, err := conductor.BuildContinuation(ctx, continuationInput)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if err := conductor.PersistContinuation(ctx, decision, continuation); err != nil {
|
|
return false
|
|
}
|
|
next, err := s.profileExecutionResolver.ResolveExisting(
|
|
ctx, session.ID, session.AgentProfileID, decision.ExecutionProfileID,
|
|
decision.Generation, decision.ProfileVersion, decision.Reason,
|
|
)
|
|
if err != nil || next.ExecutionProfileID == "" {
|
|
return false
|
|
}
|
|
if err := s.persistDynamicLaunchDecision(ctx, session.ID, next.Decision); err != nil {
|
|
s.logger.Warn("failed to persist dynamic fallback attribution",
|
|
zap.String("session_id", session.ID), zap.Error(err))
|
|
return false
|
|
}
|
|
return s.launchDynamicSuccessorDetached(ctx, data, next.ExecutionProfileID)
|
|
}
|
|
|
|
// launchDynamicSuccessorDetached runs the predecessor stop and successor
|
|
// launch outside the agent.failed dispatch that decided the route.
|
|
//
|
|
// The failure event is published synchronously by the lifecycle completion
|
|
// handler while it still holds the execution's prompt lifecycle lock, and the
|
|
// in-memory event bus delivers it on the publisher's goroutine. Stopping the
|
|
// predecessor from that goroutine publishes agent.stopped for the same
|
|
// execution, which needs the same lock to snapshot the prompt turn: the
|
|
// dispatch deadlocks, the successor never starts, and the session stays
|
|
// RUNNING without a process. Leaving the dispatch first lets the completion
|
|
// handler release the lock before the stop runs.
|
|
// The worker runs under the service-owned dynamicSuccessorCtx rather than
|
|
// context.WithoutCancel(ctx): the launch must survive the dispatch that
|
|
// scheduled it, but it must not outlive Stop and keep mutating session state
|
|
// after shutdown has begun. Returning false means no worker was scheduled, so
|
|
// the caller falls through to the ordinary terminal-failure path instead of
|
|
// reporting a route that will never be taken.
|
|
func (s *Service) launchDynamicSuccessorDetached(
|
|
_ context.Context,
|
|
data watcher.AgentEventData,
|
|
executionProfileID string,
|
|
) bool {
|
|
s.dynamicSuccessorMu.Lock()
|
|
if s.dynamicSuccessorStopped {
|
|
s.dynamicSuccessorMu.Unlock()
|
|
s.logger.Warn("dynamic successor launch skipped; service is stopping",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("execution_profile_id", executionProfileID))
|
|
return false
|
|
}
|
|
if s.dynamicSuccessorCtx == nil {
|
|
s.dynamicSuccessorCtx, s.dynamicSuccessorCancel = context.WithCancel(context.Background())
|
|
}
|
|
workerCtx := s.dynamicSuccessorCtx
|
|
s.dynamicSuccessorWorkers.Add(1)
|
|
s.dynamicSuccessorMu.Unlock()
|
|
go func() {
|
|
defer s.dynamicSuccessorWorkers.Done()
|
|
launchCtx, cancel := context.WithTimeout(workerCtx, dynamicSuccessorDetachedTimeout)
|
|
defer cancel()
|
|
s.runDetachedDynamicSuccessorLaunch(launchCtx, data, executionProfileID)
|
|
}()
|
|
return true
|
|
}
|
|
|
|
// runDetachedDynamicSuccessorLaunch serializes with the session's cancel guard
|
|
// like every other session-backed failure decision, then launches the
|
|
// successor. A launch that cannot complete falls back to the ordinary
|
|
// recoverable-failure surface instead of leaving the session RUNNING. A
|
|
// ceiling deferral is different: the durable replay record owns the retry, so
|
|
// the automation run must remain open until that replay either succeeds or
|
|
// fails for a non-ceiling reason.
|
|
func (s *Service) runDetachedDynamicSuccessorLaunch(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
executionProfileID string,
|
|
) {
|
|
lock, release := s.acquireCancelInFlightGuard(data.SessionID)
|
|
defer release()
|
|
lock.Lock()
|
|
defer lock.Unlock()
|
|
if ctx.Err() != nil {
|
|
s.logger.Warn("dynamic successor launch abandoned before validation",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("execution_profile_id", executionProfileID),
|
|
zap.Error(ctx.Err()))
|
|
return
|
|
}
|
|
// A coordinator stop can win the guard between the route decision and this
|
|
// goroutine: it persists the session as cancelled and releases the guard
|
|
// before the launch acquires it. Relaunching from the stale event would
|
|
// reset that session back to CREATED and resurrect work after an
|
|
// acknowledged stop, so re-read the current state under the guard and
|
|
// abort unless the session is still nonterminal and still owns this
|
|
// execution.
|
|
if drop, _ := s.shouldDropSessionFailure(ctx, data, "dynamic.successor.detached", true); drop {
|
|
s.logger.Debug("dropping detached dynamic successor launch for stale session",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("agent_execution_id", data.AgentExecutionID),
|
|
zap.String("execution_profile_id", executionProfileID))
|
|
return
|
|
}
|
|
if ctx.Err() != nil {
|
|
s.logger.Warn("dynamic successor launch abandoned before start",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("execution_profile_id", executionProfileID),
|
|
zap.Error(ctx.Err()))
|
|
return
|
|
}
|
|
switch s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, launchOriginAutomatic) {
|
|
case dynamicRelaunchSucceeded:
|
|
return
|
|
case dynamicRelaunchDeferred:
|
|
s.logger.Debug("dynamic successor launch deferred by session ceiling",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("execution_profile_id", executionProfileID))
|
|
return
|
|
}
|
|
if errors.Is(ctx.Err(), context.Canceled) {
|
|
s.logger.Warn("dynamic successor launch abandoned after cancellation",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("execution_profile_id", executionProfileID),
|
|
zap.Error(ctx.Err()))
|
|
return
|
|
}
|
|
s.logger.Warn("dynamic successor launch failed; surfacing recoverable failure",
|
|
zap.String("task_id", data.TaskID),
|
|
zap.String("session_id", data.SessionID),
|
|
zap.String("agent_execution_id", data.AgentExecutionID),
|
|
zap.String("execution_profile_id", executionProfileID))
|
|
// The synchronous failure path finalized the automation run before
|
|
// surfacing the recoverable failure. handleAgentFailedLocked already
|
|
// returned here because the route was accepted, so this branch owns that
|
|
// finalization: without it an automation run stays nonterminal and holds a
|
|
// max_concurrent_runs slot and its worktree forever.
|
|
failureCtx := context.WithoutCancel(ctx)
|
|
s.retireExecutionActivityAndPublish(failureCtx, data.TaskID, data.SessionID, data.AgentExecutionID)
|
|
errMsg := data.ErrorMessage
|
|
if errMsg == "" {
|
|
errMsg = "dynamic successor launch failed"
|
|
}
|
|
s.finalizeAutomationRun(failureCtx, data.TaskID, false, errMsg)
|
|
if dispatch := s.handleRecoverableFailureLockedState(failureCtx, data, agentruntime.StopReasonRecoverableAgentFailure); dispatch != nil {
|
|
s.startAgentFailureRecovery(dispatch)
|
|
}
|
|
}
|
|
|
|
func (s *Service) resetDynamicSuccessorWorkers() {
|
|
s.dynamicSuccessorMu.Lock()
|
|
defer s.dynamicSuccessorMu.Unlock()
|
|
s.dynamicSuccessorStopped = false
|
|
s.dynamicSuccessorCtx, s.dynamicSuccessorCancel = context.WithCancel(context.Background())
|
|
}
|
|
|
|
func (s *Service) stopDynamicSuccessorWorkers() {
|
|
s.dynamicSuccessorMu.Lock()
|
|
s.dynamicSuccessorStopped = true
|
|
cancel := s.dynamicSuccessorCancel
|
|
s.dynamicSuccessorMu.Unlock()
|
|
if cancel != nil {
|
|
cancel()
|
|
}
|
|
done := make(chan struct{})
|
|
go func() {
|
|
s.dynamicSuccessorWorkers.Wait()
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(sendNowClaimRecoveryTimeout):
|
|
s.logger.Warn("timed out waiting for dynamic successor workers during shutdown")
|
|
}
|
|
}
|
|
|
|
// LaunchDynamicRouteAction completes a manual retry/try-next operation after
|
|
// the backend route-action handler has claimed the successor generation. It
|
|
// owns the predecessor shutdown, continuation persistence, successor launch,
|
|
// and final launch error surface so the UI never has to issue a second launch
|
|
// request.
|
|
func (s *Service) LaunchDynamicRouteAction(ctx context.Context, sessionID string) error {
|
|
return s.launchDynamicRouteAction(ctx, sessionID, launchOriginManual)
|
|
}
|
|
|
|
// LaunchDynamicRouteActionForRecovery is the unattended counterpart used by
|
|
// the policy recovery timer sweep: same successor-launch transaction, but the
|
|
// session was never routed by a human action in this call.
|
|
func (s *Service) LaunchDynamicRouteActionForRecovery(ctx context.Context, sessionID string) error {
|
|
return s.launchDynamicRouteAction(ctx, sessionID, launchOriginAutomatic)
|
|
}
|
|
|
|
func (s *Service) launchDynamicRouteAction(ctx context.Context, sessionID string, origin launchOrigin) error {
|
|
if s.profileExecutionResolver == nil || sessionID == "" {
|
|
return errors.New("dynamic route action launch is not configured")
|
|
}
|
|
if !s.profileExecutionResolver.Enabled() {
|
|
return agentruntime.ErrDynamicRoutingDisabled
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || session == nil {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return errors.New("dynamic route action session not found")
|
|
}
|
|
// The backend route-action handler has already claimed session.RouteGeneration
|
|
// as "starting" before calling this. Every error return below leaves that
|
|
// claim without an owner unless the deferred guard marks it action_required.
|
|
succeeded := false
|
|
defer func() {
|
|
if !succeeded {
|
|
s.markDynamicRouteActionRequired(ctx, sessionID, session.RouteGeneration, string(origin)+" dynamic route action failed")
|
|
}
|
|
}()
|
|
task, err := s.scheduler.GetTask(ctx, session.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reason := string(origin) + " dynamic route action"
|
|
input, err := s.buildDynamicContinuation(ctx, task, sessionID, "", reason)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
conductor := s.profileExecutionResolver.NewConductor(nil)
|
|
continuation, err := conductor.BuildContinuation(ctx, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
decision := dynamicruntime.RouteDecision{
|
|
SessionID: session.ID,
|
|
LogicalProfileID: session.AgentProfileID,
|
|
ExecutionProfileID: session.ExecutionProfileID,
|
|
Generation: session.RouteGeneration,
|
|
Reason: session.RouteReason,
|
|
}
|
|
if err := conductor.PersistContinuation(ctx, decision, continuation); err != nil {
|
|
return err
|
|
}
|
|
data := watcher.AgentEventData{
|
|
TaskID: task.ID,
|
|
SessionID: session.ID,
|
|
AgentExecutionID: session.AgentExecutionID,
|
|
AgentProfileID: session.AgentProfileID,
|
|
ExecutionProfileID: session.ExecutionProfileID,
|
|
ErrorMessage: reason,
|
|
}
|
|
if !s.relaunchDynamicTaskAfterFailure(ctx, data, session.ExecutionProfileID, origin) {
|
|
return errors.New("dynamic route action successor launch failed")
|
|
}
|
|
succeeded = true
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) dynamicFailureSession(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
) (*models.TaskSession, bool) {
|
|
if s.profileExecutionResolver == nil || data.SessionID == "" {
|
|
return nil, false
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, data.SessionID)
|
|
if err != nil || session == nil || session.RouteGeneration <= 0 || session.ExecutionProfileID == "" {
|
|
return nil, false
|
|
}
|
|
if session.AgentExecutionID != "" && data.AgentExecutionID != "" &&
|
|
session.AgentExecutionID != data.AgentExecutionID {
|
|
return nil, false
|
|
}
|
|
return session, true
|
|
}
|
|
|
|
type dynamicRelaunchOutcome uint8
|
|
|
|
const (
|
|
dynamicRelaunchFailed dynamicRelaunchOutcome = iota
|
|
dynamicRelaunchDeferred
|
|
dynamicRelaunchSucceeded
|
|
dynamicRelaunchSuperseded
|
|
)
|
|
|
|
// relaunchDynamicTaskAfterFailure preserves the historical boolean API for
|
|
// synchronous route actions. Callers that own terminalization use the richer
|
|
// outcome so a durable ceiling deferral is not mistaken for a failed launch.
|
|
func (s *Service) relaunchDynamicTaskAfterFailure(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
executionProfileID string,
|
|
origin launchOrigin,
|
|
) bool {
|
|
return s.relaunchDynamicTaskAfterFailureOutcome(ctx, data, executionProfileID, origin) == dynamicRelaunchSucceeded
|
|
}
|
|
|
|
func (s *Service) relaunchDynamicTaskAfterFailureOutcome(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
executionProfileID string,
|
|
origin launchOrigin,
|
|
) (outcome dynamicRelaunchOutcome) {
|
|
return s.relaunchDynamicTaskAfterFailureOutcomeWithBinding(ctx, data, executionProfileID, origin, ceilingEntryBindingFromContext(ctx))
|
|
}
|
|
|
|
//nolint:cyclop // Dynamic relaunch keeps admission, failure recovery, and generation fencing in one boundary.
|
|
func (s *Service) relaunchDynamicTaskAfterFailureOutcomeWithBinding(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
executionProfileID string,
|
|
origin launchOrigin,
|
|
binding *models.CeilingWorkflowEntryBinding,
|
|
) (outcome dynamicRelaunchOutcome) {
|
|
if binding != nil {
|
|
if err := s.validateClaimedCeilingBinding(ctx, data.TaskID, binding); err != nil {
|
|
if errors.Is(err, ErrCeilingLaunchSuperseded) {
|
|
return dynamicRelaunchSuperseded
|
|
}
|
|
return dynamicRelaunchFailed
|
|
}
|
|
ctx = withCeilingEntryBinding(ctx, binding)
|
|
}
|
|
seam5Res, deferredLaunch, err := s.admitOrDeferSeam5WithBinding(ctx, data.TaskID, origin, seam5DynamicRelaunchPayloadWithBinding(data, executionProfileID, binding), binding)
|
|
if err != nil {
|
|
s.logger.Zap().Error("could not persist a ceiling deferral; the dynamic relaunch could not be admitted or recorded",
|
|
zap.String("task_id", data.TaskID), zap.String("session_id", data.SessionID), zap.Error(err))
|
|
return dynamicRelaunchFailed
|
|
}
|
|
if deferredLaunch {
|
|
return dynamicRelaunchDeferred
|
|
}
|
|
defer seam5Res.releaseIfNotConsumed()
|
|
s.recordManualOverrideIfAdmitted(ctx, data.TaskID, data.SessionID, seam5Res.manualOverride, seam5Res.population, seam5Res.populationKnown, seam5Res.ceiling)
|
|
|
|
defer func() {
|
|
if outcome == dynamicRelaunchSucceeded || s.profileExecutionResolver == nil || data.SessionID == "" {
|
|
return
|
|
}
|
|
// This helper is used by both the synchronous route-failure path and
|
|
// the detached successor worker. The caller's deferred guard covers
|
|
// the former, but the detached path reports handled=true before the
|
|
// worker completes, so this helper must settle the claimed generation
|
|
// when the relaunch fails.
|
|
markCtx := context.WithoutCancel(ctx)
|
|
session, err := s.repo.GetTaskSession(markCtx, data.SessionID)
|
|
if err != nil || session == nil || session.RouteGeneration <= 0 {
|
|
return
|
|
}
|
|
s.markDynamicRouteActionRequired(
|
|
markCtx,
|
|
data.SessionID,
|
|
session.RouteGeneration,
|
|
"dynamic_successor_launch_failed",
|
|
)
|
|
}()
|
|
|
|
task, session, prompt, ok := s.prepareDynamicRelaunchAfterFailure(ctx, data)
|
|
if !ok {
|
|
return dynamicRelaunchFailed
|
|
}
|
|
if binding != nil {
|
|
if err := s.validateClaimedCeilingBinding(ctx, data.TaskID, binding); err != nil {
|
|
if errors.Is(err, ErrCeilingLaunchSuperseded) {
|
|
return dynamicRelaunchSuperseded
|
|
}
|
|
return dynamicRelaunchFailed
|
|
}
|
|
}
|
|
return s.launchPreparedDynamicRelaunch(ctx, data, task, session, prompt, executionProfileID, seam5Res)
|
|
}
|
|
|
|
func (s *Service) prepareDynamicRelaunchAfterFailure(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
) (*v1.Task, *models.TaskSession, capturedPrompt, bool) {
|
|
prompt, ok := s.dynamicRelaunchPrompt(ctx, data.SessionID)
|
|
if !ok {
|
|
return nil, nil, capturedPrompt{}, false
|
|
}
|
|
task, err := s.scheduler.GetTask(ctx, data.TaskID)
|
|
if err != nil {
|
|
return nil, nil, capturedPrompt{}, false
|
|
}
|
|
session, err := s.repo.GetTaskSession(ctx, data.SessionID)
|
|
if err != nil || session == nil {
|
|
return nil, nil, capturedPrompt{}, false
|
|
}
|
|
if !s.stopDynamicRelaunchPredecessor(ctx, data.AgentExecutionID) {
|
|
return nil, nil, capturedPrompt{}, false
|
|
}
|
|
if !s.resetDynamicRelaunchSession(ctx, data.SessionID) {
|
|
return nil, nil, capturedPrompt{}, false
|
|
}
|
|
s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "")
|
|
s.completeTurnForSession(ctx, data.SessionID)
|
|
s.retireExecutionActivityAndPublish(ctx, data.TaskID, data.SessionID, data.AgentExecutionID)
|
|
return task, session, prompt, true
|
|
}
|
|
|
|
func (s *Service) stopDynamicRelaunchPredecessor(ctx context.Context, agentExecutionID string) bool {
|
|
if s.lspLeases != nil {
|
|
s.lspLeases.StopLSPLeasesForExecution(agentExecutionID)
|
|
}
|
|
err := s.executor.StopExecution(ctx, agentExecutionID, "dynamic route fallback", true)
|
|
if err == nil {
|
|
return true
|
|
}
|
|
if errors.Is(err, agentruntime.ErrNotFound) {
|
|
s.logger.Debug("dynamic fallback predecessor is already absent", zap.Error(err))
|
|
return true
|
|
}
|
|
s.logger.Debug("failed to stop dynamic fallback predecessor", zap.Error(err))
|
|
return false
|
|
}
|
|
|
|
func (s *Service) resetDynamicRelaunchSession(ctx context.Context, sessionID string) bool {
|
|
// Reload immediately before the reset: StopExecution is I/O and a
|
|
// coordinator stop can commit a terminal state while it runs. A stale
|
|
// pre-stop snapshot would let this write resurrect a session the user
|
|
// already cancelled, so refuse on a reloaded terminal state and CAS the
|
|
// write on that same reload to catch a cancellation landing after it.
|
|
preResetState, err := s.repo.GetTaskSession(ctx, sessionID)
|
|
if err != nil || preResetState == nil {
|
|
return false
|
|
}
|
|
if isTerminalSessionState(preResetState.State) {
|
|
return false
|
|
}
|
|
changed, _, err := s.repo.UpdateTaskSessionStateIfCurrent(
|
|
ctx, sessionID, preResetState.State, models.TaskSessionStateCreated, "",
|
|
)
|
|
return err == nil && changed
|
|
}
|
|
|
|
func (s *Service) launchPreparedDynamicRelaunch(
|
|
ctx context.Context,
|
|
data watcher.AgentEventData,
|
|
task *v1.Task,
|
|
session *models.TaskSession,
|
|
prompt capturedPrompt,
|
|
executionProfileID string,
|
|
seam5Res *sessionKeyedCeilingReservation,
|
|
) dynamicRelaunchOutcome {
|
|
officeAgentProfileID := data.AgentProfileID
|
|
if officeAgentProfileID == "" {
|
|
officeAgentProfileID = session.AgentProfileID
|
|
}
|
|
isOfficeTask, officeErr := s.lookupOfficeTask(ctx, data.TaskID)
|
|
if officeErr == nil && !isOfficeTask {
|
|
_, err := s.StartCreatedSession(
|
|
ctx, data.TaskID, data.SessionID, session.AgentProfileID,
|
|
prompt.text, true, prompt.planMode, true, prompt.attachments, nil,
|
|
)
|
|
if err != nil {
|
|
return dynamicRelaunchFailed
|
|
}
|
|
seam5Res.consume()
|
|
return dynamicRelaunchSucceeded
|
|
}
|
|
_, err := s.launchPreparedSessionWithDynamicFallback(ctx, task, data.SessionID, executor.LaunchOptions{
|
|
AgentProfileID: executionProfileID,
|
|
OfficeAgentProfileID: officeAgentProfileID,
|
|
ExecutorID: "",
|
|
Prompt: prompt.text,
|
|
StartAgent: true,
|
|
McpMode: executor.McpModeOffice,
|
|
})
|
|
if err != nil {
|
|
return dynamicRelaunchFailed
|
|
}
|
|
seam5Res.consume()
|
|
return dynamicRelaunchSucceeded
|
|
}
|
|
|
|
// dynamicRelaunchPrompt prefers the in-memory prompt cache for an automatic
|
|
// fallback, but reconstructs the latest durable user prompt for a manual route
|
|
// action or a backend restart. A route action must not strand a claimed route
|
|
// merely because the process-local retry cache was lost.
|
|
func (s *Service) dynamicRelaunchPrompt(ctx context.Context, sessionID string) (capturedPrompt, bool) {
|
|
if value, ok := s.lastTurnPrompt.Load(sessionID); ok {
|
|
if prompt, ok := value.(capturedPrompt); ok && strings.TrimSpace(prompt.text) != "" {
|
|
return prompt, true
|
|
}
|
|
}
|
|
messages, err := s.repo.ListMessages(ctx, sessionID)
|
|
if err != nil {
|
|
return capturedPrompt{}, false
|
|
}
|
|
for index := len(messages) - 1; index >= 0; index-- {
|
|
message := messages[index]
|
|
if message == nil || message.AuthorType != models.MessageAuthorUser ||
|
|
strings.TrimSpace(message.Content) == "" {
|
|
continue
|
|
}
|
|
return capturedPrompt{text: message.Content}, true
|
|
}
|
|
return capturedPrompt{}, false
|
|
}
|