fix(backend): prevent duplicate or stranded launches when tasks auto-start on create (#3529)

Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com>
Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
Henry
2026-09-11 04:30:10 +08:00
committed by GitHub
parent 5ffe881877
commit bfdfcdef8e
10 changed files with 1195 additions and 80 deletions
@@ -564,7 +564,7 @@ func TestQueuePromotionOfBlockedWIPOverflowDoesNotLaunch(t *testing.T) {
// resume the still-pending promotion lifecycle rather than leave the token
// stranded after launching directly.
dependencies.blocked = false
svc.autoStartTaskForStep(ctx, deferredChainTaskID, "step-limited", "task.dependencies_resolved", 0)
svc.autoStartTaskForStep(ctx, deferredChainTaskID, "step-limited", "task.dependencies_resolved", 0, false)
require.True(t, counter.awaitLaunch(0), "resolved dependency did not launch the promoted task")
awaitLaunchedSession(t, repo, deferredChainTaskID)
started, err := repo.GetTask(ctx, deferredChainTaskID)
@@ -31,26 +31,33 @@ func (s *Service) SetTaskDependencyReader(reader TaskDependencyReader) {
}
// dependencyBlocksAutoStart reports whether taskID has an unresolved dependency
// and therefore must not be launched by an automated path.
// and therefore must not be launched by an automated path. gateErrored reports
// whether the block came from a failed read rather than a genuine dependency
// block: a genuine block is already covered by dependency-resolution launch
// paths (evaluateDependentAfterPredecessorChange, reconcileDependencyLaunchesOnStartup),
// but a read failure is not, so a caller holding a one-shot launch token must
// restore it only in that case — restoring on every genuine block would burn
// recoverTaskLifecycleAttempt's bounded retry budget on every startup for as
// long as the task stays blocked.
//
// Fails CLOSED: a read error returns true. Failing open would launch work whose
// predecessor may never have run, which is the single outcome task dependencies
// exist to prevent.
func (s *Service) dependencyBlocksAutoStart(ctx context.Context, taskID, eventName string) bool {
// Fails CLOSED: a read error returns blocked=true. Failing open would launch
// work whose predecessor may never have run, which is the single outcome task
// dependencies exist to prevent.
func (s *Service) dependencyBlocksAutoStart(ctx context.Context, taskID, eventName string) (blocked, gateErrored bool) {
if s.dependencyReader == nil {
return false
return false, false
}
blocked, reason, err := s.dependencyReader.DependencyGate(ctx, taskID)
isBlocked, reason, err := s.dependencyReader.DependencyGate(ctx, taskID)
if err != nil {
s.logger.Warn(eventName+": dependency lookup failed; skipping auto-start",
zap.String("task_id", taskID), zap.Error(err))
return true
return true, true
}
if blocked {
if isBlocked {
s.logger.Debug(eventName+": task has unresolved dependencies; skipping auto-start",
zap.String("task_id", taskID), zap.String("blocked_reason", reason))
}
return blocked
return isBlocked, false
}
// handleTaskDependenciesForTerminalState reacts to a task reaching a terminal
@@ -116,7 +123,7 @@ func (s *Service) evaluateDependentAfterPredecessorChange(
zap.String("task_id", task.ID))
return
}
s.autoStartTaskForStep(ctx, task.ID, task.WorkflowStepID, events.TaskDependenciesResolved, 0)
s.autoStartTaskForStep(ctx, task.ID, task.WorkflowStepID, events.TaskDependenciesResolved, 0, false)
}
// publishDependenciesResolved announces that a task's last unresolved
@@ -185,6 +192,6 @@ func (s *Service) reconcileDependencyLaunchesOnStartup(ctx context.Context) {
}
s.logger.Info("startup: launching task whose dependencies resolved while down",
zap.String("task_id", candidate.TaskID))
s.autoStartTaskForStep(ctx, candidate.TaskID, candidate.WorkflowStepID, "startup.dependencies_resolved", 0)
s.autoStartTaskForStep(ctx, candidate.TaskID, candidate.WorkflowStepID, "startup.dependencies_resolved", 0, false)
}
}
@@ -644,6 +644,7 @@ func (s *Service) autoStartReviewTask(
if s.shouldSkipTerminalPRAutoStart(ctx, task) {
return
}
createAutoStartOwned := s.ownsAutoStartOnCreateInFlight(task.ID)
// Compete for the one-shot auto-start token set at task creation.
// The promotion that ran inside CreateTask may have already triggered
// autoStartTaskForStep (Path A) asynchronously; only the first claimer
@@ -673,8 +674,16 @@ func (s *Service) autoStartReviewTask(
zap.String("task_id", task.ID),
zap.Error(err))
s.restoreAutoStartClaim(ctx, task.ID, "review.auto_start")
if createAutoStartOwned || s.ownsAutoStartOnCreateInFlight(task.ID) {
s.restoreAutoStartOnCreate(ctx, task.ID, "review.auto_start")
}
return
}
// The watcher path can win the permanent review guard while the
// create-time path is still preparing its detached launch. A successful
// watcher launch satisfies that intent too, so clear the durable marker
// owned by the other path.
s.completeAutoStartOnCreate(ctx, task.ID, "review.auto_start")
s.logger.Info("auto-started review task",
zap.String("task_id", task.ID),
zap.Int("pr_number", evt.PR.Number))
@@ -405,7 +405,7 @@ func TestAutoStart_BothPathsFireExactlyOnce(t *testing.T) {
// Path A: promotion event-driven auto-start on the same task.
// autoStartTaskForStep spawns a goroutine — we synchronise via the launched channel.
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.queue_promoted", 0)
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.queue_promoted", 0, false)
// Wait for exactly one launch. Allow 2 seconds for the async goroutine.
deadline := time.After(2 * time.Second)
@@ -568,7 +568,7 @@ func TestAutoStart_NoTokenDoesNotBlock(t *testing.T) {
}
svc := createTestServiceWithScheduler(repo, sg, taskRepo, agentMgr)
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.moved", 0)
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.moved", 0, false)
select {
case <-launched:
@@ -627,7 +627,7 @@ func TestAutoStart_FailedLaunchWithoutGuardDoesNotStampClaimed(t *testing.T) {
}
svc := createTestServiceWithScheduler(repo, sg, taskRepo, agentMgr)
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.moved", 0)
svc.autoStartTaskForStep(ctx, taskID, stepID, "task.moved", 0, false)
select {
case <-attempted:
@@ -710,7 +710,7 @@ func (s *Service) handleTaskMoved(ctx context.Context, data watcher.TaskMovedEve
return
}
if prerequisites != nil && prerequisites.targetStep != nil {
s.autoStartTaskForLoadedStep(ctx, task, prerequisites.targetStep, "task.moved", data.QueuePromotion, data.StepTransitionID)
s.autoStartTaskForLoadedStep(ctx, task, prerequisites.targetStep, "task.moved", data.QueuePromotion, data.StepTransitionID, false)
} else {
s.handleTaskMovedNoSession(ctx, data)
}
@@ -792,33 +792,66 @@ func (s *Service) loadTaskMovedSessionPrerequisites(
// If the target step has auto_start_agent, it creates a session and starts the agent
// using agent/executor profile IDs from the task's metadata.
func (s *Service) handleTaskMovedNoSession(ctx context.Context, data watcher.TaskMovedEventData) {
s.autoStartTaskForStep(ctx, data.TaskID, data.ToStepID, "task.moved", data.StepTransitionID)
s.autoStartTaskForStep(ctx, data.TaskID, data.ToStepID, "task.moved", data.StepTransitionID, false)
}
func (s *Service) handleTaskQueuePromoted(ctx context.Context, data watcher.TaskEventData) {
task, err := s.repo.GetTask(ctx, data.TaskID)
s.handleTaskQueuePromotedWithAutoStartOnCreateClaimed(ctx, data, false)
}
// loadQueuePromotedTaskAndTargetStep loads and validates the task and its
// current workflow step for a task.queue_promoted delivery, returning
// ok=false for every condition under which the event should be dropped
// (load failure, no longer queued/admitted, or a manual move still mid-flight).
func (s *Service) loadQueuePromotedTaskAndTargetStep(ctx context.Context, taskID string) (*models.Task, *wfmodels.WorkflowStep, bool) {
task, err := s.repo.GetTask(ctx, taskID)
if err != nil {
s.logger.Warn("task.queue_promoted: failed to load task", zap.String("task_id", data.TaskID), zap.Error(err))
return
s.logger.Warn("task.queue_promoted: failed to load task", zap.String("task_id", taskID), zap.Error(err))
return nil, nil, false
}
if task.QueuedForStepID != "" || !task.WIPAdmitted {
return
return nil, nil, false
}
if queuedMoveExitPending(task) || manualMoveLifecyclePending(task) {
// A manual move has not finished its lifecycle yet.
// Keep the promotion token durable; source-exit completion will trigger
// queue reconciliation and retry destination entry.
return
return nil, nil, false
}
targetStep, err := s.workflowStepGetter.GetStep(ctx, task.WorkflowStepID)
if err != nil || targetStep == nil {
s.logger.Warn("task.queue_promoted: failed to load target step",
zap.String("task_id", task.ID), zap.String("step_id", task.WorkflowStepID), zap.Error(err))
return nil, nil, false
}
return task, targetStep, true
}
// handleTaskQueuePromotedWithAutoStartOnCreateClaimed is handleTaskQueuePromoted's
// implementation, parameterized by whether the caller already claimed
// MetaKeyAutoStartOnCreate for this launch attempt (see
// claimAutoStartOnCreateForLaunch). autoStartTaskForStep's queue-promotion
// redirect calls this directly so a create-time opt-in that handleTaskCreated
// already claimed carries through promotion instead of being silently
// dropped; every other caller goes through handleTaskQueuePromoted and passes
// false.
//
//nolint:cyclop // queue promotion has independent task, dependency, session, and token recovery states
func (s *Service) handleTaskQueuePromotedWithAutoStartOnCreateClaimed(ctx context.Context, data watcher.TaskEventData, autoStartOnCreateClaimed bool) {
restoreCreateIntent := func() {
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, data.TaskID, "task.queue_promoted")
}
}
task, targetStep, ok := s.loadQueuePromotedTaskAndTargetStep(ctx, data.TaskID)
if !ok {
restoreCreateIntent()
return
}
if err := s.syncTaskStateForQueuePromotion(ctx, task, targetStep); err != nil {
s.logger.Warn("task.queue_promoted: failed to synchronize task state",
zap.String("task_id", task.ID), zap.Error(err))
restoreCreateIntent()
return
}
session, sessionErr := s.repo.GetActiveTaskSessionByTaskID(ctx, task.ID)
@@ -831,13 +864,19 @@ func (s *Service) handleTaskQueuePromoted(ctx context.Context, data watcher.Task
if sessionErr != nil {
s.logger.Warn("task.queue_promoted: failed to load active session",
zap.String("task_id", task.ID), zap.Error(sessionErr))
restoreCreateIntent()
return
}
if session == nil && s.dependencyBlocksAutoStart(ctx, task.ID, "task.queue_promoted") {
// Promotion controls WIP admission, not dependency readiness. Keep both
// the promotion lifecycle token and deferred launch intent intact so the
// dependency-resolution path can start the task once its blockers clear.
return
if session == nil {
if blocked, _ := s.dependencyBlocksAutoStart(ctx, task.ID, "task.queue_promoted"); blocked {
// Promotion controls WIP admission, not dependency readiness. Keep both
// the promotion lifecycle token and deferred launch intent intact so the
// dependency-resolution path can start the task once its blockers clear.
if autoStartOnCreateClaimed {
s.discardAutoStartOnCreate(ctx, task.ID, "task.queue_promoted")
}
return
}
}
// Read the source descriptor before claiming the one-shot promotion token.
// The claim removes the marker durably and a repository implementation may
@@ -847,10 +886,12 @@ func (s *Service) handleTaskQueuePromoted(ctx context.Context, data watcher.Task
if sourceErr != nil {
s.logger.Warn("task.queue_promoted: failed to load source step",
zap.String("task_id", task.ID), zap.Error(sourceErr))
restoreCreateIntent()
return
}
queuePromotionToken := queuePromotionLifecycleToken(task)
if !s.claimTaskEventMetadata(ctx, task, models.MetaKeyQueuePromotionPending) {
restoreCreateIntent()
return
}
if remover, ok := s.repo.(taskMetadataKeyRemover); ok {
@@ -861,15 +902,21 @@ func (s *Service) handleTaskQueuePromoted(ctx context.Context, data watcher.Task
go func() {
if err := s.finalizeStepEnter(context.WithoutCancel(ctx), task.ID, session.ID, targetStep, task.Description, targetStep.HasOnEnterAction(wfmodels.OnEnterAutoStartAgent), sourceStep, data.StepTransitionID); err != nil {
s.restoreTaskLifecycleToken(context.WithoutCancel(ctx), task.ID, models.MetaKeyQueuePromotionPending, queuePromotionToken, "task.queue_promoted")
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(context.WithoutCancel(ctx), task.ID, "task.queue_promoted")
}
return
}
if autoStartOnCreateClaimed {
s.completeAutoStartOnCreate(context.WithoutCancel(ctx), task.ID, "task.queue_promoted")
}
if s.onTaskQueuePromotionEntryComplete != nil {
s.onTaskQueuePromotionEntryComplete()
}
}()
return
}
s.autoStartTaskForLoadedStep(ctx, task, targetStep, "task.queue_promoted", true, data.StepTransitionID)
s.autoStartTaskForLoadedStep(ctx, task, targetStep, "task.queue_promoted", true, data.StepTransitionID, autoStartOnCreateClaimed)
}
// handleTaskCreated lets a task that is created directly onto a step whose
@@ -907,11 +954,11 @@ func (s *Service) handleTaskQueuePromoted(ctx context.Context, data watcher.Task
// anything with it. Either way the key can never change this handler's
// outcome, so checking it here would be dead code.
//
// The opt-in itself is claimed via claimTaskEventMetadata before launching,
// the same one-shot-token pattern used for MetaKeyQueuePromotionPending: a
// duplicate task.created delivery for the same task would otherwise see the
// key still present and launch a second time. Only the delivery that wins
// the atomic RemoveTaskMetadataKey proceeds.
// The opt-in itself is claimed before launching, the same one-shot-token
// pattern used for MetaKeyQueuePromotionPending. The claim also writes a
// durable in-flight marker before removing the intent, so a process exit
// cannot erase the last recovery signal. Only the delivery that wins the
// local and database ownership hand-off proceeds.
func (s *Service) handleTaskCreated(ctx context.Context, data watcher.TaskEventData) {
task, err := s.repo.GetTask(ctx, data.TaskID)
if err != nil || task == nil {
@@ -923,10 +970,22 @@ func (s *Service) handleTaskCreated(ctx context.Context, data watcher.TaskEventD
if !autoStartOnCreateActionable(task) {
return
}
if !s.claimTaskEventMetadata(ctx, task, models.MetaKeyAutoStartOnCreate) {
sessions, err := s.repo.ListTaskSessions(ctx, task.ID)
if err != nil {
s.logger.Warn("task.created: failed to check existing sessions", zap.String("task_id", task.ID), zap.Error(err))
return
}
s.autoStartTaskForStep(ctx, task.ID, task.WorkflowStepID, events.TaskCreated, data.StepTransitionID)
if len(sessions) > 0 {
s.completeAutoStartOnCreate(ctx, task.ID, events.TaskCreated)
return
}
if result := s.claimAutoStartOnCreateForLaunch(ctx, task, false); result != autoStartOnCreateClaimOwned {
return
}
// Carry ownership into the launch attempt so a StartTask failure before a
// durable session exists can restore the intent (see
// handleAutoStartFailure).
s.autoStartTaskForStep(ctx, task.ID, task.WorkflowStepID, events.TaskCreated, data.StepTransitionID, true)
}
func (s *Service) claimTaskEventMetadata(ctx context.Context, task *models.Task, key string) bool {
@@ -1072,8 +1131,13 @@ func (s *Service) reconcileTaskLifecycleTokens(ctx context.Context) {
s.logger.Warn("failed to list auto-start-on-create tokens for recovery", zap.Error(err))
return
}
// autoStarts is the raw, pre-filter list; only entries that pass
// autoStartOnCreateActionable below are added to jobs, so it is left out
autoStartsInFlight, err := lister.ListTasksWithMetadataKey(ctx, models.MetaKeyAutoStartOnCreateInFlight)
if err != nil {
s.logger.Warn("failed to list in-flight auto-start-on-create tokens for recovery", zap.Error(err))
return
}
// The auto-start lists are raw, pre-filter lists; only entries that pass
// autoStartOnCreateActionable below are added to jobs, so they are left out
// of this capacity hint rather than causing routine over-allocation.
jobs := make(map[string]struct{}, len(pending)+len(promotions)+len(manualPending)+len(manualCompleted))
for _, task := range pending {
@@ -1096,7 +1160,7 @@ func (s *Service) reconcileTaskLifecycleTokens(ctx context.Context) {
jobs[task.ID] = struct{}{}
}
}
for _, task := range autoStarts {
for _, task := range append(autoStarts, autoStartsInFlight...) {
// ListTasksWithMetadataKey matches key EXISTENCE, which is broader
// than what handleTaskCreated will act on. Rows it refuses keep their
// key forever, so admitting them would schedule work that can never
@@ -1275,9 +1339,30 @@ func (s *Service) recoverTaskLifecycleAttempt(ctx context.Context, taskID string
if _, err := s.clearManualMoveLifecycleMarkersIfCompleted(ctx, taskID, completedAt); err != nil {
return true
}
task, err = s.repo.GetTask(ctx, taskID)
if err != nil || task == nil {
// A transient failure here only ends this attempt; it leaves any
// surviving lifecycle marker untouched, so the next startup sweep
// re-lists and retries the task.
return false
}
}
if _, pending := task.Metadata[models.MetaKeyQueuePromotionPending]; pending {
s.handleTaskQueuePromoted(ctx, watcher.TaskEventData{TaskID: taskID})
// handleTaskQueuePromoted's no-session branch schedules a launch via
// autoStartTaskForLoadedStep, which synchronously claims
// MetaKeyAutoStartOnCreate when the task still carries it (see
// claimAutoStartOnCreateForLaunch) before this call returns. Re-fetch
// so the actionability check below observes that claim instead of the
// stale in-memory task, which would otherwise report the token as
// still pending and schedule a second, redundant launch attempt.
task, err = s.repo.GetTask(ctx, taskID)
if err != nil || task == nil {
// A transient failure here only ends this attempt; it leaves any
// surviving lifecycle marker untouched, so the next startup sweep
// re-lists and retries the task.
return false
}
}
if autoStartOnCreateActionable(task) {
s.recoverAutoStartOnCreate(ctx, task)
@@ -1411,20 +1496,22 @@ func autoStartOnCreateActionable(task *models.Task) bool {
if task == nil || task.IsFromOffice {
return false
}
return models.HasAutoStartOnCreateIntent(task.Metadata)
return models.HasAutoStartOnCreateIntent(task.Metadata) ||
models.HasAutoStartOnCreateInFlight(task.Metadata)
}
// recoverAutoStartOnCreate replays a lost task.created delivery for a task that
// still carries an actionable create-time opt-in.
//
// The opt-in is NOT proof that no launch happened. handleTaskCreated is the
// only function that claims this key; a manual StartTask, a task.moved
// auto-start and handleTaskQueuePromoted all launch without touching it. So
// after a lost delivery — the very failure this sweep repairs — an operator
// starting the task by hand leaves the token behind, and replaying it here
// would launch a second agent onto a task that is already running or already
// finished. Neither autoStartTaskForStep nor startTask has an existing-session
// guard, so that launch would go all the way through.
// The opt-in is NOT proof that no launch happened. A manual StartTask launches
// without ever touching this key, so an operator starting the task by hand
// after a lost delivery leaves the token behind. Every automated auto-start
// path (task.moved, dependency resolution, handleTaskQueuePromoted) now claims
// this key itself via claimAutoStartOnCreateForLaunch before it launches. The
// durable in-flight marker remains until a session or run is created, while
// the local ownership map prevents this process from reclaiming it during the
// detached launch. Replaying here without the session check would launch a
// second agent onto a task that is already running or already finished.
func (s *Service) recoverAutoStartOnCreate(ctx context.Context, task *models.Task) {
sessions, err := s.repo.ListTaskSessions(ctx, task.ID)
if err != nil {
@@ -1435,13 +1522,13 @@ func (s *Service) recoverAutoStartOnCreate(ctx context.Context, task *models.Tas
return
}
if len(sessions) > 0 {
// The task was started by one of those other paths, so this one-shot
// token is spent. Claim it rather than merely skipping, so the row
// converges instead of being re-scanned on every future startup.
if s.claimTaskEventMetadata(ctx, task, models.MetaKeyAutoStartOnCreate) {
s.logger.Info("discarded spent auto-start-on-create token: task already has a session",
zap.String("task_id", task.ID), zap.Int("session_count", len(sessions)))
}
// The task was started by one of those other paths, so both the
// intent and any in-flight hand-off are spent. Clear them rather than
// merely skipping, so the row converges instead of being re-scanned on
// every future startup.
s.completeAutoStartOnCreate(ctx, task.ID, "auto-start-on-create recovery")
s.logger.Info("discarded spent auto-start-on-create token: task already has a session",
zap.String("task_id", task.ID), zap.Int("session_count", len(sessions)))
return
}
// Re-enter handleTaskCreated exactly like a live delivery would, so the
@@ -1503,36 +1590,73 @@ func (s *Service) syncTaskStateForQueuePromotion(ctx context.Context, task *mode
return nil
}
func (s *Service) autoStartTaskForStep(ctx context.Context, taskID, stepID, eventName string, stepTransitionID int64) {
// autoStartTaskForStep evaluates a target step's on_enter auto-start action
// for a task with no session yet. autoStartOnCreateClaimed is true only when
// the caller (handleTaskCreated) already reserved the create-time launch
// marker before dispatching here. Every other caller passes false and lets
// autoStartTaskForLoadedStep claim the key itself if the task still carries
// it, so a concurrent launch attempt cannot observe the token as if nobody
// had scheduled a launch for it (see claimAutoStartOnCreateForLaunch).
//
//nolint:cyclop // the single auto-start chokepoint evaluates each lifecycle gate before dispatch
func (s *Service) autoStartTaskForStep(ctx context.Context, taskID, stepID, eventName string, stepTransitionID int64, autoStartOnCreateClaimed bool) {
task, err := s.repo.GetTask(ctx, taskID)
if err != nil {
s.logger.Warn(eventName+": failed to load task for auto-start",
zap.String("task_id", taskID), zap.Error(err))
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
if task == nil || task.QueuedForStepID != "" {
if task == nil {
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
if task.QueuedForStepID != "" {
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
// Dependency gate. Sits here — the single automated-launch chokepoint — so
// one check covers task.moved, task.queue_promoted, watcher auto-start and
// dependency resolution itself. Placed BEFORE launchDeferredTask so a
// blocked task neither starts a session nor consumes its launch intent.
if s.dependencyBlocksAutoStart(ctx, taskID, eventName) {
// blocked task never starts a session. A genuine block still consumes the
// caller's launch intent: evaluateDependentAfterPredecessorChange and
// reconcileDependencyLaunchesOnStartup independently launch this task once
// its dependency resolves. A failed dependency read has no such fallback,
// so it restores the token instead of stranding it.
if blocked, gateErrored := s.dependencyBlocksAutoStart(ctx, taskID, eventName); blocked {
if gateErrored && autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
} else if autoStartOnCreateClaimed {
s.discardAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
if hasQueuePromotionPending(task) {
// A dependency may resolve after same-step WIP promotion was admitted.
// Resume through the promotion handler so destination-entry lifecycle is
// claimed and its token participates in deferred-launch failure recovery.
s.handleTaskQueuePromoted(ctx, watcher.TaskEventData{
// autoStartOnCreateClaimed carries forward here: if handleTaskCreated
// already claimed MetaKeyAutoStartOnCreate for this attempt, the
// promotion path must inherit that ownership rather than starting a
// fresh, unclaimed one (see claimAutoStartOnCreateForLaunch).
s.handleTaskQueuePromotedWithAutoStartOnCreateClaimed(ctx, watcher.TaskEventData{
TaskID: task.ID, StepTransitionID: stepTransitionID,
})
}, autoStartOnCreateClaimed)
return
}
if task != nil && s.shouldSkipTerminalPRAutoStart(ctx, task) {
if autoStartOnCreateClaimed {
s.discardAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
if s.launchDeferredTask(ctx, task, eventName, false) {
if s.launchDeferredTask(ctx, task, eventName, false, autoStartOnCreateClaimed) {
return
}
@@ -1543,30 +1667,230 @@ func (s *Service) autoStartTaskForStep(ctx context.Context, taskID, stepID, even
zap.String("task_id", taskID),
zap.String("to_step_id", stepID),
zap.Error(err))
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
}
return
}
s.autoStartTaskForLoadedStep(ctx, task, step, eventName, false, stepTransitionID)
s.autoStartTaskForLoadedStep(ctx, task, step, eventName, false, stepTransitionID, autoStartOnCreateClaimed)
}
func (s *Service) autoStartTaskForLoadedStep(ctx context.Context, task *models.Task, step *wfmodels.WorkflowStep, eventName string, restoreQueuePromotion bool, stepTransitionID int64) {
if task == nil || task.QueuedForStepID != "" || step == nil {
// autoStartLaunchTokens carries the one-shot lifecycle tokens a launch
// attempt must restore if it fails before producing a durable session or
// run, so the next startup sweep retries instead of the task silently
// losing its opt-in.
type autoStartLaunchTokens struct {
hasGuard bool
restoreQueuePromotion bool
queuePromotionToken interface{}
restoreAutoStartOnCreate bool
}
type autoStartOnCreateClaimResult uint8
const (
autoStartOnCreateNoIntent autoStartOnCreateClaimResult = iota
autoStartOnCreateClaimOwned
autoStartOnCreateClaimLost
autoStartOnCreateClaimError
)
// claimAutoStartOnCreateForLaunch reserves the create-time launch intent
// before a detached launch starts. The original intent is removed only after
// the durable in-flight marker is written. A second caller therefore sees a
// lost claim, while a process restart can recover the in-flight marker.
//
//nolint:cyclop // durable claim/recovery ordering has separate stale, write, and ownership outcomes
func (s *Service) claimAutoStartOnCreateForLaunch(ctx context.Context, task *models.Task, alreadyClaimed bool) autoStartOnCreateClaimResult {
if task == nil {
return autoStartOnCreateClaimError
}
if alreadyClaimed {
s.autoStartOnCreateMu.Lock()
if s.autoStartOnCreateInFlight == nil {
s.autoStartOnCreateInFlight = make(map[string]struct{})
}
s.autoStartOnCreateInFlight[task.ID] = struct{}{}
s.autoStartOnCreateMu.Unlock()
return autoStartOnCreateClaimOwned
}
s.autoStartOnCreateMu.Lock()
defer s.autoStartOnCreateMu.Unlock()
if s.autoStartOnCreateInFlight == nil {
s.autoStartOnCreateInFlight = make(map[string]struct{})
}
if _, inFlight := s.autoStartOnCreateInFlight[task.ID]; inFlight {
return autoStartOnCreateClaimLost
}
intent := models.HasAutoStartOnCreateIntent(task.Metadata)
staleInFlight := models.HasAutoStartOnCreateInFlight(task.Metadata)
if !intent && !staleInFlight {
return autoStartOnCreateNoIntent
}
setter, setterOK := s.repo.(taskMetadataKeySetter)
remover, removerOK := s.repo.(taskMetadataKeyRemover)
if !setterOK || !removerOK {
s.logger.Warn("auto-start-on-create claim unavailable: repository lacks metadata primitives",
zap.String("task_id", task.ID))
return autoStartOnCreateClaimError
}
if staleInFlight {
// An in-flight marker without a local owner belongs to a previous
// process, or to a claim that crashed between its two writes. Rebuild
// the original intent before taking a fresh reservation.
if !intent {
if err := setter.SetTaskMetadataKey(ctx, task.ID, models.MetaKeyAutoStartOnCreate, true); err != nil {
s.logger.Warn("failed to restore stale auto-start-on-create intent",
zap.String("task_id", task.ID), zap.Error(err))
return autoStartOnCreateClaimError
}
}
if _, err := remover.RemoveTaskMetadataKey(ctx, task.ID, models.MetaKeyAutoStartOnCreateInFlight); err != nil {
s.logger.Warn("failed to clear stale auto-start-on-create marker",
zap.String("task_id", task.ID), zap.Error(err))
return autoStartOnCreateClaimError
}
}
if err := setter.SetTaskMetadataKey(ctx, task.ID, models.MetaKeyAutoStartOnCreateInFlight, true); err != nil {
s.logger.Warn("failed to persist auto-start-on-create in-flight marker",
zap.String("task_id", task.ID), zap.Error(err))
return autoStartOnCreateClaimError
}
claimed, err := remover.RemoveTaskMetadataKey(ctx, task.ID, models.MetaKeyAutoStartOnCreate)
if err != nil {
s.logger.Warn("failed to claim auto-start-on-create intent",
zap.String("task_id", task.ID), zap.Error(err))
return autoStartOnCreateClaimError
}
if !claimed {
_, _ = remover.RemoveTaskMetadataKey(ctx, task.ID, models.MetaKeyAutoStartOnCreateInFlight)
return autoStartOnCreateClaimLost
}
s.autoStartOnCreateInFlight[task.ID] = struct{}{}
return autoStartOnCreateClaimOwned
}
func (s *Service) clearAutoStartOnCreateInFlight(taskID string) {
s.autoStartOnCreateMu.Lock()
delete(s.autoStartOnCreateInFlight, taskID)
s.autoStartOnCreateMu.Unlock()
}
func (s *Service) ownsAutoStartOnCreateInFlight(taskID string) bool {
s.autoStartOnCreateMu.Lock()
defer s.autoStartOnCreateMu.Unlock()
_, owned := s.autoStartOnCreateInFlight[taskID]
return owned
}
// restoreAutoStartOnCreate restores a failed reservation and leaves a
// durable intent for the next lifecycle sweep. The original intent is
// written before the in-flight marker is removed, so either write ordering
// still leaves recovery work after a process exit.
func (s *Service) restoreAutoStartOnCreate(ctx context.Context, taskID, eventName string) {
setter, setterOK := s.repo.(taskMetadataKeySetter)
remover, removerOK := s.repo.(taskMetadataKeyRemover)
if !setterOK || !removerOK {
s.logger.Warn(eventName+": repository cannot restore auto-start-on-create reservation",
zap.String("task_id", taskID))
s.clearAutoStartOnCreateInFlight(taskID)
return
}
if err := setter.SetTaskMetadataKey(ctx, taskID, models.MetaKeyAutoStartOnCreate, true); err != nil {
s.logger.Warn(eventName+": failed to restore auto-start-on-create intent",
zap.String("task_id", taskID), zap.Error(err))
s.clearAutoStartOnCreateInFlight(taskID)
return
}
if _, err := remover.RemoveTaskMetadataKey(ctx, taskID, models.MetaKeyAutoStartOnCreateInFlight); err != nil {
s.logger.Warn(eventName+": failed to clear auto-start-on-create in-flight marker",
zap.String("task_id", taskID), zap.Error(err))
}
s.clearAutoStartOnCreateInFlight(taskID)
}
// discardAutoStartOnCreate consumes a reservation when the task no longer
// needs a create-time launch (for example, a terminal PR or a step without
// auto-start). It is intentionally separate from restoreAutoStartOnCreate.
func (s *Service) discardAutoStartOnCreate(ctx context.Context, taskID, eventName string) {
if remover, ok := s.repo.(taskMetadataKeyRemover); ok {
if _, err := remover.RemoveTaskMetadataKey(ctx, taskID, models.MetaKeyAutoStartOnCreateInFlight); err != nil {
s.logger.Warn(eventName+": failed to clear auto-start-on-create in-flight marker",
zap.String("task_id", taskID), zap.Error(err))
}
}
s.clearAutoStartOnCreateInFlight(taskID)
}
// completeAutoStartOnCreate clears both lifecycle markers after a session or
// run is durable. A session-existence check in startup recovery also calls
// this helper, which makes cleanup idempotent after a process restart.
func (s *Service) completeAutoStartOnCreate(ctx context.Context, taskID, eventName string) {
if remover, ok := s.repo.(taskMetadataKeyRemover); ok {
for _, key := range []string{models.MetaKeyAutoStartOnCreate, models.MetaKeyAutoStartOnCreateInFlight} {
if _, err := remover.RemoveTaskMetadataKey(ctx, taskID, key); err != nil {
s.logger.Warn(eventName+": failed to clear auto-start-on-create marker",
zap.String("task_id", taskID), zap.String("metadata_key", key), zap.Error(err))
}
}
}
s.clearAutoStartOnCreateInFlight(taskID)
}
//nolint:cyclop,gocognit,funlen // launch dispatch combines existing workflow gates with durable token ownership
func (s *Service) autoStartTaskForLoadedStep(ctx context.Context, task *models.Task, step *wfmodels.WorkflowStep, eventName string, restoreQueuePromotion bool, stepTransitionID int64, autoStartOnCreateClaimed bool) {
if task == nil || task.QueuedForStepID != "" || step == nil {
if autoStartOnCreateClaimed && task != nil {
s.restoreAutoStartOnCreate(ctx, task.ID, eventName)
}
return
}
if models.HasAutoStartOnCreateIntent(task.Metadata) || models.HasAutoStartOnCreateInFlight(task.Metadata) || autoStartOnCreateClaimed {
sessions, err := s.repo.ListTaskSessions(ctx, task.ID)
if err != nil {
s.logger.Warn(eventName+": failed to check existing sessions before auto-start-on-create claim",
zap.String("task_id", task.ID), zap.Error(err))
if autoStartOnCreateClaimed {
s.restoreAutoStartOnCreate(ctx, task.ID, eventName)
}
return
}
if len(sessions) > 0 {
s.completeAutoStartOnCreate(ctx, task.ID, eventName)
return
}
}
if s.shouldSkipTerminalPRAutoStart(ctx, task) {
if autoStartOnCreateClaimed {
s.discardAutoStartOnCreate(ctx, task.ID, eventName)
}
return
}
if s.launchDeferredTask(ctx, task, eventName, restoreQueuePromotion) {
if s.launchDeferredTask(ctx, task, eventName, restoreQueuePromotion, autoStartOnCreateClaimed) {
return
}
if step == nil || !step.HasOnEnterAction(wfmodels.OnEnterAutoStartAgent) {
s.logger.Debug(eventName+": target step has no auto-start",
zap.String("task_id", task.ID),
zap.String("to_step_id", step.ID))
if autoStartOnCreateClaimed {
s.discardAutoStartOnCreate(ctx, task.ID, eventName)
}
return
}
claimResult := s.claimAutoStartOnCreateForLaunch(ctx, task, autoStartOnCreateClaimed)
if claimResult == autoStartOnCreateClaimLost || claimResult == autoStartOnCreateClaimError {
s.logger.Debug(eventName+": auto-start-on-create claim did not win",
zap.String("task_id", task.ID), zap.Uint8("claim_result", uint8(claimResult)))
return
}
restoreAutoStartOnCreate := claimResult == autoStartOnCreateClaimOwned
if s.isOfficeTask(ctx, task.ID) {
s.autoStartOfficeTaskForLoadedStep(ctx, task, step, eventName, restoreQueuePromotion, stepTransitionID)
s.autoStartOfficeTaskForLoadedStep(ctx, task, step, eventName, restoreQueuePromotion, stepTransitionID, restoreAutoStartOnCreate)
return
}
@@ -1615,7 +1939,16 @@ func (s *Service) autoStartTaskForLoadedStep(ctx context.Context, task *models.T
s.logger.Error(eventName+": failed to auto-start task",
zap.String("task_id", task.ID),
zap.Error(err))
s.handleAutoStartFailure(asyncCtx, task.ID, eventName, hasGuard, restoreQueuePromotion, queuePromotionToken)
s.handleAutoStartFailure(asyncCtx, task.ID, eventName, autoStartLaunchTokens{
hasGuard: hasGuard,
restoreQueuePromotion: restoreQueuePromotion,
queuePromotionToken: queuePromotionToken,
restoreAutoStartOnCreate: restoreAutoStartOnCreate,
})
return
}
if restoreAutoStartOnCreate {
s.completeAutoStartOnCreate(asyncCtx, task.ID, eventName)
}
}()
}
@@ -1627,7 +1960,7 @@ func (s *Service) autoStartTaskForLoadedStep(ctx context.Context, task *models.T
// a run through the engine's Office adapters instead, the same mechanism the
// scheduler's recovery sweep and the "assign task" flow already use to start
// Office work (internal/office/service/scheduler_recovery.go).
func (s *Service) autoStartOfficeTaskForLoadedStep(ctx context.Context, task *models.Task, step *wfmodels.WorkflowStep, eventName string, restoreQueuePromotion bool, stepTransitionID int64) {
func (s *Service) autoStartOfficeTaskForLoadedStep(ctx context.Context, task *models.Task, step *wfmodels.WorkflowStep, eventName string, restoreQueuePromotion bool, stepTransitionID int64, restoreAutoStartOnCreate bool) {
queuePromotionToken := queuePromotionLifecycleToken(task)
// Async for the same reason as the kanban path above: the event bus
// delivers synchronously and blocking here would stall the HTTP handler
@@ -1649,7 +1982,12 @@ func (s *Service) autoStartOfficeTaskForLoadedStep(ctx context.Context, task *mo
s.logger.Error(eventName+": failed to queue office auto-start run",
zap.String("task_id", task.ID),
zap.Error(err))
s.handleAutoStartFailure(asyncCtx, task.ID, eventName, hasGuard, restoreQueuePromotion, queuePromotionToken)
s.handleAutoStartFailure(asyncCtx, task.ID, eventName, autoStartLaunchTokens{
hasGuard: hasGuard,
restoreQueuePromotion: restoreQueuePromotion,
queuePromotionToken: queuePromotionToken,
restoreAutoStartOnCreate: restoreAutoStartOnCreate,
})
return
}
// Log the outcome only after the attempt actually resolves — the log
@@ -1663,12 +2001,18 @@ func (s *Service) autoStartOfficeTaskForLoadedStep(ctx context.Context, task *mo
s.logger.Info(eventName+": queued office run (no session, auto-start step)",
zap.String("task_id", task.ID),
zap.String("to_step_id", step.ID))
if restoreAutoStartOnCreate {
s.completeAutoStartOnCreate(asyncCtx, task.ID, eventName)
}
return
}
s.logger.Debug(eventName+": office auto-start run not queued",
zap.String("task_id", task.ID),
zap.String("to_step_id", step.ID),
zap.String("outcome", string(outcome)))
if restoreAutoStartOnCreate {
s.completeAutoStartOnCreate(asyncCtx, task.ID, eventName)
}
}()
}
@@ -1727,20 +2071,27 @@ func officeAutoStartIdempotencyKey(task *models.Task, agentProfileID, stepID str
// only ever called (and the token only ever taken) when the task carries
// MetaKeyAutoStartGuard, so restoring it unconditionally stamps
// auto_start_claimed onto tasks that never carried the guard, corrupting the
// invariant documented at MetaKeyAutoStartGuard. It also stamps
// MetaKeyAutoStartFailed so the failure surfaces on the task card instead of
// only in backend logs.
func (s *Service) handleAutoStartFailure(ctx context.Context, taskID, eventName string, hasGuard, restoreQueuePromotion bool, queuePromotionToken interface{}) {
if hasGuard {
// invariant documented at MetaKeyAutoStartGuard. It also restores
// MetaKeyAutoStartOnCreate when this attempt claimed it, so a StartTask/queue
// failure before a durable session or run exists leaves the task retryable by
// the next startup sweep instead of stranded with no marker at all. It also
// stamps MetaKeyAutoStartFailed so the failure surfaces on the task card
// instead of only in backend logs.
func (s *Service) handleAutoStartFailure(ctx context.Context, taskID, eventName string, tokens autoStartLaunchTokens) {
if tokens.hasGuard {
s.restoreAutoStartClaim(ctx, taskID, eventName)
}
if restoreQueuePromotion {
s.restoreTaskLifecycleToken(ctx, taskID, models.MetaKeyQueuePromotionPending, queuePromotionToken, eventName)
if tokens.restoreQueuePromotion {
s.restoreTaskLifecycleToken(ctx, taskID, models.MetaKeyQueuePromotionPending, tokens.queuePromotionToken, eventName)
}
if tokens.restoreAutoStartOnCreate {
s.restoreAutoStartOnCreate(ctx, taskID, eventName)
}
s.setTaskAutoStartFailedMarker(ctx, taskID, eventName)
}
func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eventName string, restoreQueuePromotion bool) bool {
//nolint:cyclop,gocognit,funlen // deferred launches coordinate two durable tokens and async success/failure cleanup
func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eventName string, restoreQueuePromotion bool, autoStartOnCreateClaimed ...bool) bool {
if task.Metadata == nil {
return false
}
@@ -1750,6 +2101,9 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve
}
encoded, err := json.Marshal(raw)
if err != nil {
if len(autoStartOnCreateClaimed) > 0 && autoStartOnCreateClaimed[0] {
s.restoreAutoStartOnCreate(ctx, task.ID, eventName)
}
return false
}
var intent struct {
@@ -1765,8 +2119,26 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve
}
if err := json.Unmarshal(encoded, &intent); err != nil || intent.AgentProfileID == "" {
s.logger.Warn(eventName+": invalid deferred launch intent", zap.String("task_id", task.ID), zap.Error(err))
if len(autoStartOnCreateClaimed) > 0 && autoStartOnCreateClaimed[0] {
s.restoreAutoStartOnCreate(ctx, task.ID, eventName)
}
return true
}
createClaimed := false
requestedCreateClaim := len(autoStartOnCreateClaimed) > 0 && autoStartOnCreateClaimed[0]
if requestedCreateClaim || models.HasAutoStartOnCreateIntent(task.Metadata) || models.HasAutoStartOnCreateInFlight(task.Metadata) {
claimResult := s.claimAutoStartOnCreateForLaunch(ctx, task, requestedCreateClaim)
switch claimResult {
case autoStartOnCreateClaimOwned:
createClaimed = true
case autoStartOnCreateNoIntent:
if requestedCreateClaim {
return true
}
default:
return true
}
}
launchIntent := IntentStart
if intent.Intent == "prepare" {
launchIntent = IntentPrepare
@@ -1775,6 +2147,9 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve
launchCtx := context.WithoutCancel(ctx)
metadataClaimed, claimOK := s.claimDeferredLaunch(launchCtx, task.ID, eventName)
if !claimOK {
if createClaimed {
s.restoreAutoStartOnCreate(launchCtx, task.ID, eventName)
}
return
}
launchResp, launchErr := s.LaunchSession(launchCtx, &LaunchSessionRequest{
@@ -1792,8 +2167,14 @@ func (s *Service) launchDeferredTask(ctx context.Context, task *models.Task, eve
if restoreQueuePromotion {
s.restoreTaskLifecycleToken(launchCtx, task.ID, models.MetaKeyQueuePromotionPending, queuePromotionLifecycleToken(task), eventName)
}
if createClaimed {
s.restoreAutoStartOnCreate(launchCtx, task.ID, eventName)
}
return
}
if createClaimed {
s.completeAutoStartOnCreate(launchCtx, task.ID, eventName)
}
if intent.RecordRecentUse {
s.recordSuccessfulDeferredTaskProfileAsync(launchCtx, intent.UserID, launchResp.AgentProfileID)
}
@@ -0,0 +1,131 @@
package orchestrator
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/task/models"
wfmodels "github.com/kandev/kandev/internal/workflow/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
type failAutoStartOnCreateClaimRepository struct {
sessionExecutorStore
}
func (r *failAutoStartOnCreateClaimRepository) RemoveTaskMetadataKey(ctx context.Context, taskID, key string) (bool, error) {
if key == models.MetaKeyAutoStartOnCreate {
return false, errors.New("injected auto-start claim failure")
}
return r.sessionExecutorStore.RemoveTaskMetadataKey(ctx, taskID, key)
}
func TestAutoStartOnCreateClaimLeavesDurableInFlightMarker(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-inflight", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-inflight", WorkspaceID: "ws-inflight", Name: "WF", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "task-inflight", WorkspaceID: "ws-inflight", WorkflowID: "wf-inflight",
WorkflowStepID: "step-inflight", State: v1.TaskStateCreated,
Metadata: map[string]interface{}{models.MetaKeyAutoStartOnCreate: true},
CreatedAt: now, UpdatedAt: now,
}))
svc := &Service{repo: repo, logger: testLogger()}
task, err := repo.GetTask(ctx, "task-inflight")
requireNoError(t, err)
if got := svc.claimAutoStartOnCreateForLaunch(ctx, task, false); got != autoStartOnCreateClaimOwned {
t.Fatalf("claim result = %d, want owned", got)
}
reloaded, err := repo.GetTask(ctx, task.ID)
requireNoError(t, err)
if models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("auto-start-on-create intent remained after claim")
}
if !models.HasAutoStartOnCreateInFlight(reloaded.Metadata) {
t.Fatal("in-flight marker was not persisted before launch")
}
if got := svc.claimAutoStartOnCreateForLaunch(ctx, reloaded, false); got != autoStartOnCreateClaimLost {
t.Fatalf("second claim result = %d, want lost", got)
}
svc.completeAutoStartOnCreate(ctx, task.ID, "test")
reloaded, err = repo.GetTask(ctx, task.ID)
requireNoError(t, err)
if models.HasAutoStartOnCreateIntent(reloaded.Metadata) || models.HasAutoStartOnCreateInFlight(reloaded.Metadata) {
t.Fatalf("auto-start markers remained after completion: %#v", reloaded.Metadata)
}
}
func TestAutoStartOnCreateClaimReclaimsStaleInFlightMarker(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-stale", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-stale", WorkspaceID: "ws-stale", Name: "WF", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "task-stale", WorkspaceID: "ws-stale", WorkflowID: "wf-stale",
WorkflowStepID: "step-stale", State: v1.TaskStateCreated,
Metadata: map[string]interface{}{models.MetaKeyAutoStartOnCreateInFlight: true},
CreatedAt: now, UpdatedAt: now,
}))
svc := &Service{repo: repo, logger: testLogger()}
task, err := repo.GetTask(ctx, "task-stale")
requireNoError(t, err)
if got := svc.claimAutoStartOnCreateForLaunch(ctx, task, false); got != autoStartOnCreateClaimOwned {
t.Fatalf("stale claim result = %d, want owned", got)
}
reloaded, err := repo.GetTask(ctx, task.ID)
requireNoError(t, err)
if models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("stale recovery left the rebuilt intent in place")
}
if !models.HasAutoStartOnCreateInFlight(reloaded.Metadata) {
t.Fatal("stale recovery did not create a fresh in-flight marker")
}
}
func TestAutoStartOnCreateClaimFailureSkipsLaunch(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-claim-error", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-claim-error", WorkspaceID: "ws-claim-error", Name: "WF", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "task-claim-error", WorkspaceID: "ws-claim-error", WorkflowID: "wf-claim-error",
WorkflowStepID: "step-claim-error", State: v1.TaskStateCreated,
Metadata: map[string]interface{}{models.MetaKeyAutoStartOnCreate: true},
CreatedAt: now, UpdatedAt: now,
}))
step := &wfmodels.WorkflowStep{
ID: "step-claim-error", WorkflowID: "wf-claim-error", Name: "Start", Position: 0,
Events: wfmodels.StepEvents{OnEnter: []wfmodels.OnEnterAction{{Type: wfmodels.OnEnterAutoStartAgent}}},
}
stepGetter := newMockStepGetter()
stepGetter.steps[step.ID] = step
failingRepo := &failAutoStartOnCreateClaimRepository{sessionExecutorStore: repo}
svc := &Service{repo: failingRepo, workflowStepGetter: stepGetter, logger: testLogger()}
task, err := repo.GetTask(ctx, "task-claim-error")
requireNoError(t, err)
// The claim must fail closed before any goroutine or executor call. The
// executor field is deliberately unset, so a launch would panic this test.
svc.autoStartTaskForLoadedStep(ctx, task, step, "test.claim_error", false, 0, false)
reloaded, err := repo.GetTask(ctx, task.ID)
requireNoError(t, err)
if !models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("claim failure removed the original intent")
}
if !models.HasAutoStartOnCreateInFlight(reloaded.Metadata) {
t.Fatal("claim failure did not leave a recoverable in-flight marker")
}
}
@@ -0,0 +1,552 @@
package orchestrator
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/orchestrator/watcher"
"github.com/kandev/kandev/internal/task/models"
taskservice "github.com/kandev/kandev/internal/task/service"
wfmodels "github.com/kandev/kandev/internal/workflow/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// This file covers the two latent auto-start recovery defects filed as a
// sibling of WO-36.1: a replay double-launch race between the queue-promotion
// lifecycle token and the create-time auto-start opt-in (Gap A), and a
// pre-session StartTask failure that stranded the create-time opt-in with no
// durable marker (Gap B). Both are fixed by the same mechanism: the opt-in
// is claimed synchronously by the launch attempt itself (autoStartTaskForLoadedStep,
// via claimAutoStartOnCreateForLaunch), not merely observed, and restored on
// failure (handleAutoStartFailure) exactly like the existing queue-promotion
// token.
// TestRecoverTaskLifecycleAttemptConcurrentPromotionAndAutoStartOnCreateLaunchesOnce
// covers Gap A: recoverTaskLifecycleAttempt used to evaluate
// autoStartOnCreateActionable against the SAME in-memory task it loaded at
// the top of the function, without re-fetching after the queue-promotion
// branch ran. handleTaskQueuePromoted's no-session path schedules a launch
// through autoStartTaskForLoadedStep, but that launch's actual session
// creation happens in a detached goroutine — so a stale read could still see
// MetaKeyAutoStartOnCreate present and schedule a second, redundant launch via
// recoverAutoStartOnCreate.
//
// The fix makes the race structural rather than timing-dependent:
// autoStartTaskForLoadedStep claims MetaKeyAutoStartOnCreate synchronously,
// before spawning its launch goroutine, and recoverTaskLifecycleAttempt
// re-fetches the task after the promotion branch. So by the time
// autoStartOnCreateActionable evaluates, the token is already gone.
func TestRecoverTaskLifecycleAttemptConcurrentPromotionAndAutoStartOnCreateLaunchesOnce(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws1", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf1", WorkspaceID: "ws1", Name: "WF", CreatedAt: now, UpdatedAt: now}))
metadata := map[string]interface{}{
models.MetaKeyQueuePromotionPending: true,
models.MetaKeyAutoStartOnCreate: true,
models.MetaKeyAgentProfileID: "routine-assignee",
}
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-race",
WorkspaceID: "ws1",
WorkflowID: "wf1",
WorkflowStepID: "step1",
WIPAdmitted: true,
Title: "Concurrent tokens",
Description: "prompt",
State: v1.TaskStateTODO,
Metadata: metadata,
CreatedAt: now,
UpdatedAt: now,
}))
stepGetter := newMockStepGetter()
stepGetter.steps["step1"] = &wfmodels.WorkflowStep{
ID: "step1", WorkflowID: "wf1", Name: "Auto Start Step", Position: 0,
Events: wfmodels.StepEvents{
OnEnter: []wfmodels.OnEnterAction{
{Type: wfmodels.OnEnterAutoStartAgent},
},
},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["t-race"] = &v1.Task{
ID: "t-race",
WorkspaceID: "ws1",
WorkflowID: "wf1",
Description: "prompt",
State: v1.TaskStateTODO,
Metadata: metadata,
}
launched := make(chan string, 4)
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
agentProfileID, _ := req.Metadata[models.MetaKeyAgentProfileID].(string)
launched <- agentProfileID
return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil
},
}
svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr)
svc.recoverTaskLifecycleAttempt(ctx, "t-race")
select {
case got := <-launched:
if got != "routine-assignee" {
t.Fatalf("AgentProfileID = %q, want routine-assignee", got)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for the auto-start launch")
}
// Settle window: prove no second launch follows the first, rather than
// just reading one value off the channel and declaring victory.
select {
case got := <-launched:
t.Fatalf("unexpected second launch for agent profile %q", got)
case <-time.After(300 * time.Millisecond):
}
sessions, err := repo.ListTaskSessions(ctx, "t-race")
requireNoError(t, err)
if len(sessions) != 1 {
t.Fatalf("ListTaskSessions(t-race) = %d sessions, want 1", len(sessions))
}
}
// TestRecoverTaskLifecycleAttemptConcurrentPromotionLaunchesOnceWithoutWorkspaceBindingElection
// covers the same Gap A race, but on the session-create route that does NOT
// go through CreateTaskSessionWithWorkspaceBinding's election
// (executor_execute.go: bindWorkspace && !taskUsesDeferredEnvironmentInheritance).
// Today that election is what accidentally limits the race to one launch on
// the ordinary path: the loser's session-create call fails with "workspace is
// preparing", and handleAutoStartFailure restores the promotion token for the
// next sweep to converge. A task using workspace.mode=inherit_parent takes the
// CreateTaskSessionWithInitialRuntimeSeed / plain CreateTaskSession route
// instead, neither of which enforces per-task session uniqueness — so this is
// the test that proves the actual fix (the synchronous token claim), not the
// incidental mitigation.
func TestRecoverTaskLifecycleAttemptConcurrentPromotionLaunchesOnceWithoutWorkspaceBindingElection(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws1", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf1", WorkspaceID: "ws1", Name: "WF", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-race-parent", WorkspaceID: "ws1", WorkflowID: "wf1", WorkflowStepID: "step1",
Title: "Parent", State: v1.TaskStateInProgress, CreatedAt: now, UpdatedAt: now,
}))
// ExecutorType intentionally left empty: the child launch below resolves
// no executor either (no MetaKeyExecutorID, no default configured in this
// test harness), and LaunchPreparedSession rejects an inherited
// environment whose non-empty ExecutorType disagrees with the launch's
// resolved one (executor_execute.go's workspace-reuse-unsafe guard).
requireNoError(t, repo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{
ID: "env-parent", TaskID: "t-race-parent",
Status: models.TaskEnvironmentStatusReady,
}))
requireNoError(t, repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "ps1", TaskID: "t-race-parent", State: models.TaskSessionStateRunning,
IsPrimary: true, TaskEnvironmentID: "env-parent", StartedAt: now, UpdatedAt: now,
}))
metadata := map[string]interface{}{
models.MetaKeyQueuePromotionPending: true,
models.MetaKeyAutoStartOnCreate: true,
models.MetaKeyAgentProfileID: "routine-assignee",
"workspace": map[string]interface{}{"mode": "inherit_parent"},
}
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-race-child",
ParentID: "t-race-parent",
WorkspaceID: "ws1",
WorkflowID: "wf1",
WorkflowStepID: "step1",
WIPAdmitted: true,
Title: "Concurrent tokens, inherited workspace",
Description: "prompt",
State: v1.TaskStateTODO,
Metadata: metadata,
CreatedAt: now,
UpdatedAt: now,
}))
stepGetter := newMockStepGetter()
stepGetter.steps["step1"] = &wfmodels.WorkflowStep{
ID: "step1", WorkflowID: "wf1", Name: "Auto Start Step", Position: 0,
Events: wfmodels.StepEvents{
OnEnter: []wfmodels.OnEnterAction{
{Type: wfmodels.OnEnterAutoStartAgent},
},
},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["t-race-child"] = &v1.Task{
ID: "t-race-child",
ParentID: "t-race-parent",
WorkspaceID: "ws1",
WorkflowID: "wf1",
Description: "prompt",
State: v1.TaskStateTODO,
Metadata: metadata,
}
launched := make(chan string, 4)
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
agentProfileID, _ := req.Metadata[models.MetaKeyAgentProfileID].(string)
launched <- agentProfileID
return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil
},
}
svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr)
svc.recoverTaskLifecycleAttempt(ctx, "t-race-child")
select {
case got := <-launched:
if got != "routine-assignee" {
t.Fatalf("AgentProfileID = %q, want routine-assignee", got)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for the auto-start launch")
}
select {
case got := <-launched:
t.Fatalf("unexpected second launch for agent profile %q", got)
case <-time.After(300 * time.Millisecond):
}
sessions, err := repo.ListTaskSessions(ctx, "t-race-child")
requireNoError(t, err)
if len(sessions) != 1 {
t.Fatalf("ListTaskSessions(t-race-child) = %d sessions, want 1", len(sessions))
}
if sessions[0].TaskEnvironmentID != "env-parent" {
t.Fatalf("session TaskEnvironmentID = %q, want env-parent (inherited, non-bindWorkspace route confirms this isn't the workspace-binding election doing the work)", sessions[0].TaskEnvironmentID)
}
}
// TestHandleTaskCreatedRestoresAutoStartOnCreateOnPreSessionFailure covers Gap
// B: handleTaskCreated claims (removes) MetaKeyAutoStartOnCreate synchronously
// before handing the launch to autoStartTaskForLoadedStep's detached
// goroutine. Before this fix, handleAutoStartFailure restored
// MetaKeyAutoStartGuard and MetaKeyQueuePromotionPending on failure but never
// MetaKeyAutoStartOnCreate, so a StartTask failure before a session exists
// stranded the task with no durable marker for the next startup sweep to
// find.
//
// getTaskErr fails scheduler.GetTask, which startTask calls after resolving
// the target step but before prepareSessionForStart creates any session row
// — a deterministic pre-session failure, no sleep-and-hope.
func TestHandleTaskCreatedRestoresAutoStartOnCreateOnPreSessionFailure(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws1", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf1", WorkspaceID: "ws1", Name: "WF", CreatedAt: now, UpdatedAt: now}))
metadata := map[string]interface{}{
models.MetaKeyAutoStartOnCreate: true,
models.MetaKeyAgentProfileID: "routine-assignee",
}
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-pre-session-failure",
WorkspaceID: "ws1",
WorkflowID: "wf1",
WorkflowStepID: "step1",
Title: "Routine run",
Description: "prompt",
State: v1.TaskStateCreated,
Metadata: metadata,
CreatedAt: now,
UpdatedAt: now,
}))
stepGetter := newMockStepGetter()
stepGetter.steps["step1"] = &wfmodels.WorkflowStep{
ID: "step1", WorkflowID: "wf1", Name: "Routine Start", Position: 0,
Events: wfmodels.StepEvents{
OnEnter: []wfmodels.OnEnterAction{
{Type: wfmodels.OnEnterAutoStartAgent},
},
},
}
launched := make(chan string, 2)
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
agentProfileID, _ := req.Metadata[models.MetaKeyAgentProfileID].(string)
launched <- agentProfileID
return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil
},
}
taskRepo := newMockTaskRepo()
taskRepo.getTaskErr = errors.New("boom: transient task lookup failure")
svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr)
svc.handleTaskCreated(ctx, watcher.TaskEventData{TaskID: "t-pre-session-failure"})
waitForAutoStartFailedMarker(t, repo, ctx, "t-pre-session-failure")
select {
case got := <-launched:
t.Fatalf("unexpected launch despite pre-session task lookup failure: %q", got)
default:
}
reloaded, err := repo.GetTask(ctx, "t-pre-session-failure")
requireNoError(t, err)
if !models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("MetaKeyAutoStartOnCreate was not restored after a pre-session StartTask failure; the task is stranded with no durable marker for the next startup sweep")
}
sessions, err := repo.ListTaskSessions(ctx, "t-pre-session-failure")
requireNoError(t, err)
if len(sessions) != 0 {
t.Fatalf("ListTaskSessions = %d, want 0 (the injected failure must happen before session persistence)", len(sessions))
}
// Clear the injected failure and let a subsequent startup sweep retry
// using the restored token. Locked because the first launch attempt's
// detached goroutine (see autoStartTaskForLoadedStep) can still be
// finishing unrelated work on taskRepo after waitForAutoStartFailedMarker
// observes the marker on the separate repo above; GetTask takes the same
// lock on its read side.
taskRepo.mu.Lock()
taskRepo.getTaskErr = nil
taskRepo.tasks["t-pre-session-failure"] = &v1.Task{
ID: "t-pre-session-failure",
WorkspaceID: "ws1",
WorkflowID: "wf1",
Description: "prompt",
State: v1.TaskStateCreated,
Metadata: metadata,
}
taskRepo.mu.Unlock()
svc.reconcileTaskLifecycleTokens(ctx)
select {
case got := <-launched:
if got != "routine-assignee" {
t.Fatalf("AgentProfileID = %q, want routine-assignee", got)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for the retried auto-start launch")
}
}
// TestHandleTaskCreatedWithQueuePromotionPendingRestoresAutoStartOnCreateOnPreSessionFailure
// covers Review Round 1's F1: a task carrying BOTH MetaKeyAutoStartOnCreate and
// MetaKeyQueuePromotionPending reaches handleTaskCreated (as recoverAutoStartOnCreate's
// replay does), which claims MetaKeyAutoStartOnCreate and calls autoStartTaskForStep
// with autoStartOnCreateClaimed=true. autoStartTaskForStep sees the promotion
// token and redirects into the queue-promotion handler; before the fix that
// redirect dropped autoStartOnCreateClaimed, so claimAutoStartOnCreateForLaunch
// saw the (already-removed) key as unclaimed and a pre-session StartTask
// failure restored MetaKeyQueuePromotionPending but never
// MetaKeyAutoStartOnCreate, stranding the task on exactly the recovery path
// this card exists to close.
func TestHandleTaskCreatedWithQueuePromotionPendingRestoresAutoStartOnCreateOnPreSessionFailure(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws1", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf1", WorkspaceID: "ws1", Name: "WF", CreatedAt: now, UpdatedAt: now}))
metadata := map[string]interface{}{
models.MetaKeyAutoStartOnCreate: true,
models.MetaKeyQueuePromotionPending: true,
models.MetaKeyAgentProfileID: "routine-assignee",
}
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-promotion-and-create",
WorkspaceID: "ws1",
WorkflowID: "wf1",
WorkflowStepID: "step1",
WIPAdmitted: true,
Title: "Promoted routine run",
Description: "prompt",
State: v1.TaskStateTODO,
Metadata: metadata,
CreatedAt: now,
UpdatedAt: now,
}))
stepGetter := newMockStepGetter()
stepGetter.steps["step1"] = &wfmodels.WorkflowStep{
ID: "step1", WorkflowID: "wf1", Name: "Routine Start", Position: 0,
Events: wfmodels.StepEvents{
OnEnter: []wfmodels.OnEnterAction{
{Type: wfmodels.OnEnterAutoStartAgent},
},
},
}
launched := make(chan string, 2)
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
agentProfileID, _ := req.Metadata[models.MetaKeyAgentProfileID].(string)
launched <- agentProfileID
return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil
},
}
taskRepo := newMockTaskRepo()
taskRepo.getTaskErr = errors.New("boom: transient task lookup failure")
svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr)
svc.handleTaskCreated(ctx, watcher.TaskEventData{TaskID: "t-promotion-and-create"})
waitForAutoStartFailedMarker(t, repo, ctx, "t-promotion-and-create")
select {
case got := <-launched:
t.Fatalf("unexpected launch despite pre-session task lookup failure: %q", got)
default:
}
reloaded, err := repo.GetTask(ctx, "t-promotion-and-create")
requireNoError(t, err)
if !models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("MetaKeyAutoStartOnCreate was not restored after a pre-session StartTask failure reached through the queue-promotion redirect")
}
if _, pending := reloaded.Metadata[models.MetaKeyQueuePromotionPending]; !pending {
t.Fatal("MetaKeyQueuePromotionPending was not restored after the pre-session StartTask failure")
}
sessions, err := repo.ListTaskSessions(ctx, "t-promotion-and-create")
requireNoError(t, err)
if len(sessions) != 0 {
t.Fatalf("ListTaskSessions = %d, want 0 (the injected failure must happen before session persistence)", len(sessions))
}
}
// erroringDependencyReader models a transient DependencyGate read failure,
// distinct from a genuine dependency block: DependencyGate itself errors, so
// dependencyBlocksAutoStart fails closed with gateErrored=true regardless of
// whether the task has any dependency edges at all.
type erroringDependencyReader struct {
err error
}
func (r *erroringDependencyReader) DependencyGate(context.Context, string) (bool, string, error) {
return false, "", r.err
}
func (r *erroringDependencyReader) ListDependentTaskIDs(context.Context, string) ([]string, error) {
return nil, nil
}
func (r *erroringDependencyReader) ListPendingDependencyLaunches(context.Context) ([]taskservice.PendingDependencyLaunch, error) {
return nil, nil
}
// TestAutoStartTaskForStepRestoresAutoStartOnCreateOnDependencyGateError covers
// Review Round 2's F4: autoStartTaskForStep's dependency gate is its third
// early return, and unlike the other two (GetTask, GetStep failures, both
// already fixed), it did not restore MetaKeyAutoStartOnCreate when
// autoStartOnCreateClaimed was true. handleTaskCreated already removes the key
// before dispatching here, and the startup sweep discovers candidates only by
// that key's existence — so a DependencyGate read failure (not a genuine
// block) stranded the task permanently: it has no dependency edge for
// reconcileDependencyLaunchesOnStartup to find, and no other path re-lists it.
//
// A genuine block is deliberately NOT covered by this restore (see
// dependencyBlocksAutoStart's gateErrored split): it is already covered by
// evaluateDependentAfterPredecessorChange / reconcileDependencyLaunchesOnStartup
// once the dependency resolves, so restoring there would only burn
// recoverTaskLifecycleAttempt's bounded retry budget on every startup for as
// long as the task stays blocked.
func TestAutoStartTaskForStepRestoresAutoStartOnCreateOnDependencyGateError(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
now := time.Now().UTC()
requireNoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws1", Name: "Test", CreatedAt: now, UpdatedAt: now}))
requireNoError(t, repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf1", WorkspaceID: "ws1", Name: "WF", CreatedAt: now, UpdatedAt: now}))
metadata := map[string]interface{}{
models.MetaKeyAutoStartOnCreate: true,
models.MetaKeyAgentProfileID: "routine-assignee",
}
requireNoError(t, repo.CreateTask(ctx, &models.Task{
ID: "t-dependency-gate-error",
WorkspaceID: "ws1",
WorkflowID: "wf1",
WorkflowStepID: "step1",
Title: "Routine run",
Description: "prompt",
State: v1.TaskStateCreated,
Metadata: metadata,
CreatedAt: now,
UpdatedAt: now,
}))
stepGetter := newMockStepGetter()
stepGetter.steps["step1"] = &wfmodels.WorkflowStep{
ID: "step1", WorkflowID: "wf1", Name: "Routine Start", Position: 0,
Events: wfmodels.StepEvents{
OnEnter: []wfmodels.OnEnterAction{
{Type: wfmodels.OnEnterAutoStartAgent},
},
},
}
launched := make(chan string, 2)
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
agentProfileID, _ := req.Metadata[models.MetaKeyAgentProfileID].(string)
launched <- agentProfileID
return &executor.LaunchAgentResponse{AgentExecutionID: "exec-1"}, nil
},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["t-dependency-gate-error"] = &v1.Task{
ID: "t-dependency-gate-error",
WorkspaceID: "ws1",
WorkflowID: "wf1",
Description: "prompt",
State: v1.TaskStateCreated,
Metadata: metadata,
}
svc := createTestServiceWithScheduler(repo, stepGetter, taskRepo, agentMgr)
svc.SetTaskDependencyReader(&erroringDependencyReader{err: errors.New("boom: transient dependency gate failure")})
svc.handleTaskCreated(ctx, watcher.TaskEventData{TaskID: "t-dependency-gate-error"})
select {
case got := <-launched:
t.Fatalf("unexpected launch despite dependency gate error: %q", got)
default:
}
reloaded, err := repo.GetTask(ctx, "t-dependency-gate-error")
requireNoError(t, err)
if !models.HasAutoStartOnCreateIntent(reloaded.Metadata) {
t.Fatal("MetaKeyAutoStartOnCreate was not restored after a DependencyGate error; the task is stranded with no dependency edge for reconcileDependencyLaunchesOnStartup to find and no other marker for the next sweep")
}
sessions, err := repo.ListTaskSessions(ctx, "t-dependency-gate-error")
requireNoError(t, err)
if len(sessions) != 0 {
t.Fatalf("ListTaskSessions = %d, want 0 (the dependency gate must block before any session is created)", len(sessions))
}
// Clear the injected failure and let a subsequent startup sweep retry using
// the restored token.
svc.SetTaskDependencyReader(&resolvedDependencyReader{})
svc.reconcileTaskLifecycleTokens(ctx)
select {
case got := <-launched:
if got != "routine-assignee" {
t.Fatalf("AgentProfileID = %q, want routine-assignee", got)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for the retried auto-start launch")
}
}
@@ -628,6 +628,13 @@ type Service struct {
passthroughDispatchMu sync.Mutex
passthroughDispatches map[string]map[*passthroughDispatchToken]struct{}
// autoStartOnCreateMu serializes the local ownership hand-off for the
// durable auto-start-on-create marker. The database marker survives a
// process restart; this map prevents recovery and event delivery from
// reclaiming the same marker while its detached launch is still running.
autoStartOnCreateMu sync.Mutex
autoStartOnCreateInFlight map[string]struct{}
// Message creator for saving agent responses
messageCreator MessageCreator
@@ -1546,6 +1553,7 @@ func NewService(
executor: exec,
scheduler: sched,
messageQueue: msgQueue,
autoStartOnCreateInFlight: make(map[string]struct{}),
taskLaunchRecoveryRepo: taskLaunchRecoveryRepo,
clarificationWatchdogTimeout: 15 * time.Second,
gitSnapshotCache: newGitSnapshotCache(),
@@ -276,14 +276,26 @@ func (s *Service) isPassthroughProfile(ctx context.Context, profileID string) bo
return info.CLIPassthrough
}
// blocksAutoStartLaunch reports whether an auto-start request must be
// downgraded to a prepare, either because the task's current step does not
// allow it or because it has an unresolved dependency. The dependency gate's
// launch-token restore concern does not apply here: this path owns no
// lifecycle token to restore.
func (s *Service) blocksAutoStartLaunch(ctx context.Context, req *LaunchSessionRequest) bool {
if s.shouldBlockAutoStart(ctx, req) {
return true
}
blocked, _ := s.dependencyBlocksAutoStart(ctx, req.TaskID, "session.launch")
return blocked
}
// launchStart creates a new session and launches the agent.
// If the request is an auto-start and the task's current workflow step does not
// have auto_start_agent, or the task has unresolved dependencies, the request
// is downgraded to a prepare (workspace-only, no agent) to prevent unwanted
// auto-starts from the frontend's useAutoStartSession hook.
func (s *Service) launchStart(ctx context.Context, req *LaunchSessionRequest) (*LaunchSessionResponse, error) {
if req.AutoStart && (s.shouldBlockAutoStart(ctx, req) ||
s.dependencyBlocksAutoStart(ctx, req.TaskID, "session.launch")) {
if req.AutoStart && s.blocksAutoStartLaunch(ctx, req) {
req.LaunchWorkspace = true
return s.launchPrepare(ctx, req)
}
@@ -221,6 +221,12 @@ const (
// whose Routine workflow start step has no other transition to carry it
// into an auto_start_agent evaluation.
MetaKeyAutoStartOnCreate = "auto_start_on_create"
// MetaKeyAutoStartOnCreateInFlight is a durable hand-off marker for an
// auto-start-on-create launch. The original intent is consumed before the
// detached launch starts, but this marker remains until a session or run
// is durable. That lets startup recovery retry a process that exits in the
// gap instead of losing the last recovery signal.
MetaKeyAutoStartOnCreateInFlight = "auto_start_on_create_in_flight"
// MetaKeyStepHandoffCarry is a single-slot, task-scoped token carrying one
// consuming transition's completion handoff exactly one hop, to the next
// step's first dispatched prompt. Its value is a StepHandoffCarryToken.
@@ -346,6 +352,15 @@ func HasAutoStartOnCreateIntent(metadata map[string]interface{}) bool {
return ok && intent
}
// HasAutoStartOnCreateInFlight reports whether a launch attempt still owns
// the durable hand-off marker for a create-time auto-start. Only an explicit
// true value counts; JSON rehydration and in-process callers use the same
// representation as the other lifecycle markers.
func HasAutoStartOnCreateInFlight(metadata map[string]interface{}) bool {
inFlight, ok := metadata[MetaKeyAutoStartOnCreateInFlight].(bool)
return ok && inFlight
}
// TaskSession.Metadata key that records how the session came into existence.
// workflow_switch means the session profile was selected by workflow routing
// rather than direct user selection.