fix: preserve resumed agents after accepted turn cancellation (#3791)

This commit is contained in:
Carlos Florêncio
2026-09-18 10:33:26 +01:00
committed by GitHub
parent b1b095f0c0
commit 2f15db8a14
21 changed files with 2069 additions and 32 deletions
@@ -99,6 +99,19 @@ func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt st
return m.PromptAgentWithDispatchCallback(ctx, executionID, prompt, attachments, dispatchOnly, nil)
}
// RegisterInitialPromptDispatchCallbacks installs one-shot callbacks for the
// initial prompt sent during StartAgentProcess. Model-switch startup launches
// that prompt asynchronously, so callers that own startup cancellation must
// wait for either provider acceptance or a pre-acceptance delivery failure.
func (m *Manager) RegisterInitialPromptDispatchCallbacks(executionID string, onDispatched, onFailure func()) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound)
}
execution.setInitialPromptDispatchCallbacks(onDispatched, onFailure)
return nil
}
// PromptAgentWithDispatchCallback exposes agentctl acceptance to callers that
// must keep admission serialized until the queued prompt is actually dispatched.
func (m *Manager) PromptAgentWithDispatchCallback(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment, dispatchOnly bool, onDispatched func()) (*PromptResult, error) {
@@ -952,6 +952,19 @@ func (sm *SessionManager) dispatchInitialPrompt(ctx context.Context, execution *
zap.Int("effective_length", len(effectivePrompt)))
}
acpAttachments := convertAttachments(attachments)
onDispatched, onInitialPromptFailure := execution.takeInitialPromptDispatchCallbacks()
var failureHandler func(InitialPromptFailure)
if onInitialPromptFailure != nil {
initialPromptFailure := sm.initialPromptFailure
failureHandler = func(failure InitialPromptFailure) {
onInitialPromptFailure()
if initialPromptFailure != nil {
initialPromptFailure(failure)
}
}
} else {
failureHandler = sm.initialPromptFailure
}
go func() {
promptCtx, cancel := appctx.Detached(ctx, sm.stopCh, 0)
defer cancel()
@@ -962,7 +975,7 @@ func (sm *SessionManager) dispatchInitialPrompt(ctx context.Context, execution *
false,
acpAttachments,
false,
sendPromptCallbacks{onFailure: sm.initialPromptFailure},
sendPromptCallbacks{onDispatched: onDispatched, onFailure: failureHandler},
false,
)
if err != nil {
@@ -628,7 +628,6 @@ func TestInitializeAndPrompt_StreamBeforeInitialize(t *testing.T) {
agentctl: client,
promptDoneCh: make(chan PromptCompletionSignal, 1),
}
agentConfig := &testAgent{
id: "test-agent",
enabled: true,
@@ -1235,6 +1234,8 @@ func TestInitializeAndPrompt_WithTaskDescription(t *testing.T) {
agentctl: client,
promptDoneCh: make(chan PromptCompletionSignal, 1),
}
dispatched := make(chan struct{}, 1)
execution.setInitialPromptDispatchCallbacks(func() { dispatched <- struct{}{} }, nil)
agentConfig := &testAgent{
id: "test-agent",
@@ -1260,7 +1261,11 @@ func TestInitializeAndPrompt_WithTaskDescription(t *testing.T) {
}
// Wait for the prompt to be sent asynchronously
time.Sleep(500 * time.Millisecond)
select {
case <-dispatched:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for initial prompt dispatch callback")
}
actions := mock.getActionLog()
@@ -274,6 +274,13 @@ type AgentExecution struct {
// asynchronously, so its transport-level gate alone cannot provide this.
promptMu sync.Mutex
dispatchedPromptPending atomic.Bool
// Initial-prompt callbacks are installed before StartAgentProcess for
// model-switch launches. Lifecycle sends the initial prompt asynchronously,
// so they must be captured before startup begins and consumed once that
// prompt is accepted or fails before acceptance.
initialPromptDispatchCallback func()
initialPromptFailureCallback func()
initialPromptDispatchCallbackMu sync.Mutex
// Closed when the current SendPrompt returns, so CancelAgent can wait
// for the in-flight prompt to finish before the caller retries.
@@ -346,6 +353,23 @@ func (e *AgentExecution) setSessionInitialized(value bool) {
e.sessionInitializedMu.Unlock()
}
func (e *AgentExecution) setInitialPromptDispatchCallbacks(onDispatched, onFailure func()) {
e.initialPromptDispatchCallbackMu.Lock()
e.initialPromptDispatchCallback = onDispatched
e.initialPromptFailureCallback = onFailure
e.initialPromptDispatchCallbackMu.Unlock()
}
func (e *AgentExecution) takeInitialPromptDispatchCallbacks() (func(), func()) {
e.initialPromptDispatchCallbackMu.Lock()
onDispatched := e.initialPromptDispatchCallback
onFailure := e.initialPromptFailureCallback
e.initialPromptDispatchCallback = nil
e.initialPromptFailureCallback = nil
e.initialPromptDispatchCallbackMu.Unlock()
return onDispatched, onFailure
}
type activeTopLevelTool struct {
ToolCallID string
Name string
@@ -301,6 +301,8 @@ type mockAgentManager struct {
promptAcceptedOnError bool
promptAgentFunc func(context.Context, string, string, []v1.MessageAttachment, bool) (*executor.PromptResult, error)
launchAgentFunc func(context.Context, *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error)
initialPromptDispatchCallback func()
initialPromptFailureCallback func()
startAgentProcessCalls []string
startAgentProcessErr error
startAgentProcessFunc func(context.Context, string) error
@@ -469,6 +471,15 @@ func (m *mockAgentManager) StartAgentProcess(ctx context.Context, sessionID stri
}
return err
}
func (m *mockAgentManager) RegisterInitialPromptDispatchCallbacks(_ string, onDispatched, onFailure func()) error {
m.mu.Lock()
m.initialPromptDispatchCallback = onDispatched
m.initialPromptFailureCallback = onFailure
m.mu.Unlock()
return nil
}
func (m *mockAgentManager) IsAgentCommandConfigured(_ string) bool { return true }
func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID string, force bool) error {
m.mu.Lock()
@@ -889,6 +889,28 @@ func workspaceFromTaskEnvironment(env *models.TaskEnvironment) string {
// the agent doesn't support in-place switching, it falls back to stopping and
// restarting the agent with the new model.
func (e *Executor) SwitchModel(ctx context.Context, taskID, sessionID, newModel, prompt string) (*PromptResult, error) {
return e.switchModel(ctx, taskID, sessionID, newModel, prompt, nil, nil)
}
// SwitchModelWithDispatchCallbacks is the model-switch variant used by a
// resume-owned prompt. The fallback startup sends its initial prompt from a
// lifecycle goroutine, so the caller must receive the actual acceptance or
// pre-acceptance failure instead of treating StartAgentProcess as acceptance.
func (e *Executor) SwitchModelWithDispatchCallbacks(
ctx context.Context,
taskID, sessionID, newModel, prompt string,
onDispatched func(executionID string),
onFailure func(),
) (*PromptResult, error) {
return e.switchModel(ctx, taskID, sessionID, newModel, prompt, onDispatched, onFailure)
}
func (e *Executor) switchModel(
ctx context.Context,
taskID, sessionID, newModel, prompt string,
onDispatched func(executionID string),
onFailure func(),
) (*PromptResult, error) {
e.logger.Info("switching model for session",
zap.String("task_id", taskID),
zap.String("session_id", sessionID),
@@ -954,7 +976,7 @@ func (e *Executor) SwitchModel(ctx context.Context, taskID, sessionID, newModel,
zap.Bool("use_worktree", req.UseWorktree),
zap.String("repository_path", req.RepositoryPath))
if err := e.launchModelSwitchAgent(launchCtx, task.ID, sessionID, newModel, session, req, existingRunning); err != nil {
if err := e.launchModelSwitchAgent(launchCtx, task.ID, sessionID, newModel, session, req, existingRunning, onDispatched, onFailure); err != nil {
return nil, err
}
@@ -1123,7 +1145,15 @@ func (e *Executor) stopPreparedModelSwitchAgent(
}
// launchModelSwitchAgent launches the new agent, persists state, and starts the process.
func (e *Executor) launchModelSwitchAgent(ctx context.Context, taskID, sessionID, newModel string, session *models.TaskSession, req *LaunchAgentRequest, existingRunning *models.ExecutorRunning) error {
func (e *Executor) launchModelSwitchAgent(
ctx context.Context,
taskID, sessionID, newModel string,
session *models.TaskSession,
req *LaunchAgentRequest,
existingRunning *models.ExecutorRunning,
onDispatched func(executionID string),
onFailure func(),
) error {
resp, err := e.agentManager.LaunchAgent(ctx, req)
if err != nil {
e.logger.Error("failed to launch agent with new model",
@@ -1132,6 +1162,28 @@ func (e *Executor) launchModelSwitchAgent(ctx context.Context, taskID, sessionID
zap.Error(err))
return fmt.Errorf("failed to launch agent with new model: %w", err)
}
if onDispatched != nil || onFailure != nil {
registrar, ok := e.agentManager.(interface {
RegisterInitialPromptDispatchCallbacks(string, func(), func()) error
})
if !ok {
registrationErr := errors.New("agent manager cannot register initial prompt dispatch callbacks")
e.cleanupUnstartedExecutionAfterPersistError(ctx, sessionID, resp.AgentExecutionID, registrationErr)
return registrationErr
}
if err := registrar.RegisterInitialPromptDispatchCallbacks(
resp.AgentExecutionID,
func() {
if onDispatched != nil {
onDispatched(resp.AgentExecutionID)
}
},
onFailure,
); err != nil {
e.cleanupUnstartedExecutionAfterPersistError(ctx, sessionID, resp.AgentExecutionID, err)
return fmt.Errorf("failed to register initial prompt dispatch callbacks: %w", err)
}
}
if err := e.persistModelSwitchState(ctx, taskID, sessionID, session, newModel); err != nil {
e.cleanupUnstartedExecutionAfterPersistError(ctx, sessionID, resp.AgentExecutionID, err)
@@ -831,6 +831,8 @@ func TestLaunchModelSwitchAgent_CleansStartedExecutionAfterTerminalRace(t *testi
&callerSession,
&LaunchAgentRequest{},
nil,
nil,
nil,
)
if !errors.Is(err, ErrSessionStateSuperseded) {
@@ -37,6 +37,19 @@ type resumeAttempt struct {
// can be retained when a replacement is admitted and retained again when
// its owner eventually returns; both paths must describe one tombstone.
retained bool
// accepted is protected by resumeAttemptRegistry.mu. Once provider
// acceptance is recorded, explicit session cancellation no longer owns
// startup teardown for this attempt.
accepted bool
// awaitingInitialPrompt keeps startup ownership active after the model
// switch path returns. Its initial prompt is dispatched asynchronously by
// lifecycle, so promptTask's deferred finish must wait for the real provider
// acceptance callback.
awaitingInitialPrompt bool
// finishRequested is set when the owning operation returns while the
// initial prompt is still pending. The acceptance callback completes the
// attempt after it transfers ownership.
finishRequested bool
}
type resumeAttemptRegistry struct {
@@ -56,6 +69,7 @@ type resumeAttemptTombstone struct {
id uint64
executionID string
cancelled bool
accepted bool
}
func newResumeAttemptRegistry() *resumeAttemptRegistry {
@@ -109,12 +123,20 @@ func (r *resumeAttemptRegistry) begin(parent context.Context, taskID, sessionID
// against prompt admission and lifecycle event handling.
func (r *resumeAttemptRegistry) invalidate(sessionID string) bool {
r.mu.Lock()
defer r.mu.Unlock()
attempt := r.attempts[sessionID]
if attempt == nil || attempt.ctx.Err() != nil {
if attempt == nil || attempt.ctx.Err() != nil || attempt.accepted {
r.mu.Unlock()
return false
}
attempt.cancel()
finishPending := attempt.awaitingInitialPrompt
r.mu.Unlock()
if finishPending {
// The operation that owns this attempt may already have returned while
// lifecycle is still dispatching its initial prompt. Finish the pending
// ownership record now so a cancelled callback cannot leave it active.
attempt.finish(r)
}
return true
}
@@ -161,6 +183,9 @@ func (r *resumeAttemptRegistry) canCleanup(attempt *resumeAttempt) bool {
if current := r.attempts[attempt.sessionID]; current != nil && current != attempt {
return false
}
if attempt.accepted {
return false
}
for _, tombstone := range r.tombstones[attempt.sessionID] {
if tombstone.id > attempt.id && tombstone.executionID == executionID {
return false
@@ -203,6 +228,9 @@ func (r *resumeAttemptRegistry) retainLocked(attempt *resumeAttempt) {
for index := range entries {
if entries[index].id == attempt.id && entries[index].executionID == "" && executionID != "" {
entries[index].executionID = executionID
}
if entries[index].id == attempt.id {
entries[index].accepted = attempt.accepted
break
}
}
@@ -215,6 +243,7 @@ func (r *resumeAttemptRegistry) retainLocked(attempt *resumeAttempt) {
id: attempt.id,
executionID: executionID,
cancelled: attempt.ctx.Err() != nil,
accepted: attempt.accepted,
})
if len(entries) > maxResumeAttemptTombstones {
entries = entries[len(entries)-maxResumeAttemptTombstones:]
@@ -253,12 +282,103 @@ func (r *resumeAttemptRegistry) canCleanupIdentity(sessionID, executionID, origi
if !found || tombstone.executionID != executionID {
return false
}
if tombstone.accepted {
return false
}
if current := r.attempts[sessionID]; current != nil && current.id > id {
return false
}
return r.latestExecution[sessionID][executionID] <= id
}
// accept transfers startup authority to the provider turn while retaining the
// attempt identity for later execution events. The registry lock orders this
// transition with explicit cancellation, so an invalidated attempt cannot be
// revived by a late provider callback.
func (r *resumeAttemptRegistry) accept(attempt *resumeAttempt, executionID string) bool {
if attempt == nil || executionID == "" {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
if r.attempts[attempt.sessionID] != attempt || attempt.ctx.Err() != nil {
return false
}
attempt.executionMu.Lock()
knownExecutionID := attempt.executionID
if knownExecutionID == "" {
attempt.executionID = executionID
knownExecutionID = executionID
}
attempt.executionMu.Unlock()
if knownExecutionID != executionID {
return false
}
attempt.accepted = true
return true
}
// holdForInitialPrompt keeps an admitted model-switch attempt active until
// lifecycle reports acceptance of the replacement's asynchronously dispatched
// initial prompt. The caller must set this before starting the replacement.
func (r *resumeAttemptRegistry) holdForInitialPrompt(attempt *resumeAttempt) bool {
if attempt == nil {
return false
}
r.mu.Lock()
defer r.mu.Unlock()
if r.attempts[attempt.sessionID] != attempt || attempt.ctx.Err() != nil || attempt.accepted {
return false
}
attempt.awaitingInitialPrompt = true
return true
}
// releaseInitialPromptHold releases the model-switch-specific deferred finish
// when the model switch stayed in place and ordinary prompt dispatch will own
// the acceptance boundary instead.
func (r *resumeAttemptRegistry) releaseInitialPromptHold(attempt *resumeAttempt) {
if attempt == nil {
return
}
r.mu.Lock()
if r.attempts[attempt.sessionID] == attempt {
attempt.awaitingInitialPrompt = false
}
r.mu.Unlock()
}
// abortInitialPromptHold closes a model-switch attempt when lifecycle reports
// a delivery failure before provider acceptance. There is no later owner that
// can safely finish the attempt after the asynchronous failure callback.
func (r *resumeAttemptRegistry) abortInitialPromptHold(attempt *resumeAttempt) {
if attempt == nil {
return
}
r.mu.Lock()
if r.attempts[attempt.sessionID] == attempt {
attempt.awaitingInitialPrompt = false
}
r.mu.Unlock()
attempt.finish(r)
}
// finishAfterInitialPromptAcceptance completes a deferred owner only after
// the callback has transferred startup authority. If the outer operation is
// still active, its normal finish call remains responsible for completion.
func (r *resumeAttemptRegistry) finishAfterInitialPromptAcceptance(attempt *resumeAttempt) {
if attempt == nil {
return
}
r.mu.Lock()
shouldFinish := r.attempts[attempt.sessionID] == attempt &&
attempt.accepted && attempt.finishRequested
r.mu.Unlock()
if shouldFinish {
attempt.finish(r)
}
}
func (r *resumeAttemptRegistry) cancelledExecutionLocked(sessionID, executionID string) bool {
if executionID == "" {
return false
@@ -317,17 +437,21 @@ func (attempt *resumeAttempt) finish(registry *resumeAttemptRegistry) {
if attempt == nil {
return
}
attempt.finishOnce.Do(func() {
if registry != nil {
registry.mu.Lock()
if registry.attempts[attempt.sessionID] == attempt {
delete(registry.attempts, attempt.sessionID)
}
registry.retainLocked(attempt)
if registry != nil {
registry.mu.Lock()
if registry.attempts[attempt.sessionID] == attempt &&
attempt.awaitingInitialPrompt && !attempt.accepted && attempt.ctx.Err() == nil {
attempt.finishRequested = true
registry.mu.Unlock()
return
}
close(attempt.done)
})
if registry.attempts[attempt.sessionID] == attempt {
delete(registry.attempts, attempt.sessionID)
}
registry.retainLocked(attempt)
registry.mu.Unlock()
}
attempt.finishOnce.Do(func() { close(attempt.done) })
}
func (attempt *resumeAttempt) wait(ctx context.Context) error {
@@ -5922,6 +5922,7 @@ func (s *Service) runPromptTurn(
promptCtx = dispatchCtx
onDispatched, dispatchOutcome := s.preparePromptDispatchCallback(
promptCtx, taskID, sessionID, session, rollback, options, foregroundDispatch, releaseDispatchGuard,
resumeAttempt,
)
result, execErr := s.executor.PromptWithDispatchCallback(
promptCtx, taskID, sessionID, effectivePrompt, attachments, dispatchOnly,
@@ -6015,7 +6016,7 @@ func (s *Service) resolveModelSwitchAttempt(
) (switchResult *PromptResult, handled bool, remainingGuard *lockedCancelInFlightGuard, switchErr error) {
result, handledSwitch, attemptErr := s.attemptModelSwitchForPrompt(
ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, runBeforeDispatch,
runAfterDispatchAdmission,
runAfterDispatchAdmission, resumeAttempt,
)
if !handledSwitch {
if modelSwitchGuard != nil {
@@ -6296,13 +6297,14 @@ func (s *Service) validateAndRunDispatchBoundary(
// preparePromptDispatchCallback binds the turn, arms the interactive-prompt
// attempt evidence, and builds the onDispatched callback executor.
// PromptWithDispatchCallback invokes once agentctl accepts the prompt —
// wrapping in the caller's afterDispatch hook and the nonterminal dispatch
// guard release, innermost first, when either applies.
// PromptWithDispatchCallback invokes once agentctl accepts the prompt. For a
// resumed turn, ownership transfer wraps the durable afterDispatch hook and
// identity publication; the nonterminal dispatch guard releases last.
func (s *Service) preparePromptDispatchCallback(
promptCtx context.Context, taskID, sessionID string, session *models.TaskSession,
rollback promptClaimRollback, options promptTaskOptions, foregroundDispatch *foregroundDispatch,
releaseDispatchGuard func(),
resumeAttempt *resumeAttempt,
) (onDispatched func(), dispatchOutcome *promptDispatchOutcome) {
s.bindPromptTurnID(promptCtx, session, rollback.turnID)
s.beginInteractivePromptAttempt(
@@ -6323,9 +6325,10 @@ func (s *Service) preparePromptDispatchCallback(
}
}
}
acceptedExecutionID := session.AgentExecutionID
onDispatched = s.promptDispatchCallbackForIdentity(
promptCtx, taskID, sessionID, rollback.sessionIdentity,
session.AgentExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome,
acceptedExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome,
)
if options.afterDispatch != nil {
originalOnDispatched := onDispatched
@@ -6336,6 +6339,15 @@ func (s *Service) preparePromptDispatchCallback(
dispatchOutcome.recordAccepted(errors.Join(durableOutcomeErr, publicationErr))
}
}
if resumeAttempt != nil {
originalOnDispatched := onDispatched
onDispatched = func() {
// Transfer startup ownership before any durable post-acceptance hook
// or identity publication can fail and before the dispatch guard releases.
s.acceptResumeAttemptAtPromptAcceptance(acceptedExecutionID, resumeAttempt)
originalOnDispatched()
}
}
if releaseDispatchGuard != nil {
originalOnDispatched := onDispatched
onDispatched = func() {
@@ -6350,6 +6362,25 @@ func (s *Service) preparePromptDispatchCallback(
return onDispatched, dispatchOutcome
}
func (s *Service) acceptResumeAttemptAtPromptAcceptance(
expectedExecutionID string,
attempt *resumeAttempt,
) bool {
if attempt == nil {
return false
}
if expectedExecutionID == "" {
expectedExecutionID = attempt.execution()
}
if expectedExecutionID == "" {
return false
}
// Provider acceptance is the irreversible boundary for startup teardown.
// Use the execution admitted before the provider call; stale-incarnation
// publication remains separately fenced by promptDispatchCallbackForIdentity.
return s.resumeAttemptStore().accept(attempt, expectedExecutionID)
}
func (s *Service) finishPromptDispatchFailure(
ctx context.Context,
taskID, sessionID, prompt string,
@@ -6782,8 +6813,12 @@ func (s *Service) promptDispatchCallbackForIdentity(
func (s *Service) attemptModelSwitchForPrompt(
ctx context.Context, taskID, sessionID, model, effectivePrompt string, session *models.TaskSession,
foregroundDispatch *foregroundDispatch, runBeforeDispatch func() error,
runAfterDispatchAdmission func() error,
runAfterDispatchAdmission func() error, resumeAttempts ...*resumeAttempt,
) (result *PromptResult, handled bool, err error) {
var resumeAttempt *resumeAttempt
if len(resumeAttempts) > 0 {
resumeAttempt = resumeAttempts[0]
}
if modelSwitchRequired(session, model) {
s.beginInitialPromptAttempt(sessionID, s.isDynamicPromptSession(session))
admissionErr := runBeforeDispatch()
@@ -6799,7 +6834,9 @@ func (s *Service) attemptModelSwitchForPrompt(
return nil, true, admissionErr
}
}
result, handled, switchErr := s.trySwitchModelForPrompt(ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch)
result, handled, switchErr := s.trySwitchModelForPrompt(
ctx, taskID, sessionID, model, effectivePrompt, session, foregroundDispatch, resumeAttempt,
)
if handled && switchErr != nil {
s.clearPromptAttemptEvidence(sessionID, "", 0)
}
@@ -6808,17 +6845,37 @@ func (s *Service) attemptModelSwitchForPrompt(
// trySwitchModelForPrompt keeps foreground admission consistent when a model
// switch either dispatches the prompt itself or fails before reaching the agent.
func (s *Service) trySwitchModelForPrompt(ctx context.Context, taskID, sessionID, model, prompt string, session *models.TaskSession, dispatch *foregroundDispatch) (*PromptResult, bool, error) {
result, switched, err := s.trySwitchModel(ctx, taskID, sessionID, model, prompt, session)
func (s *Service) trySwitchModelForPrompt(
ctx context.Context,
taskID, sessionID, model, prompt string,
session *models.TaskSession,
dispatch *foregroundDispatch,
resumeAttempt *resumeAttempt,
) (*PromptResult, bool, error) {
releaseHold, onDispatched, onFailure, holdErr := s.modelSwitchResumeCallbacks(session, model, resumeAttempt)
if holdErr != nil {
return nil, true, holdErr
}
result, switched, err := s.trySwitchModelWithDispatchCallbacks(
ctx, taskID, sessionID, model, prompt, session, onDispatched, onFailure,
)
if !switched && err == nil {
releaseHold()
return nil, false, nil
}
if err != nil {
releaseHold()
s.rollbackForegroundDispatchOnFailure(ctx, taskID, sessionID, dispatch)
return result, true, err
}
if executionID, executionErr := s.agentManager.GetExecutionIDForSession(ctx, sessionID); executionErr == nil && executionID != "" {
s.bindPromptAttemptToExecution(ctx, sessionID, executionID)
if resumeAttempt != nil {
// The replacement's initial prompt is dispatched asynchronously by
// lifecycle. Bind its immutable execution identity now; startup
// ownership transfers only from the callback installed for that prompt.
resumeAttempt.setExecutionID(executionID)
}
}
revealedBackground := s.acceptForegroundDispatch(dispatch)
if revealedBackground || dispatch.yieldedBeforeBegin || s.acceptedForegroundDispatchClaim(dispatch) {
@@ -6827,6 +6884,30 @@ func (s *Service) trySwitchModelForPrompt(ctx context.Context, taskID, sessionID
return result, true, nil
}
func (s *Service) modelSwitchResumeCallbacks(
session *models.TaskSession,
model string,
resumeAttempt *resumeAttempt,
) (release func(), onDispatched func(executionID string), onFailure func(), err error) {
release = func() {}
if resumeAttempt == nil || !modelSwitchRequired(session, model) {
return release, nil, nil, nil
}
registry := s.resumeAttemptStore()
if !registry.holdForInitialPrompt(resumeAttempt) {
s.cleanupCancelledResumeAttempt(resumeAttempt)
return nil, nil, nil, ErrResumeAttemptCancelled
}
release = func() { registry.releaseInitialPromptHold(resumeAttempt) }
onDispatched = func(executionID string) {
if s.acceptResumeAttemptAtPromptAcceptance(executionID, resumeAttempt) {
registry.finishAfterInitialPromptAcceptance(resumeAttempt)
}
}
onFailure = func() { registry.abortInitialPromptHold(resumeAttempt) }
return release, onDispatched, onFailure, nil
}
type promptClaimRollback struct {
previousSessionState models.TaskSessionState
previousTaskState v1.TaskState
@@ -7888,6 +7969,16 @@ func modelSwitchRequired(session *models.TaskSession, model string) bool {
}
func (s *Service) trySwitchModel(ctx context.Context, taskID, sessionID, model, effectivePrompt string, session *models.TaskSession) (*PromptResult, bool, error) {
return s.trySwitchModelWithDispatchCallbacks(ctx, taskID, sessionID, model, effectivePrompt, session, nil, nil)
}
func (s *Service) trySwitchModelWithDispatchCallbacks(
ctx context.Context,
taskID, sessionID, model, effectivePrompt string,
session *models.TaskSession,
onDispatched func(executionID string),
onFailure func(),
) (*PromptResult, bool, error) {
if !modelSwitchRequired(session, model) {
return nil, false, nil
}
@@ -7903,7 +7994,9 @@ func (s *Service) trySwitchModel(ctx context.Context, taskID, sessionID, model,
zap.String("from", currentModel),
zap.String("to", model))
switchCtx := context.WithoutCancel(ctx)
switchResult, err := s.executor.SwitchModel(switchCtx, taskID, sessionID, model, effectivePrompt)
switchResult, err := s.executor.SwitchModelWithDispatchCallbacks(
switchCtx, taskID, sessionID, model, effectivePrompt, onDispatched, onFailure,
)
if err != nil {
return nil, true, fmt.Errorf("model switch failed: %w", err)
}
@@ -0,0 +1,113 @@
package orchestrator
import (
"context"
"testing"
"github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.7
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
func TestResumeAttempt_ModelSwitchFallbackCancellationBeforeInitialPromptAcceptance(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-model-switch-before-acceptance"
sessionID = "session-model-switch-before-acceptance"
oldExecution = "execution-model-switch-before-old"
newExecution = "execution-model-switch-before-new"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileSnapshot = map[string]interface{}{"model": "old-model"}
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session model: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, oldExecution)
manager := &mockAgentManager{
repoForExecutionLookup: repo,
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{
ID: sessionID, SessionID: sessionID, TaskID: taskID,
AgentExecutionID: newExecution, Status: "starting",
}); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: newExecution}, nil
},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, manager)
svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{})
attempt, owner, err := svc.beginResumeAttempt(ctx, taskID, sessionID)
if err != nil || !owner {
t.Fatalf("begin model-switch attempt: attempt=%v owner=%v err=%v", attempt, owner, err)
}
attempt.setExecutionID(oldExecution)
t.Cleanup(func() { attempt.finish(svc.resumeAttemptStore()) })
_, handled, err := svc.trySwitchModelForPrompt(
ctx, taskID, sessionID, "new-model", "model-switch prompt", session,
&foregroundDispatch{}, attempt,
)
if err != nil {
t.Fatalf("model-switch fallback: %v", err)
}
if !handled {
t.Fatal("model-switch fallback was not handled")
}
manager.mu.Lock()
onDispatched := manager.initialPromptDispatchCallback
started := append([]string(nil), manager.startAgentProcessCalls...)
manager.mu.Unlock()
if len(started) != 1 || started[0] != newExecution {
t.Fatalf("StartAgentProcess calls = %#v, want [%q]", started, newExecution)
}
if onDispatched == nil {
t.Fatal("model-switch fallback did not retain the initial-prompt callback")
}
// promptTask returns before lifecycle's asynchronous initial prompt callback.
// Its normal deferred finish must leave the attempt cancellable in this gap.
attempt.finish(svc.resumeAttemptStore())
if _, active := svc.resumeAttemptStore().current(sessionID); !active {
t.Fatal("prompt-task completion removed the pending model-switch attempt before acceptance")
}
registry := svc.resumeAttemptStore()
registry.mu.Lock()
acceptedBeforeCancel := attempt.accepted
registry.mu.Unlock()
if acceptedBeforeCancel {
t.Fatal("startup ownership transferred before the provider accepted the initial prompt")
}
svc.invalidateResumeAttempt(sessionID)
svc.cleanupCancelledResumeAttempt(attempt)
manager.mu.Lock()
stops := append([]stopAgentCall(nil), manager.stopAgentWithReasonArgs...)
manager.mu.Unlock()
if len(stops) != 1 {
t.Fatalf("forced cleanup calls = %#v, want one call", stops)
}
if stops[0].ExecutionID != newExecution || !stops[0].Force {
t.Fatalf("forced cleanup call = %#v, want execution %q with force", stops[0], newExecution)
}
// A delayed lifecycle callback from the cancelled startup cannot revive
// provider ownership after exact-execution cleanup has won.
onDispatched()
registry.mu.Lock()
acceptedAfterLateCallback := attempt.accepted
registry.mu.Unlock()
if acceptedAfterLateCallback {
t.Fatal("late initial-prompt callback revived a cancelled resume attempt")
}
}
@@ -0,0 +1,833 @@
package orchestrator
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/orchestrator/messagequeue"
"github.com/kandev/kandev/internal/orchestrator/watcher"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// acceptedTurnPromptManager lets the first provider turn report acceptance
// before it waits for the cancellation owner to release the turn.
type acceptedTurnPromptManager struct {
*mockAgentManager
accepted chan struct{}
release chan struct{}
releaseOnce sync.Once
firstPrompt atomic.Bool
firstPromptErr error
beforeDispatch func()
}
func (m *acceptedTurnPromptManager) PromptAgentWithDispatchCallback(
ctx context.Context,
executionID, prompt string,
attachments []v1.MessageAttachment,
dispatchOnly bool,
onDispatched func(),
) (*executor.PromptResult, error) {
_, err := m.PromptAgent(ctx, executionID, prompt, attachments, dispatchOnly)
if err != nil {
return nil, err
}
first := m.firstPrompt.CompareAndSwap(false, true)
if first && m.beforeDispatch != nil {
m.beforeDispatch()
}
if onDispatched != nil {
onDispatched()
}
if !first {
return &executor.PromptResult{
StopReason: "end_turn",
AgentMessage: "follow-up response",
}, nil
}
close(m.accepted)
<-m.release
return &executor.PromptResult{StopReason: "cancelled"}, m.firstPromptErr
}
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.7
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.9
func TestPromptTask_ResumedTurnCancelPreservesExecution(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-resumed-turn-cancel"
sessionID = "session-resumed-turn-cancel"
executionID = "execution-resumed-turn-cancel"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileID = "profile1"
session.AgentExecutionID = executionID
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session profile: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, "execution-before-resume")
var launches atomic.Int32
var currentExecution atomic.Value
currentExecution.Store(executionID)
agentRunning := &atomic.Bool{}
baseManager := &mockAgentManager{
repoForExecutionLookup: repo,
getExecutionIDForSessionFunc: func(context.Context, string) (string, error) {
return currentExecution.Load().(string), nil
},
isAgentRunningFn: func(context.Context, string) bool {
return agentRunning.Load()
},
isAgentReadyFn: func(context.Context, string) bool {
return agentRunning.Load()
},
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
launches.Add(1)
currentExecution.Store(executionID)
agentRunning.Store(true)
if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{
ID: sessionID, SessionID: sessionID, TaskID: taskID,
AgentExecutionID: executionID, Status: "ready",
}); err != nil {
return nil, err
}
if err := repo.UpdateTaskSessionState(
context.Background(), req.SessionID, models.TaskSessionStateWaitingForInput, "",
); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: executionID}, nil
},
}
manager := &acceptedTurnPromptManager{
mockAgentManager: baseManager,
accepted: make(chan struct{}),
release: make(chan struct{}),
}
baseManager.cancelAgentFunc = func(context.Context, string) error {
manager.releaseOnce.Do(func() { close(manager.release) })
return nil
}
baseManager.stopAgentWithReasonFunc = func(_ context.Context, _ string, _ string, _ bool) error {
agentRunning.Store(false)
return nil
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, manager)
svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{})
firstPromptDone := make(chan error, 1)
go func() {
_, promptErr := svc.PromptTask(ctx, taskID, sessionID, "first resumed prompt", "", false, nil, false)
firstPromptDone <- promptErr
}()
select {
case <-manager.accepted:
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("resumed prompt did not reach provider acceptance")
}
acceptedAttempt, active := svc.resumeAttemptStore().current(sessionID)
if !active || acceptedAttempt == nil {
t.Fatal("accepted resume attempt was removed before the provider turn returned")
}
registry := svc.resumeAttemptStore()
registry.mu.Lock()
isAccepted := acceptedAttempt.accepted
registry.mu.Unlock()
if !isAccepted {
t.Fatal("provider acceptance did not transfer startup ownership")
}
cancelDone := make(chan error, 1)
go func() { cancelDone <- svc.CancelAgent(ctx, sessionID) }()
select {
case cancelErr := <-cancelDone:
if cancelErr != nil {
t.Fatalf("CancelAgent: %v", cancelErr)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("CancelAgent did not settle accepted turn")
}
select {
case promptErr := <-firstPromptDone:
if promptErr != nil {
t.Fatalf("accepted cancelled prompt: %v", promptErr)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("accepted prompt did not return after cancellation")
}
baseManager.mu.Lock()
forcedStops := append([]stopAgentCall(nil), baseManager.stopAgentWithReasonArgs...)
baseManager.mu.Unlock()
if len(forcedStops) != 0 {
t.Fatalf("forced execution stops after accepted cancellation = %d, want 0: %#v", len(forcedStops), forcedStops)
}
if got := launches.Load(); got != 1 {
t.Fatalf("resume launches after accepted cancellation = %d, want 1", got)
}
assertResumeSessionState(t, repo, sessionID, models.TaskSessionStateWaitingForInput)
running, err := repo.GetExecutorRunningBySessionID(ctx, sessionID)
if err != nil {
t.Fatalf("load preserved execution: %v", err)
}
if running == nil || running.AgentExecutionID != executionID {
t.Fatalf("preserved execution = %#v, want %q", running, executionID)
}
if _, err := svc.PromptTask(ctx, taskID, sessionID, "follow-up prompt", "", false, nil, false); err != nil {
t.Fatalf("follow-up prompt: %v", err)
}
if got := launches.Load(); got != 1 {
t.Fatalf("follow-up caused %d resume launches, want 1", got)
}
baseManager.mu.Lock()
promptCalls := append([]promptCall(nil), baseManager.capturedPromptCalls...)
baseManager.mu.Unlock()
if len(promptCalls) != 2 {
t.Fatalf("prompt calls = %d, want 2: %#v", len(promptCalls), promptCalls)
}
for index, call := range promptCalls {
if call.ExecutionID != executionID {
t.Errorf("prompt %d execution ID = %q, want %q", index, call.ExecutionID, executionID)
}
}
}
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.7
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.9
func TestPromptTask_QueuedAcceptedTurnIdentityReadFailurePreservesExecution(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-queued-accepted-identity-failure"
sessionID = "session-queued-accepted-identity-failure"
executionID = "execution-queued-accepted-identity-failure"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileID = "profile1"
session.AgentExecutionID = executionID
if session.QueueIncarnationID == "" {
session.QueueIncarnationID = "queue-incarnation-accepted-identity-failure"
}
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session profile: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, "execution-before-queued-resume")
var launches atomic.Int32
var currentExecution atomic.Value
currentExecution.Store(executionID)
agentRunning := &atomic.Bool{}
identityReadFailure := &atomic.Bool{}
var identityReadFailures atomic.Int32
queueRepo := messagequeue.NewMemoryRepositoryWithAuthority(
func(ctx context.Context, requestedTaskID, requestedSessionID string) (messagequeue.QueueSessionIdentity, error) {
if identityReadFailure.Load() {
identityReadFailures.Add(1)
return messagequeue.QueueSessionIdentity{}, errors.New("injected session identity read failure")
}
current, resolveErr := repo.GetTaskSession(ctx, requestedSessionID)
if resolveErr != nil {
return messagequeue.QueueSessionIdentity{}, resolveErr
}
if current == nil || current.TaskID != requestedTaskID || current.QueueIncarnationID == "" {
return messagequeue.QueueSessionIdentity{}, messagequeue.ErrSessionIdentityMismatch
}
return messagequeue.QueueSessionIdentity{
TaskID: requestedTaskID,
SessionID: requestedSessionID,
SessionIncarnationID: current.QueueIncarnationID,
}, nil
},
)
baseManager := &mockAgentManager{
repoForExecutionLookup: repo,
getExecutionIDForSessionFunc: func(context.Context, string) (string, error) {
return currentExecution.Load().(string), nil
},
isAgentRunningFn: func(context.Context, string) bool {
return agentRunning.Load()
},
isAgentReadyFn: func(context.Context, string) bool {
return agentRunning.Load()
},
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
launches.Add(1)
currentExecution.Store(executionID)
agentRunning.Store(true)
if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{
ID: sessionID, SessionID: sessionID, TaskID: taskID,
AgentExecutionID: executionID, Status: "ready",
}); err != nil {
return nil, err
}
if err := repo.UpdateTaskSessionState(
context.Background(), req.SessionID, models.TaskSessionStateWaitingForInput, "",
); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: executionID}, nil
},
}
manager := &acceptedTurnPromptManager{
mockAgentManager: baseManager,
accepted: make(chan struct{}),
release: make(chan struct{}),
beforeDispatch: func() { identityReadFailure.Store(true) },
}
baseManager.cancelAgentFunc = func(context.Context, string) error {
manager.releaseOnce.Do(func() { close(manager.release) })
return nil
}
baseManager.stopAgentWithReasonFunc = func(_ context.Context, _ string, _ string, _ bool) error {
agentRunning.Store(false)
return nil
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, manager)
svc.messageQueue = messagequeue.NewService(queueRepo, messagequeue.DefaultMaxPerSession, testLogger())
svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{})
t.Cleanup(func() {
svc.clearAcceptedQueuedDispatch(sessionID)
svc.stopSendNowWorkers()
})
identity, err := svc.messageQueue.ResolveSessionIdentity(ctx, taskID, sessionID)
if err != nil {
t.Fatalf("resolve queued session identity: %v", err)
}
if identity.SessionIncarnationID == "" {
t.Fatal("queued session identity has no incarnation")
}
if err := svc.messageQueue.SetAutoRun(ctx, sessionID, true); err != nil {
t.Fatalf("enable auto-run: %v", err)
}
if _, err := svc.messageQueue.QueueMessageWithMetadata(
ctx, sessionID, taskID, "queued accepted identity failure prompt", "",
messagequeue.QueuedByUser, false, nil, nil,
); err != nil {
t.Fatalf("queue prompt: %v", err)
}
queued, ok, _, err := svc.messageQueue.ReserveQueuedWithAutoRunForSession(ctx, identity)
if err != nil || !ok || queued == nil {
t.Fatalf("reserve queued prompt: queued=%+v ok=%v err=%v", queued, ok, err)
}
if reservation := svc.markQueuedDispatchInFlightWithIdentityLocked(identity, queued.ID, queued); reservation == nil {
t.Fatal("mark queued dispatch in flight returned nil")
}
var afterDispatchSawAccepted atomic.Bool
promptDone := make(chan error, 1)
go func() {
_, promptErr := svc.promptTask(
ctx, taskID, sessionID, "queued accepted identity failure prompt", "", false, nil, true,
launchOriginAutomatic,
promptTaskOptions{
claimEntryID: queued.ID,
expectedSessionIdentity: &identity,
afterDispatch: func() error {
afterDispatchSawAccepted.Store(activeResumeAttemptAccepted(svc, sessionID))
return nil
},
},
)
promptDone <- promptErr
}()
select {
case <-manager.accepted:
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("queued resumed prompt did not reach provider acceptance")
}
if identityReadFailures.Load() == 0 {
t.Fatal("acceptance callback did not exercise the injected identity read failure")
}
if !afterDispatchSawAccepted.Load() {
t.Fatal("afterDispatch ran before startup ownership transferred")
}
assertActiveResumeAttemptAccepted(t, svc, sessionID)
cancelDone := make(chan error, 1)
go func() { cancelDone <- svc.CancelAgent(ctx, sessionID) }()
select {
case cancelErr := <-cancelDone:
if cancelErr != nil {
t.Fatalf("CancelAgent: %v", cancelErr)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("CancelAgent did not settle queued accepted turn")
}
select {
case promptErr := <-promptDone:
if promptErr != nil {
t.Fatalf("queued accepted prompt: %v", promptErr)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("queued accepted prompt did not return after cancellation")
}
baseManager.mu.Lock()
forcedStops := append([]stopAgentCall(nil), baseManager.stopAgentWithReasonArgs...)
baseManager.mu.Unlock()
if len(forcedStops) != 0 {
t.Fatalf("forced execution stops after queued accepted cancellation = %d, want 0: %#v", len(forcedStops), forcedStops)
}
if got := launches.Load(); got != 1 {
t.Fatalf("queued resume launches after accepted cancellation = %d, want 1", got)
}
assertResumeSessionState(t, repo, sessionID, models.TaskSessionStateWaitingForInput)
running, err := repo.GetExecutorRunningBySessionID(ctx, sessionID)
if err != nil {
t.Fatalf("load preserved queued execution: %v", err)
}
if running == nil || running.AgentExecutionID != executionID {
t.Fatalf("preserved queued execution = %#v, want %q", running, executionID)
}
if _, err := svc.PromptTask(ctx, taskID, sessionID, "queued identity follow-up", "", false, nil, false); err != nil {
t.Fatalf("queued identity follow-up prompt: %v", err)
}
if got := launches.Load(); got != 1 {
t.Fatalf("queued identity follow-up caused %d resume launches, want 1", got)
}
baseManager.mu.Lock()
promptCalls := append([]promptCall(nil), baseManager.capturedPromptCalls...)
baseManager.mu.Unlock()
if len(promptCalls) != 2 {
t.Fatalf("queued identity prompt calls = %d, want 2: %#v", len(promptCalls), promptCalls)
}
for index, call := range promptCalls {
if call.ExecutionID != executionID {
t.Errorf("queued identity prompt %d execution ID = %q, want %q", index, call.ExecutionID, executionID)
}
}
}
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.7
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.9
func TestResumeTaskSessionAndPrompt_AcceptedTurnCancelPreservesExecution(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-compound-resumed-turn-cancel"
sessionID = "session-compound-resumed-turn-cancel"
executionID = "execution-compound-resumed-turn-cancel"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileID = "profile1"
session.AgentExecutionID = executionID
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session profile: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, executionID)
var launches atomic.Int32
agentRunning := &atomic.Bool{}
baseManager := &mockAgentManager{
repoForExecutionLookup: repo,
isAgentRunningFn: func(context.Context, string) bool {
return agentRunning.Load()
},
isAgentReadyFn: func(context.Context, string) bool {
return agentRunning.Load()
},
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
launches.Add(1)
agentRunning.Store(true)
if err := repo.UpdateTaskSessionState(
context.Background(), req.SessionID, models.TaskSessionStateWaitingForInput, "",
); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: executionID}, nil
},
}
manager := &acceptedTurnPromptManager{
mockAgentManager: baseManager,
accepted: make(chan struct{}),
release: make(chan struct{}),
firstPromptErr: fmt.Errorf("prompt abandoned after cancel: %w", lifecycle.ErrCancelEscalated),
}
baseManager.cancelAgentFunc = func(context.Context, string) error {
manager.releaseOnce.Do(func() { close(manager.release) })
return nil
}
baseManager.stopAgentWithReasonFunc = func(_ context.Context, _ string, _ string, _ bool) error {
agentRunning.Store(false)
return nil
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, manager)
svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{})
resultDone := make(chan struct {
result *PromptResult
err error
}, 1)
go func() {
result, promptErr := svc.ResumeTaskSessionAndPrompt(
ctx, taskID, sessionID, "first compound prompt", "", false, nil,
)
resultDone <- struct {
result *PromptResult
err error
}{result: result, err: promptErr}
}()
select {
case <-manager.accepted:
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("compound resume did not reach provider acceptance")
}
assertActiveResumeAttemptAccepted(t, svc, sessionID)
cancelDone := make(chan error, 1)
go func() { cancelDone <- svc.CancelAgent(ctx, sessionID) }()
select {
case cancelErr := <-cancelDone:
if cancelErr != nil {
t.Fatalf("CancelAgent: %v", cancelErr)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("CancelAgent did not settle the accepted compound turn")
}
select {
case outcome := <-resultDone:
if outcome.err == nil || !errors.Is(outcome.err, lifecycle.ErrCancelEscalated) {
t.Fatalf("accepted compound error = %v, want wrapped ErrCancelEscalated", outcome.err)
}
case <-time.After(resumeCancellationTestTimeout(t)):
t.Fatal("accepted compound prompt did not return after cancellation")
}
baseManager.mu.Lock()
forcedStops := append([]stopAgentCall(nil), baseManager.stopAgentWithReasonArgs...)
baseManager.mu.Unlock()
if len(forcedStops) != 0 {
t.Fatalf("forced execution stops after accepted compound cancellation = %d, want 0: %#v", len(forcedStops), forcedStops)
}
if got := launches.Load(); got != 1 {
t.Fatalf("compound resume launches = %d, want 1", got)
}
assertResumeSessionState(t, repo, sessionID, models.TaskSessionStateWaitingForInput)
if _, err := svc.PromptTask(ctx, taskID, sessionID, "compound follow-up", "", false, nil, false); err != nil {
t.Fatalf("compound follow-up prompt: %v", err)
}
if got := launches.Load(); got != 1 {
t.Fatalf("compound follow-up caused %d resume launches, want 1", got)
}
}
func TestResumeAttempt_AcceptanceCancellationOrdering(t *testing.T) {
t.Run("provider acceptance wins", func(t *testing.T) {
registry := newResumeAttemptRegistry()
attempt, owner := registry.begin(context.Background(), "task-acceptance-order", "session-acceptance-order")
if !owner {
t.Fatal("acceptance-order attempt was not admitted")
}
attempt.setExecutionID("execution-acceptance-order")
if !registry.accept(attempt, "execution-acceptance-order") {
t.Fatal("provider acceptance did not win before cancellation")
}
if registry.invalidate(attempt.sessionID) {
t.Fatal("accepted attempt remained startup-cancellable")
}
if err := attempt.context().Err(); err != nil {
t.Fatalf("accepted attempt context was cancelled: %v", err)
}
if !registry.accept(attempt, "execution-acceptance-order") {
t.Fatal("repeated acceptance was not idempotent")
}
if registry.canCleanup(attempt) {
t.Fatal("accepted attempt retained startup cleanup authority")
}
attempt.finish(registry)
if registry.canCleanupIdentity(
attempt.sessionID, "execution-acceptance-order", attempt.identity(),
) {
t.Fatal("accepted tombstone retained startup cleanup authority")
}
})
t.Run("startup cancellation wins", func(t *testing.T) {
registry := newResumeAttemptRegistry()
attempt, owner := registry.begin(context.Background(), "task-cancellation-order", "session-cancellation-order")
if !owner {
t.Fatal("cancellation-order attempt was not admitted")
}
attempt.setExecutionID("execution-cancellation-order")
if !registry.invalidate(attempt.sessionID) {
t.Fatal("startup attempt was not invalidated")
}
if registry.accept(attempt, "execution-cancellation-order") {
t.Fatal("late provider acceptance revived cancelled startup")
}
if attempt.context().Err() == nil {
t.Fatal("startup cancellation did not cancel the attempt context")
}
if !registry.canCleanup(attempt) {
t.Fatal("cancelled startup lost cleanup authority")
}
attempt.finish(registry)
})
}
func TestResumeAttempt_AcceptedExecutionEventsRemainValid(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-accepted-events"
sessionID = "session-accepted-events"
executionID = "execution-accepted-events"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateStarting)
seedExecutorRunning(t, repo, sessionID, taskID, executionID)
svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo())
attempt, owner, err := svc.beginResumeAttempt(ctx, taskID, sessionID)
if err != nil || !owner {
t.Fatalf("begin accepted event attempt: attempt=%v owner=%v err=%v", attempt, owner, err)
}
attempt.setExecutionID(executionID)
if !svc.resumeAttemptStore().accept(attempt, executionID) {
t.Fatal("accepted event attempt was not accepted")
}
attemptID := attempt.identity()
attempt.finish(svc.resumeAttemptStore())
if !svc.resumeAttemptAllowsExecution(sessionID, executionID, attemptID) {
t.Fatal("accepted execution event was rejected after attempt completion")
}
if svc.resumeAttemptAllowsExecution(sessionID, executionID, "resume-999999") {
t.Fatal("unknown accepted event identity was allowed")
}
svc.handleACPSessionCreated(ctx, watcher.ACPSessionEventData{
TaskID: taskID,
SessionID: sessionID,
AgentExecutionID: executionID,
AttemptID: attemptID,
ACPSessionID: "accepted-provider-session",
})
assertResumeToken(t, repo, sessionID, "accepted-provider-session")
svc.handleACPSessionCreated(ctx, watcher.ACPSessionEventData{
TaskID: taskID,
SessionID: sessionID,
AgentExecutionID: executionID,
AttemptID: "resume-999999",
ACPSessionID: "stale-provider-session",
})
assertResumeToken(t, repo, sessionID, "accepted-provider-session")
}
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
func TestResumeAttempt_AcceptedPublicationFailureDoesNotCleanup(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-accepted-publication-failure"
sessionID = "session-accepted-publication-failure"
executionID = "execution-accepted-publication-failure"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileID = "profile1"
session.AgentExecutionID = executionID
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session profile: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, executionID)
var launches atomic.Int32
agentRunning := &atomic.Bool{}
publicationErr := errors.New("accepted turn publication failed")
agentManager := &mockAgentManager{
repoForExecutionLookup: repo,
isAgentRunningFn: func(context.Context, string) bool {
return agentRunning.Load()
},
isAgentReadyFn: func(context.Context, string) bool {
return agentRunning.Load()
},
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
launches.Add(1)
agentRunning.Store(true)
if err := repo.UpdateTaskSessionState(
context.Background(), req.SessionID, models.TaskSessionStateWaitingForInput, "",
); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: executionID}, nil
},
promptResult: &executor.PromptResult{StopReason: "end_turn", AgentMessage: "accepted"},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentManager)
svc.executor = executor.NewExecutor(agentManager, repo, testLogger(), executor.ExecutorConfig{})
svc.turnService = failingReservedTurnPublisher{
TurnService: &repoBackedTurnService{repo: repo},
err: publicationErr,
}
_, err = svc.promptTask(
ctx, taskID, sessionID, "accepted publication prompt", "", false, nil, false,
launchOriginManual,
promptTaskOptions{reserveTurnUntilDispatch: true},
)
if !errors.Is(err, publicationErr) {
t.Fatalf("promptTask error = %v, want publication error %v", err, publicationErr)
}
var accepted interface{ DetachedResumeAccepted() bool }
if !errors.As(err, &accepted) || !accepted.DetachedResumeAccepted() {
t.Fatalf("promptTask error = %v, want accepted-dispatch marker", err)
}
if got := launches.Load(); got != 1 {
t.Fatalf("resume launches = %d, want 1", got)
}
agentManager.mu.Lock()
forcedStops := append([]stopAgentCall(nil), agentManager.stopAgentWithReasonArgs...)
agentManager.mu.Unlock()
if len(forcedStops) != 0 {
t.Fatalf("accepted publication failure forced %d startup stops: %#v", len(forcedStops), forcedStops)
}
if pending := svc.reservedPromptTurnID(sessionID); pending != "" {
t.Fatalf("accepted publication failure left turn rollback-eligible: %q", pending)
}
}
// @covers AC-AGENTS-AGENT-RESUME-RUNTIME-RECOVERY-007.8
func TestResumeAttempt_ModelSwitchFallbackTransfersAcceptance(t *testing.T) {
ctx := context.Background()
const (
taskID = "task-model-switch-acceptance"
sessionID = "session-model-switch-acceptance"
oldExecution = "execution-model-switch-old"
newExecution = "execution-model-switch-new"
)
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("load session: %v", err)
}
session.AgentProfileSnapshot = map[string]interface{}{"model": "old-model"}
if err := repo.UpdateTaskSession(ctx, session); err != nil {
t.Fatalf("update session model: %v", err)
}
seedExecutorRunning(t, repo, sessionID, taskID, oldExecution)
manager := &mockAgentManager{
repoForExecutionLookup: repo,
isAgentRunning: true,
launchAgentFunc: func(_ context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{
ID: sessionID, SessionID: sessionID, TaskID: taskID,
AgentExecutionID: newExecution, Status: "ready",
}); err != nil {
return nil, err
}
return &executor.LaunchAgentResponse{AgentExecutionID: newExecution}, nil
},
}
taskRepo := newMockTaskRepo()
taskRepo.tasks[taskID] = &v1.Task{ID: taskID, State: v1.TaskStateInProgress}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, manager)
svc.executor = executor.NewExecutor(manager, repo, testLogger(), executor.ExecutorConfig{})
attempt, owner, err := svc.beginResumeAttempt(ctx, taskID, sessionID)
if err != nil || !owner {
t.Fatalf("begin model-switch attempt: attempt=%v owner=%v err=%v", attempt, owner, err)
}
attempt.setExecutionID(oldExecution)
t.Cleanup(func() { attempt.finish(svc.resumeAttemptStore()) })
result, handled, err := svc.trySwitchModelForPrompt(
ctx, taskID, sessionID, "new-model", "model-switch prompt", session,
&foregroundDispatch{}, attempt,
)
if err != nil {
t.Fatalf("model-switch fallback: %v", err)
}
if !handled {
t.Fatal("model-switch fallback was not handled")
}
if result == nil || result.StopReason != "model_switched" {
t.Fatalf("model-switch result = %#v, want model_switched", result)
}
manager.mu.Lock()
onDispatched := manager.initialPromptDispatchCallback
manager.mu.Unlock()
if onDispatched == nil {
t.Fatal("model-switch fallback did not register its initial-prompt acceptance callback")
}
registry := svc.resumeAttemptStore()
registry.mu.Lock()
acceptedBeforeDispatch := attempt.accepted
registry.mu.Unlock()
if acceptedBeforeDispatch {
t.Fatal("model-switch fallback transferred startup ownership before provider acceptance")
}
onDispatched()
if got := attempt.execution(); got != newExecution {
t.Fatalf("resume attempt execution = %q, want %q", got, newExecution)
}
registry.mu.Lock()
accepted := attempt.accepted
registry.mu.Unlock()
if !accepted {
t.Fatal("model-switch fallback did not transfer startup ownership")
}
if err := attempt.validate(registry); err != nil {
t.Fatalf("accepted model-switch attempt validation: %v", err)
}
}
func assertActiveResumeAttemptAccepted(t *testing.T, svc *Service, sessionID string) {
t.Helper()
if !activeResumeAttemptAccepted(svc, sessionID) {
t.Fatalf("provider acceptance did not transfer startup ownership for %s", sessionID)
}
}
func activeResumeAttemptAccepted(svc *Service, sessionID string) bool {
attempt, active := svc.resumeAttemptStore().current(sessionID)
if !active || attempt == nil {
return false
}
registry := svc.resumeAttemptStore()
registry.mu.Lock()
defer registry.mu.Unlock()
return attempt.accepted
}
+1
View File
@@ -2522,6 +2522,7 @@ export class ApiClient {
id: string;
task_id: string;
queue_incarnation_id: string;
agent_execution_id?: string;
agent_profile_id?: string;
executor_id?: string;
executor_profile_id?: string;
@@ -182,3 +182,80 @@ export async function waitForQueuedCount(
})
.toBe(count);
}
export type SessionRuntimeIdentity = {
executionId: string;
acpSessionId: string;
};
/** Read the durable execution and provider conversation identities for a session. */
export async function readSessionRuntimeIdentity(
apiClient: ApiClient,
taskId: string,
sessionId: string,
): Promise<SessionRuntimeIdentity> {
const { sessions } = await apiClient.listTaskSessions(taskId);
const session = sessions.find((candidate) => candidate.id === sessionId);
const status = await apiClient.wsRequest<{ acp_session_id?: string }>("task.session.status", {
task_id: taskId,
session_id: sessionId,
});
if (!session?.agent_execution_id) {
throw new Error(`Session ${sessionId} has no durable agent execution identity`);
}
if (!status.acp_session_id) {
throw new Error(`Session ${sessionId} has no durable ACP conversation identity`);
}
return {
executionId: session.agent_execution_id,
acpSessionId: status.acp_session_id,
};
}
/** Read message IDs whose persisted content contains a turn-specific marker. */
export async function readSessionMessageIdsContaining(
apiClient: ApiClient,
sessionId: string,
marker: string,
): Promise<Set<string>> {
const { messages } = await apiClient.listSessionMessages(sessionId);
return new Set(
messages.filter((message) => message.content.includes(marker)).map((message) => message.id),
);
}
/** Wait until a new persisted message contains the marker. */
export async function waitForNewSessionMessage(
apiClient: ApiClient,
sessionId: string,
previousMessageIds: ReadonlySet<string>,
marker: string,
timeout = 30_000,
): Promise<void> {
await expect
.poll(
async () => {
const { messages } = await apiClient.listSessionMessages(sessionId);
return messages.some(
(message) => !previousMessageIds.has(message.id) && message.content.includes(marker),
);
},
{
timeout,
message: `Waiting for a new session message containing ${marker}`,
},
)
.toBe(true);
}
/** Count persisted resume boot messages without relying on the UI deduplication. */
export async function countResumeBootMessages(
apiClient: ApiClient,
sessionId: string,
): Promise<number> {
const { messages } = await apiClient.listSessionMessages(sessionId);
return messages.filter(
(message) =>
message.metadata?.script_type === "agent_boot" && message.metadata?.is_resuming === true,
).length;
}
@@ -6,8 +6,12 @@ import { waitForSessionState } from "../../helpers/session";
import { SessionPage } from "../../pages/session-page";
import {
cleanupDelayedResumeFixture,
countResumeBootMessages,
createFailOnResumeProfile,
readSessionMessageIdsContaining,
readSessionRuntimeIdentity,
seedDelayedResumeFixture,
waitForNewSessionMessage,
waitForSessionReady,
} from "../../helpers/session-resume-prompt-queue";
import {
@@ -101,6 +105,116 @@ test.describe("mobile: delayed resume cancellation", () => {
await cleanupDelayedResumeFixture(apiClient, fixture);
}
});
test("pausing an accepted lazy resume preserves the runtime for later turns", async ({
testPage,
apiClient,
seedData,
backend,
}) => {
test.setTimeout(150_000);
await apiClient.saveUserSettings({ prevent_auto_start_agent_on_open: true });
try {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Mobile accepted lazy resume pause recovery",
seedData.agentProfileId,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
if (!task.session_id) throw new Error("mobile accepted lazy resume task has no session_id");
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
await expect(session.chat.getByText("simple mock response", { exact: false })).toBeVisible({
timeout: 30_000,
});
await session.waitForChatIdle({ timeout: 30_000 });
await backend.restart();
await testPage.reload();
await session.waitForLoad();
await expect(testPage.getByTestId("composer-agent-start-hint")).toBeVisible({
timeout: 60_000,
});
const resumeBootsBeforeMessage = await countResumeBootMessages(apiClient, task.session_id);
// Provider output proves that the resumed prompt crossed acceptance
// before the touch cancellation is sent.
await session.sendMessageViaButton("/slow 8s");
await expect(session.chat.getByText("Running slow response", { exact: false })).toBeVisible({
timeout: 30_000,
});
const initialRuntimeIdentity = await readSessionRuntimeIdentity(
apiClient,
task.id,
task.session_id,
);
await session.cancelAgentButton().tap();
await waitForSessionState(apiClient, {
taskId: task.id,
sessionId: task.session_id,
expectedState: "WAITING_FOR_INPUT",
message: "Waiting for mobile accepted lazy resume cancellation",
timeout: 30_000,
});
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
const resumeBootsAfterFirstPause = await countResumeBootMessages(apiClient, task.session_id);
expect(resumeBootsAfterFirstPause).toBe(resumeBootsBeforeMessage + 1);
const slowResponseMessageIdsBeforeSecond = await readSessionMessageIdsContaining(
apiClient,
task.session_id,
"Running slow response",
);
expect(slowResponseMessageIdsBeforeSecond.size).toBeGreaterThan(0);
await session.sendMessageViaButton("/slow 8s");
await waitForNewSessionMessage(
apiClient,
task.session_id,
slowResponseMessageIdsBeforeSecond,
"Running slow response",
);
await session.cancelAgentButton().tap();
await waitForSessionState(apiClient, {
taskId: task.id,
sessionId: task.session_id,
expectedState: "WAITING_FOR_INPUT",
message: "Waiting for the later mobile accepted pause",
timeout: 30_000,
});
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
expect(await countResumeBootMessages(apiClient, task.session_id)).toBe(
resumeBootsAfterFirstPause,
);
await session.sendMessageViaButton("/e2e:simple-message");
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
expect(await countResumeBootMessages(apiClient, task.session_id)).toBe(
resumeBootsAfterFirstPause,
);
await assertNoDocumentHorizontalOverflow(testPage, "mobile accepted lazy resume pause");
} finally {
await apiClient.saveUserSettings({ prevent_auto_start_agent_on_open: false });
}
});
});
test.describe("mobile: failed resume recovery", () => {
@@ -6,8 +6,12 @@ import { waitForSessionState } from "../../helpers/session";
import { SessionPage } from "../../pages/session-page";
import {
cleanupDelayedResumeFixture,
countResumeBootMessages,
createFailOnResumeProfile,
readSessionMessageIdsContaining,
readSessionRuntimeIdentity,
seedDelayedResumeFixture,
waitForNewSessionMessage,
waitForSessionReady,
waitForQueuedCount,
} from "../../helpers/session-resume-prompt-queue";
@@ -153,6 +157,116 @@ test.describe("Session recovery", () => {
}
});
test("pausing an accepted lazy resume preserves the runtime for later turns", async ({
testPage,
apiClient,
seedData,
backend,
}) => {
test.setTimeout(150_000);
await apiClient.saveUserSettings({ prevent_auto_start_agent_on_open: true });
try {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Accepted lazy resume pause recovery",
seedData.agentProfileId,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
if (!task.session_id) throw new Error("accepted lazy resume task has no session_id");
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
await expect(session.chat.getByText("simple mock response", { exact: false })).toBeVisible({
timeout: 30_000,
});
await session.waitForChatIdle({ timeout: 30_000 });
await backend.restart();
await testPage.reload();
await session.waitForLoad();
await expect(testPage.getByTestId("composer-agent-start-hint")).toBeVisible({
timeout: 60_000,
});
const resumeBootsBeforeMessage = await countResumeBootMessages(apiClient, task.session_id);
// The slow response is the acceptance witness: provider output can only
// arrive after the resumed prompt crossed the dispatch callback.
await session.sendMessage("/slow 8s");
await expect(session.chat.getByText("Running slow response", { exact: false })).toBeVisible({
timeout: 30_000,
});
const initialRuntimeIdentity = await readSessionRuntimeIdentity(
apiClient,
task.id,
task.session_id,
);
await session.cancelAgentButton().click();
await waitForSessionState(apiClient, {
taskId: task.id,
sessionId: task.session_id,
expectedState: "WAITING_FOR_INPUT",
message: "Waiting for accepted lazy resume cancellation",
timeout: 30_000,
});
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
const resumeBootsAfterFirstPause = await countResumeBootMessages(apiClient, task.session_id);
expect(resumeBootsAfterFirstPause).toBe(resumeBootsBeforeMessage + 1);
// A second accepted pause must use the same process and conversation.
const slowResponseMessageIdsBeforeSecond = await readSessionMessageIdsContaining(
apiClient,
task.session_id,
"Running slow response",
);
expect(slowResponseMessageIdsBeforeSecond.size).toBeGreaterThan(0);
await session.sendMessage("/slow 8s");
await waitForNewSessionMessage(
apiClient,
task.session_id,
slowResponseMessageIdsBeforeSecond,
"Running slow response",
);
await session.cancelAgentButton().click();
await waitForSessionState(apiClient, {
taskId: task.id,
sessionId: task.session_id,
expectedState: "WAITING_FOR_INPUT",
message: "Waiting for the later accepted pause",
timeout: 30_000,
});
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
expect(await countResumeBootMessages(apiClient, task.session_id)).toBe(
resumeBootsAfterFirstPause,
);
await session.sendMessage("/e2e:simple-message");
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
expect(await readSessionRuntimeIdentity(apiClient, task.id, task.session_id)).toEqual(
initialRuntimeIdentity,
);
expect(await countResumeBootMessages(apiClient, task.session_id)).toBe(
resumeBootsAfterFirstPause,
);
} finally {
await apiClient.saveUserSettings({ prevent_auto_start_agent_on_open: false });
}
});
test("session startup keeps the composer editable and queues a submitted prompt", async ({
testPage,
apiClient,