dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix: keep transient retry banners to one notice (#3704)

Browse Source
This commit is contained in:
Carlos Florêncio
2026-09-16 10:38:31 +01:00
committed by GitHub
parent 88c6c0fe0a
commit 173e4c8c94
78 changed files with 2811 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/internal/orchestrator/dynamic_evidence.go
+52 -2
View File
@@ -55,12 +55,22 @@ func (s *Service) beginPromptAttempt(
if sessionID == "" {
return
}
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
s.dynamicAttemptEvidence.Store(sessionID, &promptAttemptEvidence{
executionID: executionID,
promptGeneration: promptGeneration,
evidenceKnown: true,
dynamic: dynamic,
})
// A new prompt must publish its complete execution identity before it opens
// a retired retry lifecycle to provider events. Initial launches have no
// execution yet; bindPromptAttempt clears the fence after launch acceptance.
if executionID != "" {
s.clearTransientRetryNoticeFenceLocked(sessionID, state)
}
state.mu.Unlock()
release()
}
func (s *Service) beginDynamicAttempt(sessionID string) {
@@ -117,6 +127,12 @@ func (s *Service) bindPromptAttempt(sessionID, executionID string, promptGenerat
if sessionID == "" || (executionID == "" && promptGeneration == 0) {
return
}
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
defer func() {
state.mu.Unlock()
release()
}()
evidence, ok := s.promptAttemptForSession(sessionID)
if !ok {
return
@@ -137,6 +153,11 @@ func (s *Service) bindPromptAttempt(sessionID, executionID string, promptGenerat
}
evidence.promptGeneration = promptGeneration
}
if executionID != "" {
// The evidence lock is released only after the identity is complete, while
// state.mu still excludes late failure handling from reopening the fence.
s.clearTransientRetryNoticeFenceLocked(sessionID, state)
}
}
func (s *Service) bindDynamicAttemptExecution(sessionID, executionID string) {
@@ -208,6 +229,25 @@ func (s *Service) observeDynamicAttempt(sessionID, executionID string, output, e
}
func (s *Service) withPromptAttemptEvidence(data watcher.AgentEventData) watcher.AgentEventData {
if data.SessionID == "" {
return data
}
state, release := s.acquireTransientRetryNoticeState(data.SessionID)
state.mu.Lock()
defer func() {
state.mu.Unlock()
release()
}()
if state.retired.Load() {
data.EvidenceKnown = false
data.OutputObserved = false
data.EffectObserved = false
return data
}
return s.withPromptAttemptEvidenceLocked(data)
}
func (s *Service) withPromptAttemptEvidenceLocked(data watcher.AgentEventData) watcher.AgentEventData {
if data.SessionID == "" {
return data
}
@@ -378,18 +418,28 @@ func (s *Service) promptAttemptForSession(sessionID string) (*promptAttemptEvide
func (e *promptAttemptEvidence) promptIdentityMatchesLocked(executionID string, promptGeneration uint64) bool {
if e.executionID != "" {
if executionID == "" || e.executionID != executionID {
if executionID == "" {
e.evidenceKnown = false
return false
}
if e.executionID != executionID {
// A concrete event from another execution is stale. Leave the current
// attempt intact so that the stale event cannot poison its evidence.
return false
}
} else if executionID != "" {
e.executionID = executionID
}
if e.promptGeneration != 0 {
if promptGeneration == 0 || e.promptGeneration != promptGeneration {
if promptGeneration == 0 {
e.evidenceKnown = false
return false
}
if e.promptGeneration != promptGeneration {
// As with execution IDs, a concrete older generation is a delayed
// event and must not invalidate the current prompt's evidence.
return false
}
} else if promptGeneration != 0 {
e.promptGeneration = promptGeneration
}
apps/backend/internal/orchestrator/event_handlers_transient.go
+431 -29
View File
@@ -3,6 +3,7 @@ package orchestrator
import (
"context"
"fmt"
"sort"
"sync"
"time"
@@ -21,6 +22,8 @@ const transientMaxAttempts = 5
const transientRetryStopTimeout = 30 * time.Second
const defaultTransientRetryNoticeFenceTTL = 5 * time.Minute
// recoverActionCancelRetry is the session.recover action that stops an
// in-progress transient retry loop and surfaces manual recovery.
const recoverActionCancelRetry = "cancel_retry"
@@ -100,10 +103,12 @@ type capturedPrompt struct {
// transientRetryEntry tracks one session's in-progress retry loop: the current
// attempt count and the cancel func for the armed backoff timer.
type transientRetryEntry struct {
attempt int
cancel func()
mu sync.Mutex
claimed bool
attempt int
cancel func()
retryCtx context.Context
mu sync.Mutex
claimed bool
armed bool
}
func (e *transientRetryEntry) claim() bool {
@@ -116,6 +121,129 @@ func (e *transientRetryEntry) claim() bool {
return true
}
func (e *transientRetryEntry) arm() bool {
e.mu.Lock()
defer e.mu.Unlock()
if e.armed {
return false
}
e.armed = true
return true
}
func (s *Service) transientRetryNoticeFenceDuration() time.Duration {
if s.transientRetryNoticeFenceTTL > 0 {
return s.transientRetryNoticeFenceTTL
}
return defaultTransientRetryNoticeFenceTTL
}
// acquireTransientRetryNoticeState returns the one state object for a
// session. The release function keeps the map entry alive until no caller can
// still hold its mutex, even when a lifecycle operation is waiting behind
// another caller.
func (s *Service) acquireTransientRetryNoticeState(sessionID string) (*transientRetryNoticeState, func()) {
if sessionID == "" {
return nil, func() {}
}
s.transientRetryNoticeStatesMu.Lock()
if s.transientRetryNoticeStates == nil {
s.transientRetryNoticeStates = make(map[string]*transientRetryNoticeState)
}
state := s.transientRetryNoticeStates[sessionID]
if state == nil {
state = &transientRetryNoticeState{}
s.transientRetryNoticeStates[sessionID] = state
}
state.refs++
s.transientRetryNoticeStatesMu.Unlock()
var releaseOnce sync.Once
release := func() {
releaseOnce.Do(func() {
s.releaseTransientRetryNoticeState(sessionID, state)
})
}
return state, release
}
func (s *Service) releaseTransientRetryNoticeState(sessionID string, state *transientRetryNoticeState) {
s.transientRetryNoticeStatesMu.Lock()
if state.refs > 0 {
state.refs--
}
s.reclaimTransientRetryNoticeStateLocked(sessionID, state)
s.transientRetryNoticeStatesMu.Unlock()
}
func (s *Service) reclaimTransientRetryNoticeStateLocked(sessionID string, state *transientRetryNoticeState) {
if state.refs != 0 || state.owned.Load() {
return
}
if state.retired.Load() && state.retiredUntil.Load() > time.Now().UnixNano() {
return
}
if current, ok := s.transientRetryNoticeStates[sessionID]; !ok || current != state {
return
}
if state.fenceTimer != nil {
state.fenceTimer.Stop()
state.fenceTimer = nil
}
delete(s.transientRetryNoticeStates, sessionID)
}
func (s *Service) retireTransientRetryNoticeLocked(sessionID string, state *transientRetryNoticeState) {
// Callers hold state.mu. Keep this order, state.mu -> statesMu, everywhere
// that touches the lifecycle map so a waiter cannot observe a replacement
// state while another caller still owns this state's mutex.
state.retired.Store(true)
ttl := s.transientRetryNoticeFenceDuration()
state.retiredUntil.Store(time.Now().Add(ttl).UnixNano())
s.transientRetryNoticeStatesMu.Lock()
if current, ok := s.transientRetryNoticeStates[sessionID]; ok && current == state {
if state.fenceTimer != nil {
state.fenceTimer.Stop()
}
state.fenceTimer = time.AfterFunc(ttl, func() {
s.expireTransientRetryNoticeFence(sessionID, state)
})
}
s.transientRetryNoticeStatesMu.Unlock()
}
func (s *Service) expireTransientRetryNoticeFence(sessionID string, state *transientRetryNoticeState) {
s.transientRetryNoticeStatesMu.Lock()
if current, ok := s.transientRetryNoticeStates[sessionID]; !ok || current != state {
s.transientRetryNoticeStatesMu.Unlock()
return
}
state.fenceTimer = nil
if state.retired.Load() {
if remaining := time.Until(time.Unix(0, state.retiredUntil.Load())); remaining > 0 {
state.fenceTimer = time.AfterFunc(remaining, func() {
s.expireTransientRetryNoticeFence(sessionID, state)
})
} else {
s.reclaimTransientRetryNoticeStateLocked(sessionID, state)
}
}
s.transientRetryNoticeStatesMu.Unlock()
}
func (s *Service) clearTransientRetryNoticeFenceLocked(sessionID string, state *transientRetryNoticeState) {
state.retired.Store(false)
state.retiredUntil.Store(0)
s.transientRetryNoticeStatesMu.Lock()
if current, ok := s.transientRetryNoticeStates[sessionID]; ok && current == state && state.fenceTimer != nil {
state.fenceTimer.Stop()
state.fenceTimer = nil
}
s.transientRetryNoticeStatesMu.Unlock()
}
// rememberTurnPrompt caches the raw outbound prompt so a transient retry can
// re-drive the same turn without the original caller's context.
func (s *Service) rememberTurnPrompt(sessionID, text, model string, planMode bool, attachments []v1.MessageAttachment) {
@@ -144,14 +272,30 @@ func (s *Service) rememberTurnPromptWithAccepted(
// handleRecoverableFailure); false for non-transient errors, office tasks,
// or an exhausted retry budget.
func (s *Service) handleTransientFailure(ctx context.Context, data watcher.AgentEventData) bool {
data = s.withPromptAttemptEvidence(data)
// Dynamic profiles own both error classes and their retry/reset policy. The
// legacy Kanban retry ladder must not consume a configured dynamic retry
// budget before the shared evaluator sees the failure.
if data.DynamicRouteAttempt {
return false
}
if !s.promptAttemptPreResultSafe(data) {
if data.SessionID == "" {
return false
}
noticeState, releaseNoticeState := s.acquireTransientRetryNoticeState(data.SessionID)
noticeState.mu.Lock()
if noticeState.retired.Load() {
noticeState.mu.Unlock()
releaseNoticeState()
s.logger.Debug("ignoring transient failure after retry lifecycle was retired",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID))
return true
}
data = s.withPromptAttemptEvidenceLocked(data)
if data.DynamicRouteAttempt || !s.promptAttemptPreResultSafe(data) {
noticeState.mu.Unlock()
releaseNoticeState()
s.logger.Debug("refusing automatic transient retry without safe prompt-attempt evidence",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
@@ -160,19 +304,22 @@ func (s *Service) handleTransientFailure(ctx context.Context, data watcher.Agent
return false
}
classified := classifyKanbanFailure(data)
if data.SessionID == "" || routingerr.Decide(routingerr.ContextKanban, classified, time.Now().UTC()) != routingerr.DecisionShortRetry {
if routingerr.Decide(routingerr.ContextKanban, classified, time.Now().UTC()) != routingerr.DecisionShortRetry {
noticeState.mu.Unlock()
releaseNoticeState()
return false
}
// Genuine Office-owned tasks render their
// own structured error UI — keep them on the existing path rather than the
// kanban-style yellow retry card. The canonical task projection avoids
// treating ordinary Kanban tasks with an assigned profile as Office tasks.
// Genuine Office-owned tasks render their own structured error UI. Keep them
// on the existing path rather than the Kanban-style yellow retry card.
if s.isOfficeTask(ctx, data.TaskID) {
noticeState.mu.Unlock()
releaseNoticeState()
return false
}
attempt := s.nextTransientAttempt(data.SessionID)
attempt := s.nextTransientAttemptLocked(data.SessionID)
if attempt > transientMaxAttempts {
noticeState.mu.Unlock()
releaseNoticeState()
s.logger.Warn("transient retry budget exhausted; falling through to recovery banner",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
@@ -192,16 +339,12 @@ func (s *Service) handleTransientFailure(ctx context.Context, data watcher.Agent
zap.Duration("delay", delay))
// Emit the yellow status (against the failed turn) before completing it.
s.createTransientRetryStatusMessage(ctx, data, classified, attempt, delay, retryAt)
s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "")
s.completeTurnForSession(ctx, data.SessionID)
// Park the session in WAITING_FOR_INPUT (a calm, banner-less state that
// also lets the yellow retry card render — ActionMessage hides itself while
// the session is RUNNING). Deliberately NOT FAILED and NOT task→REVIEW.
s.updateTaskSessionState(ctx, data.TaskID, data.SessionID, models.TaskSessionStateWaitingForInput, "", false)
s.scheduleTransientRetryWithMetadata(
s.createTransientRetryStatusMessageLocked(noticeState, ctx, data, classified, attempt, delay, retryAt)
// Reserve the next timer while the notice lifecycle is still serialized. A
// concurrent failure can replace this reservation, but cannot arm it until
// its own failed turn has been parked.
entry := s.reserveTransientRetryWithMetadataLocked(
noticeState,
data.TaskID,
data.SessionID,
data.AgentExecutionID,
@@ -210,12 +353,48 @@ func (s *Service) handleTransientFailure(ctx context.Context, data watcher.Agent
retryAt,
classified,
)
noticeState.mu.Unlock()
releaseNoticeState()
s.reconcileCIAutoFixTurnBeforeCompletion(ctx, data.TaskID, data.SessionID, "")
s.completeTurnForSession(ctx, data.SessionID)
// Park the session in WAITING_FOR_INPUT (a calm, banner-less state that
// also lets the yellow retry card render — ActionMessage hides itself while
// the session is RUNNING). Deliberately NOT FAILED and NOT task→REVIEW.
s.updateTaskSessionState(ctx, data.TaskID, data.SessionID, models.TaskSessionStateWaitingForInput, "", false)
// Parking must complete before a zero-delay retry can dispatch. Reacquiring
// the notice mutex also lets cancellation or a later failure replace this
// reservation before it is armed.
if entry != nil {
state, release := s.acquireTransientRetryNoticeState(data.SessionID)
state.mu.Lock()
if current, ok := s.transientRetries.Load(data.SessionID); ok && current == entry && !state.retired.Load() {
s.armTransientRetryEntryLocked(data.TaskID, data.SessionID, data.AgentExecutionID, entry, delay)
}
state.mu.Unlock()
release()
}
return true
}
// nextTransientAttempt returns the next 1-based attempt number for a session,
// cancelling any still-armed timer from a prior attempt.
func (s *Service) nextTransientAttempt(sessionID string) int {
state, release := s.acquireTransientRetryNoticeState(sessionID)
if state == nil {
return 1
}
state.mu.Lock()
attempt := s.nextTransientAttemptLocked(sessionID)
state.mu.Unlock()
release()
return attempt
}
func (s *Service) nextTransientAttemptLocked(sessionID string) int {
prev := 0
if v, ok := s.transientRetries.Load(sessionID); ok {
if entry, ok := v.(*transientRetryEntry); ok {
@@ -240,10 +419,57 @@ func (s *Service) scheduleTransientRetryWithMetadata(
retryAt time.Time,
classified *routingerr.Error,
) {
state, release := s.acquireTransientRetryNoticeState(sessionID)
if state == nil {
return
}
state.mu.Lock()
s.scheduleTransientRetryWithMetadataLocked(state, taskID, sessionID, execID, attempt, delay, retryAt, classified)
state.mu.Unlock()
release()
}
func (s *Service) scheduleTransientRetryWithMetadataLocked(
state *transientRetryNoticeState,
taskID, sessionID, execID string,
attempt int,
delay time.Duration,
retryAt time.Time,
classified *routingerr.Error,
) {
entry := s.reserveTransientRetryWithMetadataLocked(state, taskID, sessionID, execID, attempt, delay, retryAt, classified)
if entry != nil {
s.armTransientRetryEntryLocked(taskID, sessionID, execID, entry, delay)
}
}
func (s *Service) reserveTransientRetryWithMetadataLocked(
state *transientRetryNoticeState,
taskID, sessionID, execID string,
attempt int,
delay time.Duration,
retryAt time.Time,
classified *routingerr.Error,
) *transientRetryEntry {
if state.retired.Load() {
return nil
}
retryCtx, cancel := context.WithCancel(context.Background())
entry := &transientRetryEntry{attempt: attempt, cancel: cancel}
entry := &transientRetryEntry{attempt: attempt, cancel: cancel, retryCtx: retryCtx}
state.owned.Store(true)
s.transientRetries.Store(sessionID, entry)
go s.runTransientRetry(retryCtx, taskID, sessionID, execID, entry, delay)
return entry
}
func (s *Service) armTransientRetryEntryLocked(
taskID, sessionID, execID string,
entry *transientRetryEntry,
delay time.Duration,
) {
if entry == nil || !entry.arm() {
return
}
go s.runTransientRetry(entry.retryCtx, taskID, sessionID, execID, entry, delay)
}
// runTransientRetry waits out the backoff (or cancellation) then re-drives the
@@ -362,9 +588,43 @@ func (s *Service) createTransientRetryStatusMessage(
delay time.Duration,
retryAt time.Time,
) {
if s.messageCreator == nil {
state, release := s.acquireTransientRetryNoticeState(data.SessionID)
if state == nil {
return
}
state.mu.Lock()
s.createTransientRetryStatusMessageLocked(state, ctx, data, classified, attempt, delay, retryAt)
state.mu.Unlock()
release()
}
func (s *Service) createTransientRetryStatusMessageLocked(
state *transientRetryNoticeState,
ctx context.Context,
data watcher.AgentEventData,
classified *routingerr.Error,
attempt int,
delay time.Duration,
retryAt time.Time,
) {
if s.messageCreator == nil || state.retired.Load() {
return
}
content, meta := transientRetryStatusMessage(data, classified, attempt, delay, retryAt)
if s.transientRetryMessages != nil {
s.updateTransientRetryStatusMessageLocked(ctx, data, content, meta)
return
}
s.createTransientRetryStatusMessageRecord(ctx, data, content, meta)
}
func transientRetryStatusMessage(
data watcher.AgentEventData,
classified *routingerr.Error,
attempt int,
delay time.Duration,
retryAt time.Time,
) (string, map[string]interface{}) {
secs := int(delay.Seconds())
label := transientFailureLabel(classified)
content := fmt.Sprintf("%s — retrying in %ds (attempt %d/%d)", label, secs, attempt, transientMaxAttempts)
@@ -396,6 +656,15 @@ func (s *Service) createTransientRetryStatusMessage(
if providerID = routingerr.Sanitize(providerID); providerID != "" {
meta["provider_name"] = providerID
}
return content, meta
}
func (s *Service) createTransientRetryStatusMessageRecord(
ctx context.Context,
data watcher.AgentEventData,
content string,
meta map[string]interface{},
) {
if err := s.messageCreator.CreateSessionMessage(
ctx,
data.TaskID,
@@ -412,6 +681,75 @@ func (s *Service) createTransientRetryStatusMessage(
}
}
func (s *Service) updateTransientRetryStatusMessageLocked(
ctx context.Context,
data watcher.AgentEventData,
content string,
metadata map[string]interface{},
) {
// state.mu intentionally covers this DB I/O. Retirement and notice writes
// must serialize so cancellation cannot delete a row that this update then
// resurrects.
messages, err := s.transientRetryMessages.ListMessages(ctx, data.SessionID)
if err != nil {
s.logger.Warn("failed to list transient retry status messages before write",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.Error(err))
return
}
notices := transientRetryNotices(messages, data.TaskID, data.SessionID)
if len(notices) == 0 {
s.createTransientRetryStatusMessageRecord(ctx, data, content, metadata)
return
}
current := notices[0]
current.Content = content
current.Type = models.MessageTypeStatus
current.Metadata = metadata
current.RequestsInput = false
if err := s.transientRetryMessages.UpdateMessage(ctx, current); err != nil {
s.logger.Warn("failed to update transient retry status message",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("message_id", current.ID),
zap.Error(err))
return
}
for _, duplicate := range notices[1:] {
if err := s.transientRetryMessages.DeleteMessage(ctx, duplicate.ID); err != nil {
s.logger.Warn("failed to delete duplicate transient retry status message",
zap.String("task_id", data.TaskID),
zap.String("session_id", data.SessionID),
zap.String("message_id", duplicate.ID),
zap.Error(err))
}
}
}
func transientRetryNotices(messages []*models.Message, taskID, sessionID string) []*models.Message {
notices := make([]*models.Message, 0, len(messages))
for _, message := range messages {
if message == nil || message.TaskID != taskID || message.TaskSessionID != sessionID ||
message.Type != models.MessageTypeStatus || message.Metadata == nil {
continue
}
if retrying, ok := message.Metadata["retrying"].(bool); !ok || !retrying {
continue
}
notices = append(notices, message)
}
sort.SliceStable(notices, func(i, j int) bool {
if notices[i].CreatedAt.Equal(notices[j].CreatedAt) {
return notices[i].ID > notices[j].ID
}
return notices[i].CreatedAt.After(notices[j].CreatedAt)
})
return notices
}
func classifyKanbanFailure(data watcher.AgentEventData) *routingerr.Error {
providerID := data.AgentID
message := data.ErrorMessage
@@ -496,6 +834,18 @@ func transientFailureExhaustedMessage(classified *routingerr.Error) string {
// clearTransientRetryState clears a session's retry entry, cancels its timer,
// and drops the cached prompt (which may hold large/sensitive attachment data).
func (s *Service) clearTransientRetryState(sessionID string) bool {
if sessionID == "" {
return false
}
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
active := s.clearTransientRetryStateLocked(sessionID, state)
state.mu.Unlock()
release()
return active
}
func (s *Service) clearTransientRetryStateLocked(sessionID string, state *transientRetryNoticeState) bool {
s.lastTurnPrompt.Delete(sessionID)
v, ok := s.transientRetries.LoadAndDelete(sessionID)
if ok {
@@ -503,6 +853,7 @@ func (s *Service) clearTransientRetryState(sessionID string) bool {
entry.cancel()
}
}
state.owned.Store(false)
return ok
}
@@ -517,10 +868,27 @@ func (s *Service) resetTransientRetry(sessionID string) {
// persisted notice can outlive the in-memory retry entry. Normal successful
// turns skip the transcript scan when no retry loop was owned.
func (s *Service) resetTransientRetryWithContext(ctx context.Context, sessionID string, forceResolve bool) {
if !s.clearTransientRetryState(sessionID) && !forceResolve {
if sessionID == "" {
return
}
s.resolveTransientRetryMessages(context.WithoutCancel(ctx), sessionID)
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
s.resetTransientRetryWithContextLocked(state, ctx, sessionID, forceResolve)
state.mu.Unlock()
release()
}
func (s *Service) resetTransientRetryWithContextLocked(
state *transientRetryNoticeState,
ctx context.Context,
sessionID string,
forceResolve bool,
) {
if !s.clearTransientRetryStateLocked(sessionID, state) && !forceResolve {
return
}
s.retireTransientRetryNoticeLocked(sessionID, state)
s.resolveTransientRetryMessagesLocked(context.WithoutCancel(ctx), sessionID)
}
// resolveTransientRetryMessages removes every persisted retry status message
@@ -528,6 +896,18 @@ func (s *Service) resetTransientRetryWithContext(ctx context.Context, sessionID
// publication. Cleanup is intentionally non-fatal to the transition that
// ended the retry loop.
func (s *Service) resolveTransientRetryMessages(ctx context.Context, sessionID string) {
if sessionID == "" {
return
}
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
s.retireTransientRetryNoticeLocked(sessionID, state)
s.resolveTransientRetryMessagesLocked(ctx, sessionID)
state.mu.Unlock()
release()
}
func (s *Service) resolveTransientRetryMessagesLocked(ctx context.Context, sessionID string) {
if s.transientRetryMessages == nil || sessionID == "" {
return
}
@@ -555,6 +935,21 @@ func (s *Service) resolveTransientRetryMessages(ctx context.Context, sessionID s
}
}
// retireAndClearTransientRetryState closes the in-memory lifecycle under the
// notice mutex. Callers use it when they already hold the task runtime mutex;
// durable cleanup remains outside that broader runtime critical section.
func (s *Service) retireAndClearTransientRetryState(sessionID string) {
if sessionID == "" {
return
}
state, release := s.acquireTransientRetryNoticeState(sessionID)
state.mu.Lock()
s.retireTransientRetryNoticeLocked(sessionID, state)
s.clearTransientRetryStateLocked(sessionID, state)
state.mu.Unlock()
release()
}
// cancelAllTransientRetries drains every armed retry timer at shutdown.
func (s *Service) cancelAllTransientRetries() {
s.transientRetries.Range(func(key, _ interface{}) bool {
@@ -579,8 +974,15 @@ func (s *Service) CancelTransientRetry(ctx context.Context, taskID, sessionID st
if err := s.authorizeTaskSessionPair(ctx, taskID, sessionID); err != nil {
return false
}
noticeState, releaseNoticeState := s.acquireTransientRetryNoticeState(sessionID)
if noticeState == nil {
return false
}
noticeState.mu.Lock()
_, active := s.transientRetries.Load(sessionID)
s.resetTransientRetryWithContext(ctx, sessionID, true)
s.resetTransientRetryWithContextLocked(noticeState, ctx, sessionID, true)
noticeState.mu.Unlock()
releaseNoticeState()
if !active {
return false
}
apps/backend/internal/orchestrator/event_handlers_transient_ordering_test.go
+70
View File
@@ -0,0 +1,70 @@
package orchestrator
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/orchestrator/watcher"
"github.com/stretchr/testify/require"
)
type blockingTransientRetryTurnService struct {
fakeOrderingTurnService
completeStarted chan struct{}
releaseComplete chan struct{}
}
func (s *blockingTransientRetryTurnService) CompleteTurn(ctx context.Context, turnID string) error {
close(s.completeStarted)
<-s.releaseComplete
return s.fakeOrderingTurnService.CompleteTurn(ctx, turnID)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestHandleTransientFailure_ArmsRetryAfterTurnParking(t *testing.T) {
svc, _ := newTransientTestService(t)
t.Cleanup(svc.cancelAllTransientRetries)
turns := &blockingTransientRetryTurnService{
fakeOrderingTurnService: fakeOrderingTurnService{
bus: &recordingEventBus{},
turnID: "turn-1",
},
completeStarted: make(chan struct{}),
releaseComplete: make(chan struct{}),
}
svc.SetTurnService(turns)
armTransientPromptEvidence(svc)
done := make(chan bool, 1)
go func() {
done <- svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
PromptGeneration: 7,
ErrorMessage: overloaded529,
})
}()
select {
case <-turns.completeStarted:
case <-time.After(time.Second):
t.Fatal("turn completion did not start")
}
value, ok := svc.transientRetries.Load("s1")
require.True(t, ok)
entry := value.(*transientRetryEntry)
entry.mu.Lock()
armedWhileTurnOpen := entry.armed
entry.mu.Unlock()
require.False(t, armedWhileTurnOpen, "retry timer armed before the failed turn was parked")
close(turns.releaseComplete)
require.True(t, <-done)
entry.mu.Lock()
armedAfterParking := entry.armed
entry.mu.Unlock()
require.True(t, armedAfterParking)
}
apps/backend/internal/orchestrator/event_handlers_transient_resolution_test.go
+510 -3
View File
@@ -3,6 +3,7 @@ package orchestrator
import (
"context"
"errors"
"sync"
"testing"
"time"
@@ -17,11 +18,34 @@ import (
type transientRetryMessageServiceStub struct {
messages []*models.Message
listErr error
updateErr error
deleteErr error
listCalls int
updated []string
deleted []string
}
func armRetiredTransientPromptEvidence(svc *Service) {
svc.dynamicAttemptEvidence.Store("s1", &promptAttemptEvidence{
executionID: "execution-1",
promptGeneration: 7,
evidenceKnown: true,
})
}
type blockingTransientRetryMessageService struct {
*transientRetryMessageServiceStub
updateStarted chan struct{}
releaseUpdate chan struct{}
updateOnce sync.Once
}
func (s *blockingTransientRetryMessageService) UpdateMessage(ctx context.Context, message *models.Message) error {
s.updateOnce.Do(func() { close(s.updateStarted) })
<-s.releaseUpdate
return s.transientRetryMessageServiceStub.UpdateMessage(ctx, message)
}
func (s *transientRetryMessageServiceStub) ListMessages(context.Context, string) ([]*models.Message, error) {
s.listCalls++
if s.listErr != nil {
@@ -30,9 +54,32 @@ func (s *transientRetryMessageServiceStub) ListMessages(context.Context, string)
return s.messages, nil
}
func (s *transientRetryMessageServiceStub) UpdateMessage(_ context.Context, message *models.Message) error {
s.updated = append(s.updated, message.ID)
if s.updateErr != nil {
return s.updateErr
}
for _, existing := range s.messages {
if existing != nil && existing.ID == message.ID {
*existing = *message
return nil
}
}
return errors.New("message not found")
}
func (s *transientRetryMessageServiceStub) DeleteMessage(_ context.Context, id string) error {
s.deleted = append(s.deleted, id)
return s.deleteErr
if s.deleteErr != nil {
return s.deleteErr
}
for index, message := range s.messages {
if message != nil && message.ID == id {
s.messages = append(s.messages[:index], s.messages[index+1:]...)
break
}
}
return nil
}
func newPersistentTransientRetryTestService(t *testing.T) (*Service, *taskservice.Service, *recordingEventBus) {
@@ -96,9 +143,20 @@ func retryingMessages(t *testing.T, taskSvc *taskservice.Service) []*models.Mess
func TestResetTransientRetry_ResolvesPersistedNotices(t *testing.T) {
svc, taskSvc, eventBus := newPersistentTransientRetryTestService(t)
createPersistedTransientRetryNotice(t, svc)
createPersistedTransientRetryNoticeAttempt(t, svc, 2)
svc.scheduleTransientRetry("t1", "s1", "", 2, time.Hour)
// Seed a second row through the task service to preserve the legacy state
// that terminal cleanup must still repair after the writer becomes
// single-row.
_, err := taskSvc.CreateMessage(context.Background(), &taskservice.CreateMessageRequest{
TaskSessionID: "s1",
TaskID: "t1",
Content: "legacy retry notice",
AuthorType: "agent",
Type: string(models.MessageTypeStatus),
Metadata: map[string]interface{}{"retrying": true},
})
require.NoError(t, err)
svc.scheduleTransientRetry("t1", "s1", "", 2, time.Hour)
_, err = taskSvc.CreateMessage(context.Background(), &taskservice.CreateMessageRequest{
TaskSessionID: "s1",
TaskID: "t1",
Content: "unrelated status",
@@ -173,6 +231,455 @@ func TestNextTransientAttempt_DoesNotResolveCurrentNotice(t *testing.T) {
require.Len(t, retryingMessages(t, taskSvc), 1)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestTransientRetryStatusMessage_UpdatesSameNotice(t *testing.T) {
svc, taskSvc, eventBus := newPersistentTransientRetryTestService(t)
createPersistedTransientRetryNoticeAttempt(t, svc, 1)
first, err := taskSvc.ListMessages(context.Background(), "s1")
require.NoError(t, err)
require.Len(t, first, 1)
firstNotice := first[0]
svc.createTransientRetryStatusMessage(
context.Background(),
watcher.AgentEventData{TaskID: "t1", SessionID: "s1", AgentID: "codex-acp"},
nil,
2,
10*time.Second,
time.Now().UTC().Add(10*time.Second),
)
updated, err := taskSvc.ListMessages(context.Background(), "s1")
require.NoError(t, err)
require.Len(t, updated, 1)
updatedNotice := updated[0]
require.Equal(t, firstNotice.ID, updatedNotice.ID)
require.Equal(t, firstNotice.CreatedAt, updatedNotice.CreatedAt)
require.Equal(t, firstNotice.TurnID, updatedNotice.TurnID)
require.Equal(t, float64(2), updatedNotice.Metadata["attempt"])
require.Equal(t, float64(10), updatedNotice.Metadata["retry_in_seconds"])
require.Contains(t, updatedNotice.Content, "attempt 2/5")
var added, changed int
for _, recorded := range eventBus.events {
switch recorded.subject {
case events.MessageAdded:
added++
case events.MessageUpdated:
changed++
}
}
require.Equal(t, 1, added)
require.Equal(t, 1, changed)
}
func transientRetryTestMessage(id, taskID, sessionID string, createdAt time.Time, metadata map[string]interface{}) *models.Message {
return &models.Message{
ID: id,
TaskID: taskID,
TaskSessionID: sessionID,
TurnID: "turn-legacy",
AuthorType: models.MessageAuthorAgent,
Type: models.MessageTypeStatus,
Content: "legacy retry notice",
Metadata: metadata,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestTransientRetryStatusMessage_ConsolidatesDuplicatesAndIgnoresUnrelatedRows(t *testing.T) {
base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
store := &transientRetryMessageServiceStub{messages: []*models.Message{
transientRetryTestMessage("retry-later", "t1", "s1", base.Add(2*time.Second), map[string]interface{}{
"retrying": true,
"provider_name": "old-provider",
}),
transientRetryTestMessage("retry-earlier", "t1", "s1", base.Add(time.Second), map[string]interface{}{
"retrying": true,
"model_id": "old-model",
}),
transientRetryTestMessage("other-task", "t2", "s1", base, map[string]interface{}{"retrying": true}),
transientRetryTestMessage("other-session", "t1", "s2", base, map[string]interface{}{"retrying": true}),
{
ID: "wrong-type",
TaskID: "t1",
TaskSessionID: "s1",
Type: models.MessageTypeMessage,
Metadata: map[string]interface{}{"retrying": true},
},
{
ID: "not-retrying",
TaskID: "t1",
TaskSessionID: "s1",
Type: models.MessageTypeStatus,
Metadata: map[string]interface{}{"retrying": "true"},
},
{
ID: "unrelated-status",
TaskID: "t1",
TaskSessionID: "s1",
Type: models.MessageTypeStatus,
Metadata: map[string]interface{}{"retrying": false},
},
nil,
}}
creator := &mockMessageCreator{}
svc := &Service{
logger: testLogger(),
messageCreator: creator,
transientRetryMessages: store,
}
svc.createTransientRetryStatusMessage(
context.Background(),
watcher.AgentEventData{TaskID: "t1", SessionID: "s1", AgentID: "codex-acp"},
nil,
3,
20*time.Second,
base.Add(20*time.Second),
)
require.Equal(t, []string{"retry-later"}, store.updated)
require.Equal(t, []string{"retry-earlier"}, store.deleted)
require.Empty(t, creator.sessionMessages)
later := store.messages[0]
require.Contains(t, later.Content, "attempt 3/5")
require.Equal(t, "codex-acp", later.Metadata["provider_name"])
require.NotContains(t, later.Metadata, "model_id")
remaining := transientRetryNotices(store.messages, "t1", "s1")
require.Len(t, remaining, 1)
require.Equal(t, "retry-later", remaining[0].ID)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestTransientRetryStatusMessage_StoreErrorsDoNotCreateExtraNotice(t *testing.T) {
data := watcher.AgentEventData{TaskID: "t1", SessionID: "s1", AgentID: "codex-acp"}
t.Run("list error", func(t *testing.T) {
store := &transientRetryMessageServiceStub{listErr: errors.New("list failed")}
creator := &mockMessageCreator{}
svc := &Service{logger: testLogger(), messageCreator: creator, transientRetryMessages: store}
svc.createTransientRetryStatusMessage(context.Background(), data, nil, 1, time.Second, time.Now().UTC())
require.Empty(t, creator.sessionMessages)
require.Empty(t, store.updated)
require.Empty(t, store.deleted)
})
t.Run("update error", func(t *testing.T) {
store := &transientRetryMessageServiceStub{
messages: []*models.Message{transientRetryTestMessage("retry-1", "t1", "s1", time.Now().UTC(), map[string]interface{}{"retrying": true})},
updateErr: errors.New("update failed"),
}
creator := &mockMessageCreator{}
svc := &Service{logger: testLogger(), messageCreator: creator, transientRetryMessages: store}
svc.createTransientRetryStatusMessage(context.Background(), data, nil, 2, time.Second, time.Now().UTC())
require.Equal(t, []string{"retry-1"}, store.updated)
require.Empty(t, creator.sessionMessages)
require.Empty(t, store.deleted)
})
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestTransientRetryStatusMessage_RetainsDuplicatesAfterDeleteErrorForNextWrite(t *testing.T) {
base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
store := &transientRetryMessageServiceStub{
messages: []*models.Message{
transientRetryTestMessage("retry-1", "t1", "s1", base, map[string]interface{}{"retrying": true}),
transientRetryTestMessage("retry-2", "t1", "s1", base.Add(time.Second), map[string]interface{}{"retrying": true}),
},
deleteErr: errors.New("delete failed"),
}
svc := &Service{
logger: testLogger(),
messageCreator: &mockMessageCreator{},
transientRetryMessages: store,
}
write := func(attempt int) {
svc.createTransientRetryStatusMessage(
context.Background(),
watcher.AgentEventData{TaskID: "t1", SessionID: "s1", AgentID: "codex-acp"},
nil,
attempt,
time.Second,
base.Add(time.Duration(attempt)*time.Second),
)
}
write(2)
require.Equal(t, []string{"retry-1"}, store.deleted)
require.Len(t, store.messages, 2)
store.deleteErr = nil
write(3)
require.Equal(t, []string{"retry-1", "retry-1"}, store.deleted)
require.Len(t, store.messages, 1)
require.Equal(t, "retry-2", store.messages[0].ID)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestTransientRetryStatusMessage_SerializesRetirement(t *testing.T) {
store := &blockingTransientRetryMessageService{
transientRetryMessageServiceStub: &transientRetryMessageServiceStub{
messages: []*models.Message{
transientRetryTestMessage(
"retry-1",
"t1",
"s1",
time.Now().UTC(),
map[string]interface{}{"retrying": true},
),
},
},
updateStarted: make(chan struct{}),
releaseUpdate: make(chan struct{}),
}
svc := &Service{
logger: testLogger(),
messageCreator: &mockMessageCreator{},
transientRetryMessages: store,
}
data := watcher.AgentEventData{TaskID: "t1", SessionID: "s1", AgentID: "codex-acp"}
writeDone := make(chan struct{})
go func() {
svc.createTransientRetryStatusMessage(
context.Background(),
data,
nil,
2,
10*time.Second,
time.Now().UTC().Add(10*time.Second),
)
close(writeDone)
}()
select {
case <-store.updateStarted:
case <-time.After(time.Second):
t.Fatal("retry notice update did not start")
}
retireDone := make(chan struct{})
go func() {
svc.resetTransientRetryWithContext(context.Background(), "s1", true)
close(retireDone)
}()
select {
case <-retireDone:
t.Fatal("retry notice retirement overtook the in-flight update")
case <-time.After(100 * time.Millisecond):
}
close(store.releaseUpdate)
select {
case <-writeDone:
case <-time.After(time.Second):
t.Fatal("retry notice update did not finish")
}
select {
case <-retireDone:
case <-time.After(time.Second):
t.Fatal("retry notice retirement did not finish")
}
require.Empty(t, transientRetryNotices(store.messages, "t1", "s1"))
require.Equal(t, []string{"retry-1"}, store.updated)
require.Equal(t, []string{"retry-1"}, store.deleted)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestHandleTransientFailure_RetiredLifecycleIgnoresLateFailure(t *testing.T) {
t.Run("explicit cancellation", func(t *testing.T) {
svc, taskSvc, _ := newPersistentTransientRetryTestService(t)
svc.rememberTurnPrompt("s1", "retry me", "", false, nil)
svc.scheduleTransientRetry("t1", "s1", "execution-1", 1, time.Hour)
require.True(t, svc.CancelTransientRetry(context.Background(), "t1", "s1"))
armRetiredTransientPromptEvidence(svc)
require.True(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
PromptGeneration: 7,
ErrorMessage: overloaded529,
}), "a late eligible failure must be consumed after cancellation")
require.Empty(t, retryingMessages(t, taskSvc))
_, retryArmed := svc.transientRetries.Load("s1")
require.False(t, retryArmed)
})
t.Run("terminal session", func(t *testing.T) {
svc, taskSvc, _ := newPersistentTransientRetryTestService(t)
repo, ok := svc.repo.(*sqliterepo.Repository)
require.True(t, ok)
svc.rememberTurnPrompt("s1", "retry me", "", false, nil)
svc.scheduleTransientRetry("t1", "s1", "execution-1", 1, time.Hour)
require.NoError(t, repo.UpdateTaskSessionState(
context.Background(), "s1", models.TaskSessionStateCancelled, "terminal cleanup",
))
// The production agent-failed path performs the terminal retirement
// before it drops the failure. The later call represents a buffered
// eligible provider event from that same execution.
svc.handleAgentFailed(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
ErrorMessage: overloaded529,
})
armRetiredTransientPromptEvidence(svc)
require.True(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
PromptGeneration: 7,
ErrorMessage: overloaded529,
}), "a late eligible failure must be consumed after terminal retirement")
require.Empty(t, retryingMessages(t, taskSvc))
_, retryArmed := svc.transientRetries.Load("s1")
require.False(t, retryArmed)
})
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestHandleTransientFailure_NewPromptAfterRetirementStartsAtAttemptOne(t *testing.T) {
svc, taskSvc, _ := newPersistentTransientRetryTestService(t)
t.Cleanup(svc.cancelAllTransientRetries)
svc.resetTransientRetryWithContext(context.Background(), "s1", true)
svc.rememberTurnPrompt("s1", "new prompt", "", false, nil)
armTransientPromptEvidence(svc)
require.True(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
PromptGeneration: 7,
ErrorMessage: overloaded529,
}))
notices := retryingMessages(t, taskSvc)
require.Len(t, notices, 1)
require.Equal(t, float64(1), notices[0].Metadata["attempt"])
entryValue, ok := svc.transientRetries.Load("s1")
require.True(t, ok)
require.Equal(t, 1, entryValue.(*transientRetryEntry).attempt)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestHandleTransientFailure_RetiredFenceWaitsForNewPromptIdentity(t *testing.T) {
svc, taskSvc, _ := newPersistentTransientRetryTestService(t)
t.Cleanup(svc.cancelAllTransientRetries)
svc.rememberTurnPrompt("s1", "old prompt", "", false, nil)
svc.scheduleTransientRetry("t1", "s1", "execution-old", 1, time.Hour)
require.True(t, svc.CancelTransientRetry(context.Background(), "t1", "s1"))
// A new initial prompt has a generation before its replacement execution is
// known. It must not open the retired lifecycle during that interval.
svc.rememberTurnPrompt("s1", "new prompt", "", false, nil)
svc.beginInitialPromptAttempt("s1", false)
require.True(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-old",
PromptGeneration: 1,
ErrorMessage: overloaded529,
}), "a late failure while the replacement execution is unbound must be consumed")
require.Empty(t, retryingMessages(t, taskSvc))
// Binding the replacement identity is the admission boundary that opens the
// fence. The same late event must then fail closed on identity mismatch.
svc.bindPromptAttempt("s1", "execution-new", 1)
require.False(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-old",
PromptGeneration: 1,
ErrorMessage: overloaded529,
}))
require.Empty(t, retryingMessages(t, taskSvc))
require.True(t, svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-new",
PromptGeneration: 1,
ErrorMessage: overloaded529,
}))
notices := retryingMessages(t, taskSvc)
require.Len(t, notices, 1)
require.Equal(t, float64(1), notices[0].Metadata["attempt"])
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
func TestHandleTransientFailure_ConcurrentFailuresShareNoticeAndTimer(t *testing.T) {
store := &blockingTransientRetryMessageService{
transientRetryMessageServiceStub: &transientRetryMessageServiceStub{
messages: []*models.Message{
transientRetryTestMessage(
"retry-1",
"t1",
"s1",
time.Now().UTC(),
map[string]interface{}{"retrying": true},
),
},
},
updateStarted: make(chan struct{}),
releaseUpdate: make(chan struct{}),
}
svc, _ := newTransientTestService(t)
svc.transientRetryMessages = store
t.Cleanup(svc.cancelAllTransientRetries)
svc.rememberTurnPrompt("s1", "retry me", "", false, nil)
armTransientPromptEvidence(svc)
data := watcher.AgentEventData{
TaskID: "t1",
SessionID: "s1",
AgentExecutionID: "execution-1",
PromptGeneration: 7,
ErrorMessage: overloaded529,
}
firstDone := make(chan bool, 1)
go func() { firstDone <- svc.handleTransientFailure(context.Background(), data) }()
select {
case <-store.updateStarted:
case <-time.After(time.Second):
t.Fatal("first transient failure did not enter the notice update")
}
secondStarted := make(chan struct{})
secondDone := make(chan bool, 1)
go func() {
close(secondStarted)
secondDone <- svc.handleTransientFailure(context.Background(), data)
}()
<-secondStarted
// The first update holds the per-session notice mutex. Releasing its
// deterministic store barrier lets the second failure take the next
// attempt under the same mutex.
close(store.releaseUpdate)
require.True(t, <-firstDone)
require.True(t, <-secondDone)
notices := transientRetryNotices(store.messages, "t1", "s1")
require.Len(t, notices, 1)
require.Equal(t, 2, notices[0].Metadata["attempt"])
entryValue, ok := svc.transientRetries.Load("s1")
require.True(t, ok)
require.Equal(t, 2, entryValue.(*transientRetryEntry).attempt)
}
// @covers AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.10
func TestResolveTransientRetryMessages_SwallowsStoreErrors(t *testing.T) {
listErr := errors.New("list failed")
apps/backend/internal/orchestrator/event_handlers_transient_state_test.go
+110
View File
@@ -0,0 +1,110 @@
package orchestrator
import (
"context"
"strconv"
"testing"
"time"
"github.com/kandev/kandev/internal/task/models"
)
func transientRetryNoticeStateCount(svc *Service) int {
svc.transientRetryNoticeStatesMu.Lock()
defer svc.transientRetryNoticeStatesMu.Unlock()
return len(svc.transientRetryNoticeStates)
}
func TestTransientRetryNoticeState_CachedPromptDoesNotOwnState(t *testing.T) {
svc := &Service{}
svc.rememberTurnPrompt("uncertain", "prompt", "", false, nil)
if got := transientRetryNoticeStateCount(svc); got != 0 {
t.Fatalf("notice state entries for an unaccepted prompt = %d, want 0", got)
}
}
func TestTransientRetryNoticeState_ReclaimsSuccessfulSessionChurn(t *testing.T) {
svc := &Service{}
for i := 0; i < 1000; i++ {
sessionID := "churn-" + strconv.Itoa(i)
svc.rememberTurnPrompt(sessionID, "prompt", "", false, nil)
svc.resetTransientRetry(sessionID)
}
if got := transientRetryNoticeStateCount(svc); got != 0 {
t.Fatalf("notice state entries after successful session churn = %d, want 0", got)
}
}
func TestTransientRetryNoticeState_ReclaimsDeletedSessionAfterFence(t *testing.T) {
repo := setupTestRepo(t)
seedTaskAndSession(t, repo, "task-delete", "deleted", models.TaskSessionStateCompleted)
svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), &mockAgentManager{})
svc.transientRetryNoticeFenceTTL = 10 * time.Millisecond
svc.rememberTurnPrompt("deleted", "prompt", "", false, nil)
svc.scheduleTransientRetry("task-delete", "deleted", "execution-1", 1, time.Hour)
t.Cleanup(svc.cancelAllTransientRetries)
// Session deletion ends the session incarnation even when no execution
// remains. The fence must remain while stale events can still arrive, then
// reclaim itself.
if err := svc.DeleteSession(context.Background(), "deleted"); err != nil {
t.Fatalf("DeleteSession: %v", err)
}
if got := transientRetryNoticeStateCount(svc); got != 1 {
t.Fatalf("notice state entries immediately after deletion = %d, want 1 fenced entry", got)
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if transientRetryNoticeStateCount(svc) == 0 {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("deleted session notice state was not reclaimed after bounded fence")
}
func TestTransientRetryNoticeState_ReferenceCountsConcurrentUsers(t *testing.T) {
svc := &Service{}
first, releaseFirst := svc.acquireTransientRetryNoticeState("shared")
first.mu.Lock()
secondState := make(chan *transientRetryNoticeState, 1)
secondAcquired := make(chan struct{})
secondLocked := make(chan struct{})
secondDone := make(chan struct{})
go func() {
second, releaseSecond := svc.acquireTransientRetryNoticeState("shared")
secondState <- second
close(secondAcquired)
second.mu.Lock()
close(secondLocked)
second.mu.Unlock()
releaseSecond()
close(secondDone)
}()
<-secondAcquired
if got := transientRetryNoticeStateCount(svc); got != 1 {
t.Fatalf("notice state entries with a mutex holder and waiter = %d, want 1", got)
}
if second := <-secondState; second != first {
t.Fatal("concurrent users acquired different notice state mutexes")
}
first.mu.Unlock()
<-secondLocked
releaseFirst()
select {
case <-secondDone:
case <-time.After(time.Second):
t.Fatal("concurrent notice state user did not finish")
}
if got := transientRetryNoticeStateCount(svc); got != 0 {
t.Fatalf("notice state entries after concurrent users released = %d, want 0", got)
}
}
apps/backend/internal/orchestrator/service.go
+27 -5
View File
@@ -143,14 +143,29 @@ type MessageCreator interface {
InvalidateModelCache(sessionID string)
}
// TransientRetryMessageService is the narrow task-service seam used to retire
// persisted retry status messages. The task service owns authorization and
// event-bus publication for both operations.
// TransientRetryMessageService is the narrow task-service seam used to write
// and retire persisted retry status messages. The task service owns
// authorization and event-bus publication for all operations.
type TransientRetryMessageService interface {
ListMessages(ctx context.Context, sessionID string) ([]*models.Message, error)
UpdateMessage(ctx context.Context, message *models.Message) error
DeleteMessage(ctx context.Context, id string) error
}
// transientRetryNoticeState serializes retry-notice storage with the in-memory
// retry lifecycle for one session. refs and fenceTimer are owned by
// Service.transientRetryNoticeStatesMu. The bounded retirement fence keeps a
// stale provider event from creating a new retry while terminal cleanup is in
// flight, then allows the session entry to be reclaimed.
type transientRetryNoticeState struct {
mu sync.Mutex
retired atomic.Bool
owned atomic.Bool
retiredUntil atomic.Int64
refs int
fenceTimer *time.Timer
}
// SessionAttachmentCleaner removes file-backed prompt attachments after a
// task session has been deleted. The task service owns storage and authorization
// details; the orchestrator only coordinates the post-commit lifecycle hook.
@@ -681,6 +696,13 @@ type Service struct {
// transientRetryMessages owns durable cleanup of persisted retry notices.
// It is optional for focused tests and pre-composition callers.
transientRetryMessages TransientRetryMessageService
// transientRetryNoticeStates serializes notice writes and cleanup by
// session. Entries are reference counted while callers use the mutex,
// retained while a prompt/retry is active, and retained for a bounded fence
// interval after retirement.
transientRetryNoticeStatesMu sync.Mutex
transientRetryNoticeStates map[string]*transientRetryNoticeState
transientRetryNoticeFenceTTL time.Duration
// sessionQueuePurgeNotifierRegistered means the task repository owns
// queue cleanup and status publication after DeleteTaskSession commits.
// DeleteSession keeps its fallback cleanup for focused compositions that
@@ -1869,8 +1891,8 @@ func (s *Service) SetMessageCreator(mc MessageCreator) {
s.messageCreator = mc
}
// SetTransientRetryMessageService wires the task service used to retire
// persisted transient-retry status messages.
// SetTransientRetryMessageService wires the task service used to write and
// retire persisted transient-retry status messages.
func (s *Service) SetTransientRetryMessageService(service TransientRetryMessageService) {
s.transientRetryMessages = service
}
apps/backend/internal/orchestrator/task_launch_failure_test.go
+7
View File
@@ -20,6 +20,7 @@ func TestHandledLaunchFailureLeavesTypedErrorOwnerWithoutLegacyGuidance(t *testi
svc := createTestService(repo, newMockStepGetter(), taskRepo)
svc.messageCreator = messages
err := errors.New("environment preparation failed: fatal: couldn't find remote ref feature/deleted")
svc.rememberTurnPrompt("session1", "retry payload", "", false, nil)
if got := svc.handleSessionLaunchFailure(ctx, "task1", "session1", err); !errors.Is(got, err) {
t.Fatalf("handleSessionLaunchFailure error = %v, want %v", got, err)
}
@@ -30,6 +31,12 @@ func TestHandledLaunchFailureLeavesTypedErrorOwnerWithoutLegacyGuidance(t *testi
if _, suppressed := svc.suppressToast.Load("session1"); suppressed {
t.Fatal("typed launch failure must not suppress the pointer toast")
}
if _, cached := svc.lastTurnPrompt.Load("session1"); cached {
t.Fatal("launch failure retained the cached prompt")
}
if got := transientRetryNoticeStateCount(svc); got != 0 {
t.Fatalf("notice state entries after launch failure = %d, want 0", got)
}
session, getErr := repo.GetTaskSession(ctx, "session1")
if getErr != nil {
t.Fatalf("get failed session: %v", getErr)
apps/backend/internal/orchestrator/task_operations.go
+41 -5
View File
@@ -12,6 +12,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"go.uber.org/zap"
@@ -959,6 +960,10 @@ func (s *Service) handleSessionLaunchFailure(
) error {
failureCtx := context.WithoutCancel(ctx)
safeErr := routingerr.SanitizeError(launchErr)
// A launch failure cannot produce a retryable prompt lifecycle. Release the
// replay payload here so an early admission or startup failure does not keep
// attachment data alive until a later session event.
s.clearTransientRetryState(sessionID)
_ = s.recordSessionLaunchFailure(
failureCtx, taskID, sessionID, safeErr, preloadedSession...,
)
@@ -4290,7 +4295,7 @@ func (s *Service) stopTaskSessionForCoordinatorLocked(
// Halt-only intent also disarms any provider-backoff retry. This must run
// even when the failed execution has already disappeared and the result is
// therefore not_running; otherwise its timer can launch replacement work.
s.clearTransientRetryState(sessionID)
s.retireAndClearTransientRetryState(sessionID)
result, stopErr := s.executor.StopSessionDetailed(ctx, session, coordinatorMCPStopReason, false)
if stopErr == nil && result.Changed {
// Cancellation takes effect before detached runtime teardown. Tombstone
@@ -4447,6 +4452,10 @@ func (s *Service) DeleteSession(ctx context.Context, sessionID string) error {
if err := s.quiesceSessionExecutionBeforeDeletion(ctx, taskID, sessionID); err != nil {
return err
}
// Deletion ends the session incarnation even when no execution remains.
// Retire the retry loop before removing the row so a buffered provider
// failure cannot recreate notice state for a deleted session ID.
s.resetTransientRetryWithContext(ctx, sessionID, true)
s.logger.Info("deleting session",
zap.String("session_id", sessionID),
@@ -5287,7 +5296,11 @@ type promptTaskOptions struct {
// onAccepted runs at the agentctl acceptance boundary, before PromptTask
// waits for the turn to finish. Automation callers use it to bind durable
// attempt identity to the exact turn.
onAccepted func(turnID string)
onAccepted func(turnID string)
// promptAccepted is owned by promptTask and keeps the replay cache alive
// only when this prompt reaches provider acceptance. It is process-local
// plumbing and is never passed to a caller.
promptAccepted *atomic.Bool
expectedSessionIdentity *messagequeue.QueueSessionIdentity
// promptAlreadyComposed and fallbackRetryPrompt mirror the composed-prompt
// seam autoStartStepPrompt's own ErrExecutionNotFound branch uses (see
@@ -5420,9 +5433,6 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom
return nil, err
}
// Apply config-mode and plan-mode prompt transforms.
effectivePrompt := s.effectivePromptForSession(sessionID, prompt, planMode, session)
// After a lazy backend restart the session may be WAITING_FOR_INPUT with no agent process yet.
_, hadExecutionBeforeEnsure := s.executor.GetExecutionBySession(sessionID)
resumedForPrompt := options.resumeAttempt != nil || !hadExecutionBeforeEnsure
@@ -5466,6 +5476,17 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom
}
runBeforeDispatch := runBeforeDispatchOnce(options.beforeDispatch)
// Keep the replay payload only after this provider attempt is accepted.
// Admission and dispatch failures must not retain prompt attachments until
// a later session event happens to replace or consume the cache.
var promptAccepted atomic.Bool
options.promptAccepted = &promptAccepted
defer func() {
if !promptAccepted.Load() {
s.lastTurnPrompt.Delete(sessionID)
}
}()
// Cache the replay identity and acquire the model-switch guard before switching.
modelSwitchGuard, err := s.prepareModelSwitchGuard(
resumePromptCtx, taskID, sessionID, prompt, model, planMode, attachments, options, session,
@@ -5487,6 +5508,9 @@ func (s *Service) promptTask(ctx context.Context, taskID, sessionID string, prom
resumeAttempt, modelSwitchGuard,
)
if switchHandled {
if switchErr == nil {
promptAccepted.Store(true)
}
return switchResult, switchErr
}
@@ -5811,6 +5835,9 @@ func (s *Service) finishPromptExecutorDispatch(
resumeAttempt *resumeAttempt,
) (*PromptResult, error) {
dispatchAccepted, publicationErr := dispatchOutcome.snapshot()
if dispatchAccepted && options.promptAccepted != nil {
options.promptAccepted.Store(true)
}
if execErr != nil {
// Missing-execution recovery reacquires the cancel guard while it resets
// the session. Release dispatch admission before entering that path.
@@ -5906,6 +5933,15 @@ func (s *Service) preparePromptDispatchCallback(
dispatchOutcome = &promptDispatchOutcome{}
dispatchOutcome.turnID = rollback.turnID
dispatchOutcome.onAccepted = options.onAccepted
if options.promptAccepted != nil {
originalOnAccepted := dispatchOutcome.onAccepted
dispatchOutcome.onAccepted = func(turnID string) {
options.promptAccepted.Store(true)
if originalOnAccepted != nil {
originalOnAccepted(turnID)
}
}
}
onDispatched = s.promptDispatchCallbackForIdentity(
promptCtx, taskID, sessionID, rollback.sessionIdentity,
session.AgentExecutionID, rollback.reservedTurn, foregroundDispatch, dispatchOutcome,
apps/web/components/task/chat/message-list-native-scroll.ts
+18 -1
View File
@@ -454,6 +454,20 @@ function useLazyLoadSentinel(params: {
}, []);
const shouldContinueWhileIntersecting = useCallback(() => {
const scrollRoot = scrollRef.current;
const sentinel = sentinelNodeRef.current;
const request = requestRef.current;
const visibleBoundaryChanged =
request !== null && getOldestVisibleBoundaryKey(itemsRef.current) !== request.boundaryBefore;
return Boolean(
hasMoreRef.current &&
scrollRoot &&
sentinel &&
!visibleBoundaryChanged &&
isElementInPreloadRegion(scrollRoot, sentinel, TRANSCRIPT_SENTINEL_ROOT_MARGIN),
);
}, [scrollRef]);
const isCurrentGeometryEligible = useCallback(() => {
const scrollRoot = scrollRef.current;
const sentinel = sentinelNodeRef.current;
return Boolean(
@@ -476,7 +490,10 @@ function useLazyLoadSentinel(params: {
shouldContinueWhileIntersecting,
// Continuation and lifecycle/input eligibility both require the
// sentinel to remain inside the transcript's current preload region.
isCurrentGeometryEligible: shouldContinueWhileIntersecting,
// The continuation predicate additionally stops after a visible row is
// added; fresh upward input must still be allowed to start the next
// request from the new boundary.
isCurrentGeometryEligible,
onLoadSettled: reportSettle,
isRequestCurrent,
},
apps/web/components/task/chat/message-list-native.test.tsx
+2 -2
View File
@@ -853,7 +853,7 @@ describe("useNativeScrollManagement transcript pagination", () => {
await pendingLoad;
});
it("continues while the sentinel remains in preload even when the visible boundary changes", async () => {
it("continues while the sentinel remains in preload and the visible boundary is stable", async () => {
const loadMore = vi.fn(async () => 20);
const newest = transcriptMessage("newest");
const { rerender } = render(
@@ -886,7 +886,7 @@ describe("useNativeScrollManagement transcript pagination", () => {
options = sharedSentinelCalls.at(-1)?.[5] as {
shouldContinueWhileIntersecting: () => boolean;
};
expect(options.shouldContinueWhileIntersecting()).toBe(true);
expect(options.shouldContinueWhileIntersecting()).toBe(false);
});
it("anchors a prepend below a fixed task description row", () => {
apps/web/e2e/helpers/api-client.test.ts
+73 -1
View File
@@ -1,5 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { ApiClient } from "./api-client";
import { ApiClient, removeRoutingProfileReferences } from "./api-client";
import { loadInterimSettingsInterlockToken } from "./interim-settings-interlock";
describe("ApiClient.createAgentProfile", () => {
afterEach(() => {
@@ -85,3 +86,74 @@ describe("ApiClient user settings", () => {
expect(saved).toEqual(baseline);
});
});
describe("loadInterimSettingsInterlockToken", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("retries the backend startup response before reading the token", async () => {
let attempts = 0;
vi.stubGlobal(
"fetch",
vi.fn(async () => {
attempts += 1;
if (attempts < 3) return new Response(null, { status: 503 });
return Response.json({ interimSettingsInterlockToken: "ready-token" });
}),
);
await expect(loadInterimSettingsInterlockToken("http://backend.test")).resolves.toBe(
"ready-token",
);
expect(attempts).toBe(3);
});
});
describe("removeRoutingProfileReferences", () => {
it("removes role-tier overrides whose last execution profile was deleted", () => {
const updated = removeRoutingProfileReferences(
{
enabled: true,
provider_order: ["claude-acp", "codex-acp"],
default_tier: "balanced",
provider_profiles: {
"claude-acp": {
execution_profile_ids: { balanced: "profile-1", economy: "economy-1" },
tier_map: { balanced: "balanced-model", economy: "economy-model" },
},
"codex-acp": {
execution_profile_ids: { balanced: "profile-2" },
},
},
role_tiers: { ceo: "balanced", worker: "economy" },
},
"profile-1",
);
expect(updated?.role_tiers).toEqual({ ceo: "balanced", worker: "economy" });
expect(updated?.provider_profiles["claude-acp"]).toEqual({
execution_profile_ids: { economy: "economy-1" },
tier_map: { economy: "economy-model" },
});
});
it("drops a role-tier override when no provider still maps that tier", () => {
const updated = removeRoutingProfileReferences(
{
enabled: true,
provider_order: ["claude-acp"],
default_tier: "balanced",
provider_profiles: {
"claude-acp": {
execution_profile_ids: { balanced: "profile-1" },
},
},
role_tiers: { ceo: "balanced" },
},
"profile-1",
);
expect(updated?.role_tiers).toEqual({});
});
});
apps/web/e2e/helpers/api-client.ts
+23 -1
View File
@@ -3823,6 +3823,7 @@ type WorkspaceRoutingConfig = {
provider_order: string[];
default_tier: string;
provider_profiles: Record<string, RoutingProviderProfile>;
role_tiers?: Record<string, string>;
[key: string]: unknown;
};
@@ -3847,7 +3848,7 @@ function routingWorkspaceIDs(body: string): string[] {
}
}
function removeRoutingProfileReferences(
export function removeRoutingProfileReferences(
config: WorkspaceRoutingConfig,
profileId: string,
): WorkspaceRoutingConfig | undefined {
@@ -3878,8 +3879,18 @@ function removeRoutingProfileReferences(
if (!removed) return undefined;
const updatedConfig = { ...config, provider_profiles: providerProfiles };
const roleTiers = config.role_tiers
? Object.fromEntries(
Object.entries(config.role_tiers).filter(
([, tier]) =>
tier === "" ||
tierMappedOnAnyProvider(tier, updatedConfig.provider_order, providerProfiles),
),
)
: undefined;
return {
...updatedConfig,
...(roleTiers ? { role_tiers: roleTiers } : {}),
enabled: config.enabled && routingConfigCanStayEnabled(updatedConfig),
};
}
@@ -3901,6 +3912,17 @@ function routingConfigCanStayEnabled(config: WorkspaceRoutingConfig): boolean {
);
}
function tierMappedOnAnyProvider(
tier: string,
providerOrder: string[],
providerProfiles: Record<string, RoutingProviderProfile>,
): boolean {
return providerOrder.some((providerID) => {
const executionProfileID = providerProfiles[providerID]?.execution_profile_ids?.[tier];
return typeof executionProfileID === "string" && executionProfileID !== "";
});
}
// --- Jira / Linear mock payload types ---
export type MockJiraProject = { id: string; key: string; name: string };
apps/web/e2e/helpers/git-helper.ts
+1 -1
View File
@@ -90,7 +90,7 @@ export async function openTaskSession(page: Page, title: string): Promise<Sessio
const kanban = new KanbanPage(page);
await kanban.goto();
const card = kanban.taskCardByTitle(title);
await expect(card).toBeVisible({ timeout: 15_000 });
await expect(card).toBeVisible({ timeout: 30_000 });
await card.click();
await expect(page).toHaveURL(/\/t\//, { timeout: 15_000 });
const session = new SessionPage(page);
apps/web/e2e/helpers/interim-settings-interlock.ts
+51 -11
View File
@@ -1,16 +1,56 @@
import { dwell } from "./causal-waits";
const appStatePath = "/api/v1/app-state?path=%2Fsettings%2Fagents";
// The backend serves a temporary bootstrap response while it restores the
// database. CI can spend tens of seconds in that window when many shards
// start together, so keep the bounded retry longer than the normal readiness
// probe without waiting forever on a genuinely unhealthy server.
const MAX_STARTUP_RETRIES = 120;
const STARTUP_RETRY_DELAY_MS = 500;
export async function loadInterimSettingsInterlockToken(baseUrl: string): Promise<string> {
const response = await fetch(`${baseUrl}${appStatePath}`);
if (!response.ok) {
throw new Error(`Unable to load E2E settings interlock (${response.status})`);
let lastError: unknown;
for (let attempt = 0; attempt < MAX_STARTUP_RETRIES; attempt++) {
let response: Response;
try {
response = await fetch(`${baseUrl}${appStatePath}`);
} catch (error) {
lastError = error;
if (attempt === MAX_STARTUP_RETRIES - 1) {
throw new Error("Unable to load E2E settings interlock after backend startup retries", {
cause: error,
});
}
await dwell(
STARTUP_RETRY_DELAY_MS,
"poll-interval",
"backend startup before the settings interlock is reachable",
);
continue;
}
if (response.ok) {
const payload = (await response.json()) as { interimSettingsInterlockToken?: unknown };
if (
typeof payload.interimSettingsInterlockToken !== "string" ||
!payload.interimSettingsInterlockToken
) {
throw new Error("E2E settings interlock token missing from boot payload");
}
return payload.interimSettingsInterlockToken;
}
if (response.status !== 503 || attempt === MAX_STARTUP_RETRIES - 1) {
throw new Error(`Unable to load E2E settings interlock (${response.status})`);
}
await dwell(
STARTUP_RETRY_DELAY_MS,
"poll-interval",
"backend startup before the settings interlock is available",
);
}
const payload = (await response.json()) as { interimSettingsInterlockToken?: unknown };
if (
typeof payload.interimSettingsInterlockToken !== "string" ||
!payload.interimSettingsInterlockToken
) {
throw new Error("E2E settings interlock token missing from boot payload");
}
return payload.interimSettingsInterlockToken;
throw new Error("Unable to load E2E settings interlock after backend startup retries", {
cause: lastError,
});
}
apps/web/e2e/helpers/transient-retry.ts
+25
View File
@@ -0,0 +1,25 @@
import type { ApiClient } from "./api-client";
type SessionMessage = Awaited<ReturnType<ApiClient["listSessionMessages"]>>["messages"][number];
export type TransientRetryNotice = {
id: string;
attempt: number;
content: string;
};
export async function listTransientRetryNotices(
apiClient: ApiClient,
sessionId: string,
): Promise<TransientRetryNotice[]> {
const { messages } = await apiClient.listSessionMessages(sessionId);
return messages.filter(isTransientRetryNotice).map((message) => ({
id: message.id,
attempt: typeof message.metadata?.attempt === "number" ? message.metadata.attempt : 0,
content: message.content,
}));
}
function isTransientRetryNotice(message: SessionMessage): boolean {
return message.type === "status" && message.metadata?.retrying === true;
}
apps/web/e2e/pages/file-tree-page.ts
+45 -1
View File
@@ -1,4 +1,4 @@
import type { Locator, Page } from "@playwright/test";
import { expect, type Locator, type Page } from "@playwright/test";
/** Focused page object for Files tree and composer context selectors. */
export class FileTreePage {
@@ -15,6 +15,50 @@ export class FileTreePage {
);
}
/**
* Wait for a tree row and reveal it when the virtualizer has mounted it
* below the current viewport. The initial direct check keeps common rows
* instant, while the bounded sweep handles large workspaces deterministically.
*/
async waitForFileTreeNode(nodePath: string, timeout = 30_000): Promise<Locator> {
const node = this.fileTreeNode(nodePath);
const directTimeout = Math.min(2_000, timeout);
if (await node.isVisible({ timeout: directTimeout }).catch(() => false)) return node;
const viewport = this.fileTreeScrollViewport();
await viewport.waitFor({ state: "visible", timeout: Math.max(1, timeout) });
await viewport.evaluate((element) => {
element.scrollTop = 0;
element.dispatchEvent(new Event("scroll", { bubbles: true }));
});
await expect
.poll(
async () => {
if (await node.isVisible().catch(() => false)) return true;
await viewport
.evaluate((element) => {
const maxScrollTop = Math.max(0, element.scrollHeight - element.clientHeight);
const increment = Math.max(element.clientHeight * 0.8, 160);
const nextScrollTop = Math.min(maxScrollTop, element.scrollTop + increment);
if (nextScrollTop <= element.scrollTop + 1) return false;
element.scrollTop = nextScrollTop;
element.dispatchEvent(new Event("scroll", { bubbles: true }));
return true;
})
.catch(() => false);
return false;
},
{
timeout: Math.max(1, timeout - directTimeout),
intervals: [100, 250, 500],
message: `Waiting for file tree node ${nodePath} to mount`,
},
)
.toBe(true);
return node;
}
/** The existing Files viewport that owns tree scrolling. */
fileTreeScrollViewport(): Locator {
return this.page.locator('[data-testid="file-tree-scroll"]:visible');
apps/web/e2e/pages/session-page.ts
+11 -4
View File
@@ -1430,10 +1430,17 @@ export class SessionPage {
return;
}
await compactStepper.click();
const moveButton = this.page.getByTestId(`workflow-step-disclosure-move-${step.id}`);
await expect(moveButton).toBeVisible();
await moveButton.click();
await expect(async () => {
const moveButton = this.page.getByTestId(`workflow-step-disclosure-move-${step.id}`);
if (!(await moveButton.isVisible())) {
await compactStepper.click();
}
await expect(moveButton).toBeVisible({ timeout: 3_000 });
await moveButton.click({ timeout: 3_000 });
}).toPass({
timeout: 15_000,
intervals: [100, 250, 500],
});
}
/**
apps/web/e2e/tests/chat/agent-message-attribution.spec.ts
+9 -2
View File
@@ -183,6 +183,7 @@ test.describe("Cross-task agent message attribution", () => {
await expect(panel.getByTestId("sender-task-badge")).toHaveCount(10);
await expect(panel.getByTestId("queue-entry-edit")).toHaveCount(0);
await expect(panel.getByTestId("queue-entry-remove")).toHaveCount(10);
const queueIdentity = await apiClient.getQueueSessionIdentity(target.id, target.sessionId);
const firstAgentEntry = panel.getByTestId("queue-entry").filter({
has: testPage.getByTestId("queue-entry-text").filter({ hasText: /^agent queued 1$/ }),
@@ -192,8 +193,14 @@ test.describe("Cross-task agent message attribution", () => {
await expect(panel).toContainText("9 of 10");
await panel.getByTestId("queue-clear-all").click();
await expect(panel).not.toBeVisible({ timeout: 10_000 });
await expect(chat.getByTestId("queue-chip")).not.toBeVisible();
await expect(panel).not.toBeVisible({ timeout: 30_000 });
await expect(chat.getByTestId("queue-chip")).not.toBeVisible({ timeout: 30_000 });
await expect
.poll(() => apiClient.getQueueStatus(queueIdentity).then((status) => status.count), {
timeout: 30_000,
message: "Waiting for clear-all to commit before admitting a new message",
})
.toBe(0);
await createSenderTaskingTarget(
apiClient,
apps/web/e2e/tests/chat/initial-task-brief-helpers.ts
+5 -2
View File
@@ -35,7 +35,10 @@ export async function runInitialTaskBriefFlow({
const session = new SessionPage(testPage);
await testPage.goto(`/t/${task.id}`);
await session.waitForLoad();
await session.waitForChatIdle({ timeout: 45_000 });
// Prepared sessions can spend longer in environment setup than ordinary
// tasks when CI shards are starting together. Keep the wait bounded, but
// allow the session's own hydration/reload recovery to finish.
await session.waitForChatIdle({ timeout: 90_000 });
await session.sendMessageViaButton(INITIAL_TASK_INSTRUCTION);
const chat = session.activeChat();
@@ -61,7 +64,7 @@ export async function runInitialTaskBriefFlow({
await expect(userBubbles.first()).toContainText(INITIAL_TASK_BRIEF);
await expect(userBubbles.first()).toContainText(INITIAL_TASK_INSTRUCTION);
await session.waitForChatIdle({ timeout: 45_000 });
await session.waitForChatIdle({ timeout: 60_000 });
await session.sendMessageViaButton(FOLLOWUP_INSTRUCTION);
await expect(userBubbles).toHaveCount(2, { timeout: 15_000 });
await expect(userBubbles.nth(1)).toContainText(FOLLOWUP_INSTRUCTION);
apps/web/e2e/tests/chat/mobile-message-queue-management.spec.ts
+5 -4
View File
@@ -40,8 +40,9 @@ async function expectTouchTarget(locator: Locator): Promise<void> {
await expect(locator).toBeVisible();
const box = await locator.boundingBox();
expect(box).not.toBeNull();
expect(box!.width).toBeGreaterThanOrEqual(44);
expect(box!.height).toBeGreaterThanOrEqual(44);
// CSS layout can report 43.999... for a 44px device-pixel target.
expect(Math.round(box!.width)).toBeGreaterThanOrEqual(44);
expect(Math.round(box!.height)).toBeGreaterThanOrEqual(44);
}
async function expectEffectiveTouchTarget(locator: Locator): Promise<void> {
@@ -55,8 +56,8 @@ async function expectEffectiveTouchTarget(locator: Locator): Promise<void> {
height: rect.height - px(after.top) - px(after.bottom),
};
});
expect(size.width).toBeGreaterThanOrEqual(44);
expect(size.height).toBeGreaterThanOrEqual(44);
expect(Math.round(size.width)).toBeGreaterThanOrEqual(44);
expect(Math.round(size.height)).toBeGreaterThanOrEqual(44);
}
function scriptedQueueMessage(marker: string, delayMs = 250): string {
apps/web/e2e/tests/chat/mobile-reset-context-confirmation.spec.ts
+5 -1
View File
@@ -4,6 +4,7 @@ import {
seedResetContextSession,
seedStaleContextWindow,
} from "./reset-context-confirmation-helpers";
import { waitForFiniteAnimations } from "../../helpers/animations";
test.describe.configure({ timeout: 120_000 });
@@ -37,6 +38,8 @@ test("mobile reset context opens a sheet and preserves composer controls", async
await expect(testPage.getByRole("dialog")).toHaveAttribute("data-slot", "drawer-content");
await expect(session.resetContextButton()).toBeAttached();
await expect(testPage.getByTestId("submit-message-button")).toBeAttached();
await waitForFiniteAnimations(testPage.getByRole("dialog"));
await warning.scrollIntoViewIfNeeded();
await prCapture.screenshot("mobile-reset-context-confirmation", {
caption: "Mobile context reset preserves the composer behind its compact sheet",
});
@@ -53,8 +56,9 @@ test("mobile reset context opens a sheet and preserves composer controls", async
expect(warningBox!.x + warningBox!.width).toBeLessThanOrEqual(viewportWidth);
const warningIsTopmost = await warning.evaluate((element) => {
const rect = element.getBoundingClientRect();
const y = rect.top + Math.min(8, Math.max(1, rect.height / 2));
return [0.25, 0.5, 0.75].every((ratio) => {
const hit = document.elementFromPoint(rect.left + rect.width * ratio, rect.top + 8);
const hit = document.elementFromPoint(rect.left + rect.width * ratio, y);
return hit === element || element.contains(hit);
});
});
apps/web/e2e/tests/chat/permission-approval.spec.ts
+4 -4
View File
@@ -77,14 +77,14 @@ test.describe("Permission approval persistence", () => {
// After the agent finishes its turn, no permission action row should be
// visible — the previous bug had them re-appear at turn-complete because a
// safety-net loop overwrote the approved status with "complete".
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await session.waitForChatIdle({ timeout: 60_000 });
await expect(session.permissionActionRows()).toHaveCount(0);
// And the resolved state must survive a page reload — i.e. backend must
// have persisted the approve decisions, not "complete".
await testPage.reload();
await session.waitForLoad();
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await session.waitForChatIdle({ timeout: 60_000 });
await expect(session.permissionActionRows()).toHaveCount(0);
});
@@ -118,7 +118,7 @@ test.describe("Permission approval persistence", () => {
await expect(session.permissionApproveButtons()).toHaveCount(1, { timeout: 30_000 });
await session.permissionApproveButtons().first().click();
}
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await session.waitForChatIdle({ timeout: 60_000 });
await expect(sidebarItem.getByTestId("task-state-pending-permission")).toHaveCount(0);
});
@@ -153,6 +153,6 @@ test.describe("Permission approval persistence", () => {
await expect(approveButton).toHaveCount(1, { timeout: 30_000 });
await approveButton.click();
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await session.waitForChatIdle({ timeout: 60_000 });
});
});
apps/web/e2e/tests/chat/quick-chat-helpers.ts
+6
View File
@@ -153,6 +153,12 @@ export async function startQuickChatFromSetup(dialog: Locator, page: Page) {
export async function openQuickChatWithAgent(page: Page, navigateHome = true): Promise<Locator> {
const dialog = await openQuickChatSetup(page, navigateHome);
await startQuickChatFromSetup(dialog, page);
// The composer becomes usable before the session model catalog is hydrated.
// Wait for the model control as well so callers can immediately inspect or
// change the session configuration without racing that second readiness gate.
await expect(dialog.getByRole("button", { name: "Session model settings" })).toBeVisible({
timeout: 30_000,
});
return dialog;
}
apps/web/e2e/tests/chat/quick-chat.spec.ts
+1 -1
View File
@@ -521,7 +521,7 @@ test.describe("Quick Chat", () => {
await leftHandle.hover();
const leftHighlightBox = await leftHandle.locator("span").boundingBox();
expect(leftHighlightBox).not.toBeNull();
expect(leftHighlightBox!.x).toBeCloseTo(rightResizedBox!.x, 0);
expect(Math.abs(leftHighlightBox!.x - rightResizedBox!.x)).toBeLessThanOrEqual(1);
await testPage.mouse.move(leftBox!.x + leftBox!.width / 2, leftBox!.y + leftBox!.height / 2);
await testPage.mouse.down();
await testPage.mouse.move(
apps/web/e2e/tests/chat/rich-output.spec.ts
+4 -1
View File
@@ -6,6 +6,7 @@ import {
RICH_OUTPUT_FILE_CONTENT,
seedRichOutputTask,
} from "./rich-output-helpers";
import { waitForFiniteAnimations } from "../../helpers/animations";
test("renders and persists native rich output with an explicit file preview", async ({
testPage,
@@ -33,7 +34,9 @@ test("renders and persists native rich output with an explicit file preview", as
await expect(lineChart.locator(".recharts-yAxis text").first()).toBeVisible();
await expect(lineChart.locator(".recharts-xAxis")).toContainText("Aug 12");
await barChart.scrollIntoViewIfNeeded();
await expect(barChart.locator(".recharts-xAxis text").first()).toBeVisible();
await waitForFiniteAnimations(barChart);
await expect(barChart.locator("svg")).toBeVisible({ timeout: 30_000 });
await expect(barChart.locator(".recharts-xAxis text").first()).toBeVisible({ timeout: 30_000 });
await expect(barChart.locator(".recharts-yAxis text").first()).toBeVisible();
await expect(barChart.locator(".recharts-xAxis")).toContainText("/api");
apps/web/e2e/tests/cli-mode/mobile-passthrough-composer.spec.ts
+3 -1
View File
@@ -257,7 +257,9 @@ test.describe("mobile CLI mode: passthrough composer", () => {
timeout: 5_000,
});
await testPage.getByTestId("submit-message-button").tap();
const submitButton = composer.getByTestId("submit-message-button");
await expect(submitButton).toBeEnabled({ timeout: 10_000 });
await submitButton.tap();
await expect(composer).toBeHidden({ timeout: 10_000 });
await session.expectPassthroughHasText("mobile context e2e", 15_000);
apps/web/e2e/tests/git/git-changes-panel.spec.ts
+4 -3
View File
@@ -2876,9 +2876,10 @@ test.describe("Git Changes Panel", () => {
await expect(reviewDialog.getByTestId("vcs-primary-push")).toBeVisible({ timeout: 15_000 });
await reviewDialog.getByRole("button", { name: "Open VCS options" }).click();
const openMenu = testPage.locator('[data-slot="dropdown-menu-content"][data-state="open"]');
await expect(
openMenu.locator('[data-slot="dropdown-menu-sub-trigger"]').filter({ hasText: /^Push/ }),
).not.toHaveAttribute("aria-disabled", "true");
const pushAction = openMenu
.locator('[data-slot="dropdown-menu-sub-trigger"]')
.filter({ hasText: /^Push/ });
await expect(pushAction).not.toHaveAttribute("aria-disabled", "true", { timeout: 15_000 });
expect(git.getCurrentSha()).toBe(localHead);
expect(git.exec("git status --porcelain").trim()).toBe("");
});
apps/web/e2e/tests/git/pr-file-row-containment-helpers.ts
+14 -6
View File
@@ -1,4 +1,5 @@
import { expect, type Locator, type Page } from "@playwright/test";
import { waitForFiniteAnimations } from "../../helpers/animations";
import type { SeedData } from "../../fixtures/test-base";
import type { ApiClient } from "../../helpers/api-client";
@@ -99,6 +100,8 @@ export async function expectLongPRRowContained(row: Locator) {
await expect(additions).toBeVisible();
await expect(deletions).toBeVisible();
await expect(status).toBeVisible();
await row.scrollIntoViewIfNeeded();
await waitForFiniteAnimations(row);
await expect
.poll(async () => {
@@ -119,12 +122,17 @@ export async function expectLongPRRowContained(row: Locator) {
})
.toBe(true);
const statusCenterHitsStatus = await status.evaluate((element) => {
const box = element.getBoundingClientRect();
const hit = document.elementFromPoint(box.x + box.width / 2, box.y + box.height / 2);
return hit === element || element.contains(hit);
});
expect(statusCenterHitsStatus).toBe(true);
await expect
.poll(
() =>
status.evaluate((element) => {
const box = element.getBoundingClientRect();
const hit = document.elementFromPoint(box.x + box.width / 2, box.y + box.height / 2);
return hit === element || element.contains(hit);
}),
{ timeout: 15_000 },
)
.toBe(true);
await expect(row.locator(`button[title="${LONG_PR_PATH}"]`)).toHaveAttribute(
"title",
apps/web/e2e/tests/git/symlink-file.spec.ts
+36 -7
View File
@@ -14,6 +14,7 @@ async function seedSimpleTask(
apiClient: ApiClient,
seedData: SeedData,
title: string,
requiredPaths: string[] = [],
): Promise<{ session: SessionPage; sessionId: string }> {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
@@ -27,6 +28,36 @@ async function seedSimpleTask(
},
);
// Environment readiness and repository materialization are separate
// transitions. Wait for the exact fixture paths before the first Files
// request, otherwise a valid early tree snapshot can be retained while the
// checkout is still being populated.
if (requiredPaths.length > 0) {
let workspacePath = "";
await expect
.poll(async () => (await apiClient.getTaskEnvironment(task.id))?.status ?? null, {
timeout: 30_000,
message: `Waiting for ${title} task environment to be ready`,
})
.toBe("ready");
await expect
.poll(
async () => {
const environment = await apiClient.getTaskEnvironment(task.id);
workspacePath =
environment?.workspace_path ?? environment?.repos?.[0]?.worktree_path ?? "";
return requiredPaths.every((requiredPath) =>
Boolean(workspacePath && fs.existsSync(path.join(workspacePath, requiredPath))),
);
},
{
timeout: 60_000,
message: `Waiting for ${requiredPaths.join(", ")} in the ${title} worktree`,
},
)
.toBe(true);
}
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
@@ -92,6 +123,7 @@ test.describe("Symlink file handling", () => {
apiClient,
seedData,
"Symlink Dir Tree Test",
["real-dir/child.txt", "link-dir/child.txt"],
);
// Open Files tab
@@ -99,18 +131,15 @@ test.describe("Symlink file handling", () => {
await expect(session.files).toBeVisible({ timeout: 5_000 });
// The symlink-to-directory should appear in the tree
const linkDirRow = session.files.getByText("link-dir");
await expect(linkDirRow).toBeVisible({ timeout: 10_000 });
await expect(session.fileTreeNode("link-dir").getByTestId("symlink-indicator")).toBeVisible();
const linkDirNode = await session.fileTree.waitForFileTreeNode("link-dir");
await expect(linkDirNode.getByTestId("symlink-indicator")).toBeVisible();
// Click to expand — if the fix is missing, this is classified as a file
// and would try to open it in the editor instead of expanding.
await linkDirRow.click();
await linkDirNode.click();
// Assert the child file inside the symlinked directory is now visible
const childFile = session.files.getByText("child.txt");
await expect(childFile).toBeVisible({ timeout: 10_000 });
await session.fileTree.waitForFileTreeNode("link-dir/child.txt");
});
for (const provider of ["monaco", "codemirror"] as const) {
apps/web/e2e/tests/kanban/large-column-virtualization-helpers.ts
+7 -2
View File
@@ -38,7 +38,12 @@ export async function mountedTaskCardIds(column: Locator): Promise<string[]> {
}
export async function expectBoundedMountedCards(column: Locator): Promise<void> {
await expect.poll(() => taskCards(column).count()).toBeLessThan(MAX_MOUNTED_TASK_CARDS);
await expect
.poll(() => taskCards(column).count(), {
timeout: 30_000,
message: "Waiting for virtualized column card count to settle",
})
.toBeLessThan(MAX_MOUNTED_TASK_CARDS);
}
export async function scrollColumnToBottom(scrollOwner: Locator): Promise<void> {
@@ -57,7 +62,7 @@ export async function scrollColumnToBottom(scrollOwner: Locator): Promise<void>
return stableBottomSamples >= 2;
},
{
timeout: 10_000,
timeout: 30_000,
intervals: [50, 100, 250],
message: "virtualized column did not settle at the bottom",
},
apps/web/e2e/tests/kanban/large-column-virtualization.spec.ts
+16 -6
View File
@@ -26,8 +26,15 @@ test("virtualizes a large desktop column while preserving reached-card navigatio
const column = kanban.columnByStepId(seedData.startStepId);
await expect(column).toBeVisible();
await expect(column.getByText(String(LARGE_COLUMN_TASK_COUNT), { exact: true })).toBeVisible();
await expect.poll(() => taskCards(column).count()).toBeGreaterThan(0);
await expect(column.getByText(String(LARGE_COLUMN_TASK_COUNT), { exact: true })).toBeVisible({
timeout: 30_000,
});
await expect
.poll(() => taskCards(column).count(), {
timeout: 30_000,
message: "Waiting for large-column cards to mount",
})
.toBeGreaterThan(0);
const initialCardIds = await mountedTaskCardIds(column);
await expectBoundedMountedCards(column);
@@ -37,10 +44,13 @@ test("virtualizes a large desktop column while preserving reached-card navigatio
await scrollColumnToBottom(scrollOwner);
await expect
.poll(async () => {
const mountedIds = await mountedTaskCardIds(column);
return mountedIds.some((id) => !initialCardIds.includes(id));
})
.poll(
async () => {
const mountedIds = await mountedTaskCardIds(column);
return mountedIds.some((id) => !initialCardIds.includes(id));
},
{ timeout: 30_000, message: "Waiting for a new large-column card after scrolling" },
)
.toBe(true);
await expectBoundedMountedCards(column);
apps/web/e2e/tests/kanban/swimlane-height.spec.ts
+1 -1
View File
@@ -36,7 +36,7 @@ test("two sparse workflows fit without scrolling the board", async ({
await apiClient.saveUserSettings({ workflow_filter_id: "", repository_ids: [] });
const kanban = new KanbanPage(testPage);
await kanban.goto();
await expect(kanban.taskCard(second.id)).toBeVisible();
await expect(kanban.taskCard(second.id)).toBeVisible({ timeout: 30_000 });
const board = testPage.getByTestId("swimlane-container");
await testInfo.attach("sparse-lane-geometry", {
body: JSON.stringify({
apps/web/e2e/tests/kanban/tablet-large-column-virtualization.spec.ts
+3 -1
View File
@@ -26,7 +26,9 @@ test("virtualizes a large tablet column in the snap-scrolling layout", async ({
const column = layout.getByTestId(`kanban-column-${seedData.startStepId}`);
await expect(column).toBeVisible();
await expect(column.getByText(String(LARGE_COLUMN_TASK_COUNT), { exact: true })).toBeVisible();
await expect(column.getByText(String(LARGE_COLUMN_TASK_COUNT), { exact: true })).toBeVisible({
timeout: 30_000,
});
await expect.poll(() => taskCards(column).count()).toBeGreaterThan(0);
const initialCardIds = await mountedTaskCardIds(column);
apps/web/e2e/tests/kanban/wip-overflow-queue.spec.ts
+1 -1
View File
@@ -36,7 +36,7 @@ test("dragging into a feeder wakes an open pull target without reload", async ({
const sourceCard = kanban.taskCard(task.id);
const feederColumn = kanban.columnByStepId(feederStep.id);
await sourceCard.waitFor({ state: "visible" });
await expect(sourceCard).toBeVisible({ timeout: 30_000 });
const sourceBox = await sourceCard.boundingBox();
expect(sourceBox).not.toBeNull();
apps/web/e2e/tests/kanban/workflow-filter.spec.ts
+6 -3
View File
@@ -3,7 +3,7 @@ import { KanbanPage } from "../../pages/kanban-page";
import type { Page } from "@playwright/test";
import { expandDisplaySettingsGroup } from "../../helpers/display-settings";
const TASK_VISIBLE_TIMEOUT = 10_000;
const TASK_VISIBLE_TIMEOUT = 30_000;
const ALPHA_TASK = "Alpha task";
const BETA_TASK = "Beta task";
@@ -45,8 +45,7 @@ test.describe("Kanban workflow filter", () => {
// Pull `testPage` so its fixture (which runs `e2eReset` and resets user
// settings) is set up before this hook seeds workflows/tasks — otherwise
// the reset wipes the seed data the moment a test first reads testPage.
// eslint-disable-next-line @typescript-eslint/no-unused-vars
test.beforeEach(async ({ apiClient, seedData, testPage }) => {
test.beforeEach(async ({ apiClient, seedData, testPage: _testPage }) => {
const workflowB = await apiClient.createWorkflow(seedData.workspaceId, "Workflow B", "simple");
workflowBId = workflowB.id;
const stepsB = (await apiClient.listWorkflowSteps(workflowB.id)).steps;
@@ -61,6 +60,10 @@ test.describe("Kanban workflow filter", () => {
workflow_step_id: startB.id,
});
betaTaskId = beta.id;
await apiClient.saveUserSettings({
workspace_id: seedData.workspaceId,
workflow_filter_id: seedData.workflowId,
});
});
test.afterEach(async ({ apiClient, seedData }) => {
apps/web/e2e/tests/layout/compact-desktop-responsive.spec.ts
+1 -1
View File
@@ -112,7 +112,7 @@ test.describe("compact desktop responsive layout", () => {
await expect(kanban.viewTogglePipeline).toBeVisible();
for (const step of seedData.steps) {
await expect(kanban.columnByStepId(step.id)).toBeAttached();
await expect(kanban.columnByStepId(step.id)).toBeAttached({ timeout: 15_000 });
}
const firstColumnBox = await kanban.columnByStepId(seedData.steps[0].id).boundingBox();
expect(firstColumnBox).not.toBeNull();
apps/web/e2e/tests/layout/right-panel-visibility.spec.ts
+5 -14
View File
@@ -2,7 +2,7 @@ import { expect, type Page } from "@playwright/test";
import { test, type SeedData } from "../../fixtures/test-base";
import type { ApiClient } from "../../helpers/api-client";
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
import { getDockviewGroupWidth } from "../../helpers/dockview-resize";
import { getDockviewGroupWidth, resizeColumnViaSplitview } from "../../helpers/dockview-resize";
import { SessionPage } from "../../pages/session-page";
async function createTask(apiClient: ApiClient, seedData: SeedData, title: string) {
@@ -349,19 +349,10 @@ for (const layout of ["default", "plan", "preview"] as const) {
}
const panelId = { default: "files", preview: "browser", plan: "plan" }[layout]!;
const before = await getDockviewGroupWidth(testPage, panelId);
const sashBox = await testPage.evaluate(() => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const api = (window as any).__dockviewApi__;
const sashes = api.component.gridview.root.splitview.sashes;
const rect = sashes.at(-1).container.getBoundingClientRect();
return { x: rect.x + rect.width / 2, y: rect.y + 100 };
});
await testPage.mouse.move(sashBox.x, sashBox.y);
await testPage.mouse.down();
await testPage.mouse.move(sashBox.x - 100, sashBox.y, { steps: 10 });
await testPage.mouse.up();
await expect.poll(() => getDockviewGroupWidth(testPage, panelId)).toBeGreaterThan(before + 50);
const resized = await getDockviewGroupWidth(testPage, panelId);
// Use the same dockview resize path as the other pane specs. A real mouse
// drag can miss the sash under CI load even when the layout is ready.
const resized = await resizeColumnViaSplitview(testPage, "right", before + 100);
expect(resized).toBeGreaterThan(before + 50);
const toggle = testPage.getByTestId("task-right-panels-toggle");
for (let cycle = 0; cycle < 3; cycle++) {
await toggle.click();
apps/web/e2e/tests/layout/sidebar-hover-reveal.spec.ts
+10 -2
View File
@@ -1,4 +1,10 @@
import { test, expect } from "../../fixtures/test-base";
import type { Locator } from "@playwright/test";
async function waitForSidebarReveal(sidebar: Locator) {
await expect(sidebar).toHaveAttribute("data-hover-revealed", "true", { timeout: 15_000 });
await expect(sidebar).toHaveCSS("width", "320px", { timeout: 15_000 });
}
// @covers AC-UI-SIDEBAR-HOVER-001.1, AC-UI-SIDEBAR-HOVER-001.2
test("collapsed sidebar reveals after dwell without moving page content", async ({
@@ -14,7 +20,7 @@ test("collapsed sidebar reveals after dwell without moving page content", async
.locator(":scope > div > main")
.boundingBox();
await sidebar.hover({ position: { x: 20, y: 200 } });
await expect(sidebar).toHaveCSS("width", "320px");
await waitForSidebarReveal(sidebar);
await expect(testPage.getByTestId("app-sidebar-layout")).toHaveCSS("width", "56px");
await expect(sidebar).toHaveAttribute("data-collapsed", "true");
const pageAfter = await testPage
@@ -37,6 +43,7 @@ test("reveal supports the workspace menu, Escape, and permanent expansion", asyn
await testPage.mouse.move(600, 400);
await expect(sidebar).toHaveCSS("width", "56px");
await sidebar.hover({ position: { x: 20, y: 200 } });
await waitForSidebarReveal(sidebar);
await expect(sidebar.getByTestId("sidebar-workspace-trigger")).toBeVisible();
await sidebar.getByTestId("sidebar-workspace-trigger").click();
const menu = testPage.getByRole("menu");
@@ -80,7 +87,7 @@ test("short hover cancels and crossing the phone breakpoint closes the reveal",
await testPage.clock.resume();
await testPage.setViewportSize({ width: 768, height: 900 });
await sidebar.hover({ position: { x: 20, y: 200 } });
await expect(sidebar).toHaveCSS("width", "320px");
await waitForSidebarReveal(sidebar);
await testPage.setViewportSize({ width: 767, height: 900 });
await expect(sidebar).toBeHidden();
await testPage.mouse.move(600, 400);
@@ -105,6 +112,7 @@ test("revealed tasks support context menus and navigation", async ({
await testPage.mouse.move(600, 400);
await expect(sidebar).toHaveCSS("width", "56px");
await sidebar.hover({ position: { x: 20, y: 200 } });
await waitForSidebarReveal(sidebar);
const row = sidebar.getByText("Hover navigation task", { exact: true });
await expect(row).toBeVisible();
await row.click({ button: "right" });
apps/web/e2e/tests/layout/toggle-sidebar-shortcut.spec.ts
+2 -1
View File
@@ -168,10 +168,11 @@ function expectStableAnimatedToggle(capture: SidebarToggleCapture): void {
const panelWidths = frames.map((frame) => frame.panelWidth);
const minimumPanelWidth = Math.min(...panelWidths);
const maximumPanelWidth = Math.max(...panelWidths);
// Browser layout can report the 200px design delta one subpixel below 200.
expect(
maximumPanelWidth - minimumPanelWidth,
"Visual sidebar should travel between its expanded and collapsed widths",
).toBeGreaterThan(200);
).toBeGreaterThanOrEqual(199);
expect(
panelWidths
.slice(1, -1)
apps/web/e2e/tests/lsp/lsp-e2e-helpers.ts
+41 -8
View File
@@ -276,32 +276,65 @@ export async function openDesktopFile(
// Git status updates can auto-activate the Changes panel just after the
// file tree renders. Re-select Files and click only while the row is still
// visible so a late panel switch cannot turn a successful visibility check
// into a 180-second click timeout. Executor startup toasts can cover the
// dockview tab, so force that tab activation and keep the file-row click
// user-facing and actionability checked.
// into a 180-second click timeout. Virtualized trees can also omit a valid
// file row from the DOM, so use the exact file search in that case.
await session.clickTab("Files", { force: true });
await expect(session.files).toBeVisible({ timeout: 30_000 });
const existingSearch = session.fileSearchInput();
if (await existingSearch.isVisible()) {
await existingSearch.press("Escape");
}
let openMode: "tree" | "search" | false = false;
await expect
.poll(
async () => {
try {
await session.clickTab("Files", { force: true });
if (!(await session.files.isVisible())) return false;
if (await session.fileSearchInput().isVisible()) {
await session.fileSearchInput().press("Escape");
return false;
}
for (let index = 1; index < pathSegments.length; index++) {
const ancestor = session.fileTreeNode(pathSegments.slice(0, index).join("/"));
if (!(await ancestor.isVisible())) return false;
if (!(await ancestor.isVisible())) {
openMode = (await session.fileSearchButton().isVisible()) ? "search" : false;
return openMode;
}
if ((await ancestor.locator(".tabler-icon-chevron-right").count()) > 0) {
await ancestor.click();
}
}
if (!(await fileNode.isVisible())) return false;
await fileNode.click({ timeout: 2_000 });
return true;
if (await fileNode.isVisible()) {
await fileNode.click({ timeout: 2_000 });
openMode = "tree";
return openMode;
}
openMode = (await session.fileSearchButton().isVisible()) ? "search" : false;
return openMode;
} catch {
openMode = false;
return false;
}
},
{ timeout: 30_000, intervals: [500, 1000, 2000] },
)
.toBe(true);
.toBeTruthy();
if (openMode === "search") {
await session.fileSearchButton().click();
const searchInput = session.fileSearchInput();
await expect(searchInput).toBeVisible({ timeout: 5_000 });
// Search matches the repository-relative path, while the task tree path
// may include a repository prefix. Query the exact filename so paths with
// spaces or punctuation remain searchable, then select by the exact path.
await searchInput.fill(path.posix.basename(filePath));
const searchResult = session.fileSearchResult(filePath);
await expect(searchResult).toBeVisible({ timeout: 15_000 });
await searchResult.click();
await searchInput.press("Escape");
}
await expect(page.locator(".dv-default-tab", { hasText: path.basename(filePath) })).toBeVisible({
timeout: 10_000,
});
apps/web/e2e/tests/lsp/lsp-file-intelligence.spec.ts
+9 -1
View File
@@ -1094,10 +1094,18 @@ test.describe("LSP file intelligence", () => {
backend,
}) => {
installFakeKotlinLsp(backend);
const archiveTarget = "Main-archive-cleanup.kt";
const task = await createKotlinTask(testPage, apiClient, seedData, backend, {
title: "Kotlin LSP Archive Cleanup",
// Keep the target outside the initial virtualized viewport. This proves
// the file-opening helper uses the exact search path when a valid row is
// not mounted in the tree.
filePaths: [
archiveTarget,
...Array.from({ length: 48 }, (_, index) => `A-archive-noise-${index}.kt`),
],
});
await openDesktopFile(testPage, task.session, task.filePaths[0]);
await openDesktopFile(testPage, task.session, archiveTarget);
const statusButton = testPage.locator('[data-testid="lsp-status-button"]:visible');
await performLspAction(testPage, "start");
await expect(statusButton).toHaveAttribute("data-lsp-state", "ready", { timeout: 15_000 });
apps/web/e2e/tests/office/agent-launch-context.spec.ts
+3 -3
View File
@@ -33,7 +33,7 @@ test.describe("Office agent launch context", () => {
officeApi,
officeSeed,
}) => {
test.setTimeout(60_000);
test.setTimeout(120_000);
// 1. Prime the bundled system-skill sync for the Office workspace
// (the lazy sync runs on the first /skills list).
@@ -74,7 +74,7 @@ test.describe("Office agent launch context", () => {
worktreePath = env?.workspace_path ?? env?.repos?.[0]?.worktree_path ?? "";
return worktreePath;
},
{ timeout: 30_000, message: "task environment workspace_path never appeared" },
{ timeout: 60_000, message: "task environment workspace_path never appeared" },
)
.not.toBe("");
@@ -84,7 +84,7 @@ test.describe("Office agent launch context", () => {
const skillFile = path.join(worktreePath, ".agents", "skills", "kandev-protocol", "SKILL.md");
await expect
.poll(() => fs.existsSync(skillFile), {
timeout: 15_000,
timeout: 30_000,
message: skillFile,
})
.toBe(true);
apps/web/e2e/tests/office/mobile-office-reparent.spec.ts
+1 -1
View File
@@ -24,7 +24,7 @@ test.describe("Office manager reassignment on mobile", () => {
await reportsTo.click();
const listbox = testPage.getByRole("listbox");
await expect(listbox).toBeVisible();
await listbox.getByRole("option", { name: "CEO" }).click();
await listbox.getByRole("option", { name: "CEO", exact: true }).click();
const saved = waitForHttp(testPage, "PATCH", new RegExp(`/api/v1/office/agents/${workerId}$`));
await testPage.getByRole("button", { name: "Save Configuration" }).click();
apps/web/e2e/tests/office/org-chart.spec.ts
+1 -1
View File
@@ -33,7 +33,7 @@ test.describe("Org chart", () => {
await testPage.getByRole("combobox", { name: "Reports to" }).click();
const listbox = testPage.getByRole("listbox");
await expect(listbox).toBeVisible();
await listbox.getByRole("option", { name: "CEO" }).click();
await listbox.getByRole("option", { name: "CEO", exact: true }).click();
const saved = waitForHttp(testPage, "PATCH", new RegExp(`/api/v1/office/agents/${workerId}$`));
await testPage.getByRole("button", { name: "Save Configuration" }).click();
apps/web/e2e/tests/office/scheduler-events.spec.ts
+5 -5
View File
@@ -66,7 +66,7 @@ test.describe("Office reactive scheduler", () => {
officeApi,
officeSeed,
}) => {
test.setTimeout(30_000);
test.setTimeout(90_000);
// Create a task without an assignee, then attach the CEO.
const task = await apiClient.createTask(
@@ -85,7 +85,7 @@ test.describe("Office reactive scheduler", () => {
const runs = await listAgentRuns(apiClient, officeSeed.agentId);
return runs.filter((r) => r.reason === "task_assigned" && r.task_id === task.id);
},
{ timeout: 20_000, message: "no task_assigned run surfaced for the new assignee" },
{ timeout: 60_000, message: "no task_assigned run surfaced for the new assignee" },
)
.not.toEqual([]);
});
@@ -95,7 +95,7 @@ test.describe("Office reactive scheduler", () => {
officeApi,
officeSeed,
}) => {
test.setTimeout(30_000);
test.setTimeout(90_000);
const task = await apiClient.createTask(
officeSeed.workspaceId,
@@ -113,7 +113,7 @@ test.describe("Office reactive scheduler", () => {
const runs = await listAgentRuns(apiClient, officeSeed.agentId);
return runs.filter((r) => r.reason === "task_assigned" && r.task_id === task.id).length;
},
{ timeout: 20_000 },
{ timeout: 60_000 },
)
.toBeGreaterThan(0);
@@ -126,7 +126,7 @@ test.describe("Office reactive scheduler", () => {
const runs = await listAgentRuns(apiClient, officeSeed.agentId);
return runs.filter((r) => r.reason === "task_comment" && r.task_id === task.id);
},
{ timeout: 20_000, message: "no task_comment run surfaced for the comment" },
{ timeout: 60_000, message: "no task_comment run surfaced for the comment" },
)
.not.toEqual([]);
});
apps/web/e2e/tests/plugins/mobile-plugin-modal.spec.ts
+25 -1
View File
@@ -17,6 +17,30 @@ test.describe("Mobile plugin modal content", () => {
await installFixturePlugin(testPage);
await testPage.goto("/");
await testPage.reload();
// The route registration and the global shortcut dispatcher are fed by
// separate React updates. Wait for the store's active-plugin snapshot so
// the dispatcher has the manifest keybinding before sending the shortcut.
await testPage.waitForFunction(
(pluginId) => {
const store = (
window as Window & {
__KANDEV_E2E_STORE__?: {
getState: () => {
plugins?: { loaded: boolean; items: Array<{ id: string; status: string }> };
};
};
}
).__KANDEV_E2E_STORE__;
const plugins = store?.getState().plugins;
return Boolean(
plugins?.loaded &&
plugins.items.some((plugin) => plugin.id === pluginId && plugin.status === "active"),
);
},
PLUGIN_ID,
{ timeout: 30_000 },
);
await testPage.goto("/plugins/e2e-hello");
await expect(testPage.locator("#hello-plugin-page")).toBeVisible({ timeout: 15_000 });
await testPage.keyboard.press("ControlOrMeta+Shift+J");
@@ -26,7 +50,7 @@ test.describe("Mobile plugin modal content", () => {
const title = dialog.locator('[data-slot="dialog-title"]');
const close = dialog.locator('[data-slot="dialog-close"]');
const finalAction = dialog.getByTestId("hello-long-modal-final-action");
await expect(dialog).toBeVisible();
await expect(dialog).toBeVisible({ timeout: 15_000 });
await expect(body).toHaveCount(1);
await expect(finalAction).toBeVisible();
await dialog.evaluate(async (element) => {
apps/web/e2e/tests/pr/mobile-pr-ci-chip.spec.ts
+26 -9
View File
@@ -11,6 +11,7 @@ import { test, expect } from "../../fixtures/test-base";
import type { SeedData } from "../../fixtures/test-base";
import { SessionPage } from "../../pages/session-page";
import type { ApiClient } from "../../helpers/api-client";
import { waitForFiniteAnimations } from "../../helpers/animations";
const OWNER = "acme";
const REPO = "demo";
@@ -149,15 +150,31 @@ test.describe("mobile PR CI chip drawer", () => {
const statusBar = session.activeChat().getByTestId("chat-status-bar");
await expect(statusBar).toHaveCSS("flex-wrap", "wrap");
expect(
await statusBar.evaluate((element) => {
const bar = element.getBoundingClientRect();
return Array.from(element.children).every((child) => {
const rect = child.getBoundingClientRect();
return rect.left >= bar.left - 1 && rect.right <= bar.right + 1;
});
}),
).toBe(true);
await expect
.poll(
async () => {
await waitForFiniteAnimations(statusBar);
return statusBar.evaluate((element) => {
const bar = element.getBoundingClientRect();
const visibleChildren = Array.from(element.children).filter((child) => {
const style = getComputedStyle(child);
const rect = child.getBoundingClientRect();
return style.display !== "none" && style.visibility !== "hidden" && rect.width > 0;
});
const childrenFit = visibleChildren.every((child) => {
const rect = child.getBoundingClientRect();
return rect.left >= bar.left - 1 && rect.right <= bar.right + 1;
});
return childrenFit && element.scrollWidth <= element.clientWidth + 1;
});
},
{
timeout: 30_000,
intervals: [100, 250, 500],
message: "Waiting for the mobile chat status bar children to fit within the bar",
},
)
.toBe(true);
expect(
await testPage.evaluate(
() => document.documentElement.scrollWidth <= document.documentElement.clientWidth,
apps/web/e2e/tests/review/code-review-on-demand.spec.ts
+6 -3
View File
@@ -67,7 +67,9 @@ test.describe("Native code review — on demand", () => {
const changesTab = testPage.getByTestId("dockview-tab-changes");
await expect(changesTab).toBeVisible();
await changesTab.click();
await expect(testPage.getByTestId(`file-row-${REVIEWED_FILE}`)).toBeVisible({
await expect(
testPage.getByTestId("unstaged-file-tree").getByTestId(`file-row-${REVIEWED_FILE}`),
).toBeVisible({
timeout: 30_000,
});
@@ -150,7 +152,6 @@ test.describe("Native code review — on demand", () => {
await testPage.getByRole("button", { name: "Expand review" }).click();
const reopened = testPage.getByRole("dialog", { name: "Review Changes" });
await expect(reopened).toBeVisible();
// The dialog renders only findings that can be anchored to the refreshed
// diff. Verify the persisted disposition through the same task-review
// snapshot used by the page backfill so an anchor refresh cannot hide a
@@ -202,7 +203,9 @@ test.describe("Native code review — on demand", () => {
git.createFile(REVIEWED_FILE, "export const unreviewable = 1;\n");
await testPage.getByTestId("dockview-tab-changes").click();
await expect(testPage.getByTestId(`file-row-${REVIEWED_FILE}`)).toBeVisible({
await expect(
testPage.getByTestId("unstaged-file-tree").getByTestId(`file-row-${REVIEWED_FILE}`),
).toBeVisible({
timeout: 30_000,
});
await testPage
apps/web/e2e/tests/review/mobile-review-findings-nav.spec.ts
+3 -1
View File
@@ -61,7 +61,9 @@ test.describe("Review findings navigator on mobile", () => {
await expect(mobileChangesButton).toBeVisible({ timeout: 15_000 });
await mobileChangesButton.tap();
await expect(testPage.getByTestId("mobile-changes-panel")).toBeVisible({ timeout: 15_000 });
await expect(testPage.getByTestId(`file-row-${REVIEWED_FILE}`)).toBeVisible({
await expect(
testPage.getByTestId("unstaged-file-tree").getByTestId(`file-row-${REVIEWED_FILE}`),
).toBeVisible({
timeout: 30_000,
});
await testPage.evaluate(() => window.dispatchEvent(new CustomEvent("open-review-dialog")));
apps/web/e2e/tests/session/mobile-transient-retry.spec.ts
+54 -1
View File
@@ -4,6 +4,8 @@
// card and its Cancel button must render and work on a narrow touch viewport.
import { test, expect } from "../../fixtures/test-base";
import { seedIdleSession } from "../../helpers/session";
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
import { listTransientRetryNotices } from "../../helpers/transient-retry";
test.describe("mobile: transient provider error retry", () => {
test("yellow retry card + Cancel works on mobile and surfaces recovery", async ({
@@ -14,14 +16,65 @@ test.describe("mobile: transient provider error retry", () => {
const session = await seedIdleSession(testPage, apiClient, seedData, "Mobile Overloaded Test");
await session.sendMessageViaButton("/overloaded:9");
const sessionId = await session.activeChat().getAttribute("data-session-id");
if (!sessionId) throw new Error("active chat did not expose a session id");
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt === 1;
},
{ timeout: 30_000 },
)
.toBe(true);
// Yellow retry card + Cancel button render on the narrow viewport.
await expect(session.transientRetryCard()).toBeVisible({ timeout: 30_000 });
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.recoveryCancelRetryButton()).toBeVisible();
await expect(session.recoveryResumeButton()).toBeHidden();
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt >= 2;
},
{ timeout: 30_000 },
)
.toBe(true);
const advanced = await listTransientRetryNotices(apiClient, sessionId);
expect(advanced).toHaveLength(1);
const firstNoticeId = advanced[0].id;
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.transientRetryCard()).toContainText(/attempt [2-5] of 5/i);
await testPage.reload();
await session.waitForLoad();
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt >= 2;
},
{ timeout: 30_000 },
)
.toBe(true);
const afterReload = await listTransientRetryNotices(apiClient, sessionId);
expect(afterReload).toHaveLength(1);
expect(afterReload[0].id).toBe(firstNoticeId);
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.transientRetryCard()).toContainText(/attempt [2-5] of 5/i);
await assertNoDocumentHorizontalOverflow(testPage);
// Tap Cancel → red recovery banner.
await session.recoveryCancelRetryButton().click();
await session.recoveryCancelRetryButton().tap();
await expect
.poll(async () => (await listTransientRetryNotices(apiClient, sessionId)).length, {
timeout: 30_000,
})
.toBe(0);
await expect(session.recoveryResumeButton()).toBeVisible({ timeout: 30_000 });
await expect(session.transientRetryCard()).toBeHidden();
});
apps/web/e2e/tests/session/multi-session.spec.ts
+1 -1
View File
@@ -95,7 +95,7 @@ test.describe("Multi-session", () => {
await kanban.goto();
const card = kanban.taskCardByTitle("Multi Session Task");
await expect(card).toBeVisible({ timeout: 10_000 });
await expect(card).toBeVisible({ timeout: 30_000 });
await card.click();
await expect(testPage).toHaveURL(/\/t\//, { timeout: 15_000 });
apps/web/e2e/tests/session/session-layout.spec.ts
+1 -1
View File
@@ -274,7 +274,7 @@ test.describe("Session tab cleanup", () => {
const kanban = new KanbanPage(testPage);
await kanban.goto();
const card = kanban.taskCardByTitle("Single Session Tab Task");
await expect(card).toBeVisible({ timeout: 10_000 });
await expect(card).toBeVisible({ timeout: 30_000 });
await card.click();
await expect(testPage).toHaveURL(/\/t\//, { timeout: 15_000 });
apps/web/e2e/tests/session/session-resume-cli-fallback.spec.ts
-1
View File
@@ -63,7 +63,6 @@ test.describe("Session resume — CLI fallback after fast-fail", () => {
// CI shard, while the task route can load the same persisted task directly.
await testPage.goto(`/t/${task.id}`);
await expect(testPage).toHaveURL(/\/t\//, { timeout: 15_000 });
const session = new SessionPage(testPage);
await session.waitForPassthroughLoad();
await session.waitForPassthroughLoaded();
apps/web/e2e/tests/session/transient-retry.spec.ts
+68 -1
View File
@@ -1,5 +1,6 @@
import { test, expect } from "../../fixtures/test-base";
import { seedIdleSession } from "../../helpers/session";
import { listTransientRetryNotices } from "../../helpers/transient-retry";
import { SessionPage } from "../../pages/session-page";
test.describe("transient provider error (529 Overloaded) retry", () => {
@@ -12,9 +13,18 @@ test.describe("transient provider error (529 Overloaded) retry", () => {
// /overloaded:9 keeps failing so the retry loop stays visible until cancel.
await session.sendMessage("/overloaded:9");
const sessionId = await session.activeChat().getAttribute("data-session-id");
if (!sessionId) throw new Error("active chat did not expose a session id");
await expect
.poll(async () => (await listTransientRetryNotices(apiClient, sessionId)).length, {
timeout: 30_000,
})
.toBe(1);
// The calm yellow "retrying" card + Cancel button must appear...
await expect(session.transientRetryCard()).toBeVisible({ timeout: 30_000 });
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.recoveryCancelRetryButton()).toBeVisible();
// ...and the red recovery banner must NOT be shown yet (retries in flight).
@@ -29,10 +39,23 @@ test.describe("transient provider error (529 Overloaded) retry", () => {
const session = await seedIdleSession(testPage, apiClient, seedData, "Overloaded Cancel Test");
await session.sendMessage("/overloaded:9");
const sessionId = await session.activeChat().getAttribute("data-session-id");
if (!sessionId) throw new Error("active chat did not expose a session id");
await expect(session.recoveryCancelRetryButton()).toBeVisible({ timeout: 30_000 });
await expect
.poll(async () => (await listTransientRetryNotices(apiClient, sessionId)).length, {
timeout: 30_000,
})
.toBe(1);
await session.recoveryCancelRetryButton().click();
await expect
.poll(async () => (await listTransientRetryNotices(apiClient, sessionId)).length, {
timeout: 30_000,
})
.toBe(0);
// Cancelling falls through to the red Resume / Start-fresh recovery banner.
await expect(session.recoveryResumeButton()).toBeVisible({ timeout: 30_000 });
await expect(session.recoveryFreshButton()).toBeVisible();
@@ -51,9 +74,53 @@ test.describe("transient provider error (529 Overloaded) retry", () => {
// progress from attempt 1 of 5 to attempt 2 of 5 — proving the retry loop is live
// and paced rather than an instant, silent resume.
await session.sendMessage("/overloaded:9");
const sessionId = await session.activeChat().getAttribute("data-session-id");
if (!sessionId) throw new Error("active chat did not expose a session id");
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt === 1;
},
{ timeout: 30_000 },
)
.toBe(true);
await expect(session.chat.getByText(/attempt 1 of 5/i)).toBeVisible({ timeout: 30_000 });
await expect(session.chat.getByText(/attempt 2 of 5/i)).toBeVisible({ timeout: 30_000 });
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt >= 2;
},
{ timeout: 30_000 },
)
.toBe(true);
const advanced = await listTransientRetryNotices(apiClient, sessionId);
expect(advanced).toHaveLength(1);
const firstNoticeId = advanced[0].id;
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.transientRetryCard()).toContainText(/attempt [2-5] of 5/i);
await testPage.reload();
await session.waitForLoad();
await expect
.poll(
async () => {
const notices = await listTransientRetryNotices(apiClient, sessionId);
return notices.length === 1 && notices[0].attempt >= 2;
},
{ timeout: 30_000 },
)
.toBe(true);
const afterReload = await listTransientRetryNotices(apiClient, sessionId);
expect(afterReload).toHaveLength(1);
expect(afterReload[0].id).toBe(firstNoticeId);
await expect(session.transientRetryCard()).toHaveCount(1);
await expect(session.transientRetryCard()).toContainText(/attempt [2-5] of 5/i);
});
test("a 529 on the very first turn retries (launch prompt is cached)", async ({
apps/web/e2e/tests/settings/mobile-workspace-repository-sets.spec.ts
+6 -1
View File
@@ -2,6 +2,7 @@ import { execSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { test, expect } from "../../fixtures/test-base";
import { waitForFiniteAnimations } from "../../helpers/animations";
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
import { makeGitEnv } from "../../helpers/git-helper";
@@ -143,7 +144,11 @@ test.describe("Mobile workspace repository sets", () => {
await search.fill("origin");
await expect(dropdown.getByRole("option", { name: /^origin\/main origin/ })).toBeVisible();
await expect(dropdown.getByRole("option", { name: /^main local/ })).toHaveCount(0);
await dropdown.getByTestId("branch-refresh-button").tap();
const refreshButton = dropdown.getByTestId("branch-refresh-button");
await expect(refreshButton).toBeVisible();
await expect(refreshButton).toBeEnabled();
await waitForFiniteAnimations(dropdown);
await refreshButton.tap({ force: true });
await expect(dropdown.getByRole("option", { name: /^origin\/main origin/ })).toBeVisible();
const dropdownBox = await dropdown.boundingBox();
apps/web/e2e/tests/ssh/launch-task.spec.ts
+2
View File
@@ -395,8 +395,10 @@ printf 'prepared\\n' > "$workspace/custom-prepare-marker"
test("stopping the session cleans up the session runtime dir but leaves the task dir", async ({
apiClient,
seedData,
backend,
}) => {
test.setTimeout(180_000);
await backend.ensureReady();
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"H7 stop cleanup",
apps/web/e2e/tests/task/autopilot-mode.spec.ts
+21 -17
View File
@@ -1,7 +1,11 @@
import { expect, test } from "../../fixtures/test-base";
import type { ApiClient } from "../../helpers/api-client";
import { useRegularMode } from "../../helpers/regular-mode";
import { waitForSessionDone, waitForSessionState } from "../../helpers/session";
import {
waitForAgentMessage,
waitForSessionDone,
waitForSessionState,
} from "../../helpers/session";
import { KanbanPage } from "../../pages/kanban-page";
import { SessionPage } from "../../pages/session-page";
@@ -59,6 +63,18 @@ async function waitForParentQuestion(apiClient: ApiClient, parentTaskID: string)
return questionID;
}
async function waitForChildAnswerTurn(apiClient: ApiClient, sessionID: string): Promise<void> {
// The mock answer turn can complete before a state poll observes RUNNING.
// A second durable turn is the stable proof that the parent answer was
// admitted and executed, even when the session is already waiting again.
await expect
.poll(async () => (await apiClient.listSessionTurns(sessionID)).turns.length, {
timeout: 60_000,
message: "parent answer should create a child turn",
})
.toBeGreaterThanOrEqual(2);
}
test.describe("Task autopilot", () => {
test("shows the profile, waits for the parent, and resumes once", async ({
testPage,
@@ -133,15 +149,7 @@ test.describe("Task autopilot", () => {
const questionID = await waitForParentQuestion(apiClient, parent.id);
await expect
.poll(
async () => {
const { sessions } = await apiClient.listTaskSessions(child.id);
return sessions[0]?.state ?? "";
},
{ timeout: 60_000, message: "parent answer should resume the child" },
)
.not.toBe("WAITING_FOR_INPUT");
await waitForChildAnswerTurn(apiClient, childSessionId);
const childSession = new SessionPage(testPage);
await testPage.goto(`/t/${child.id}`);
@@ -170,7 +178,7 @@ test.describe("Task autopilot", () => {
apiClient,
seedData,
}) => {
test.setTimeout(120_000);
test.setTimeout(240_000);
const parent = await apiClient.createTaskWithAgent(
seedData.workspaceId,
@@ -218,12 +226,8 @@ test.describe("Task autopilot", () => {
});
const questionID = await waitForParentQuestion(apiClient, parent.id);
await expect
.poll(async () => (await apiClient.listTaskSessions(child.id)).sessions[0]?.state ?? "", {
timeout: 60_000,
message: "queued parent answer should resume the child",
})
.not.toBe("WAITING_FOR_INPUT");
await waitForAgentMessage(apiClient, parent.session_id, "Parent is ready.", 60_000);
await waitForChildAnswerTurn(apiClient, child.session_id);
const childMessages = await apiClient.listSessionMessages(child.session_id);
expect(
apps/web/e2e/tests/task/file-tree-create.spec.ts
+66 -45
View File
@@ -25,17 +25,48 @@ async function setupTask(
testPage: Page,
apiClient: ApiClient,
seedData: { workspaceId: string; workflowId: string; startStepId: string; repositoryId: string },
profileName: string,
taskTitle: string,
options: { profileName: string; taskTitle: string; requiredPath?: string },
) {
const profile = await createStandardProfile(apiClient, profileName);
await apiClient.createTaskWithAgent(seedData.workspaceId, taskTitle, profile.id, {
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
});
const session = await openTaskSession(testPage, taskTitle);
const profile = await createStandardProfile(apiClient, options.profileName);
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
options.taskTitle,
profile.id,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
if (options.requiredPath) {
let workspacePath = "";
await expect
.poll(async () => (await apiClient.getTaskEnvironment(task.id))?.status ?? null, {
timeout: 30_000,
message: `Waiting for ${options.taskTitle} task environment to be ready`,
})
.toBe("ready");
await expect
.poll(
async () => {
const environment = await apiClient.getTaskEnvironment(task.id);
workspacePath =
environment?.workspace_path ?? environment?.repos?.[0]?.worktree_path ?? "";
return Boolean(
workspacePath && fs.existsSync(path.join(workspacePath, options.requiredPath!)),
);
},
{
timeout: 60_000,
message: `Waiting for ${options.requiredPath} in the ${options.taskTitle} worktree`,
},
)
.toBe(true);
}
const session = await openTaskSession(testPage, options.taskTitle);
await session.clickTab("Files");
return session;
}
@@ -74,13 +105,11 @@ test.describe("File tree create file", () => {
git.stageAll();
git.commit("seed");
const session = await setupTask(
testPage,
apiClient,
seedData,
"ft-create-root",
"FT Create Root",
);
const session = await setupTask(testPage, apiClient, seedData, {
profileName: "ft-create-root",
taskTitle: "FT Create Root",
requiredPath: "seed.ts",
});
await expect(session.fileTreeNode("seed.ts")).toBeVisible({ timeout: 15_000 });
@@ -109,13 +138,11 @@ test.describe("File tree create file", () => {
git.stageAll();
git.commit("seed select-all files");
const session = await setupTask(
testPage,
apiClient,
seedData,
"ft-create-select-all",
"FT Create Select All",
);
const session = await setupTask(testPage, apiClient, seedData, {
profileName: "ft-create-select-all",
taskTitle: "FT Create Select All",
requiredPath: "select-all-alpha.ts",
});
await expect(session.fileTreeNode("select-all-alpha.ts")).toBeVisible({ timeout: 15_000 });
const input = await startCreateAtRoot(testPage);
@@ -149,13 +176,11 @@ test.describe("File tree create file", () => {
git.stageAll();
git.commit("seed scope");
const session = await setupTask(
testPage,
apiClient,
seedData,
"ft-create-folder",
"FT Create In Folder",
);
const session = await setupTask(testPage, apiClient, seedData, {
profileName: "ft-create-folder",
taskTitle: "FT Create In Folder",
requiredPath: "scope/existing.ts",
});
// Expand the folder so it becomes the "active folder" for handleStartCreate.
const folder = session.fileTreeNode("scope");
@@ -185,13 +210,11 @@ test.describe("File tree create file", () => {
git.stageAll();
git.commit("seed");
const session = await setupTask(
testPage,
apiClient,
seedData,
"ft-create-implicit",
"FT Create Implicit Folder",
);
const session = await setupTask(testPage, apiClient, seedData, {
profileName: "ft-create-implicit",
taskTitle: "FT Create Implicit Folder",
requiredPath: "seed.ts",
});
await expect(session.fileTreeNode("seed.ts")).toBeVisible({ timeout: 15_000 });
@@ -217,13 +240,11 @@ test.describe("File tree create file", () => {
git.stageAll();
git.commit("seed");
const session = await setupTask(
testPage,
apiClient,
seedData,
"ft-create-cancel",
"FT Create Cancel",
);
const session = await setupTask(testPage, apiClient, seedData, {
profileName: "ft-create-cancel",
taskTitle: "FT Create Cancel",
requiredPath: "seed.ts",
});
await expect(session.fileTreeNode("seed.ts")).toBeVisible({ timeout: 15_000 });
apps/web/e2e/tests/task/file-tree-download.spec.ts
+57 -21
View File
@@ -3,33 +3,57 @@ import path from "node:path";
import fs from "node:fs";
import { test, expect } from "../../fixtures/test-base";
import type { ApiClient } from "../../helpers/api-client";
import {
GitHelper,
makeGitEnv,
openTaskSession,
createStandardProfile,
} from "../../helpers/git-helper";
import { GitHelper, makeGitEnv, createStandardProfile } from "../../helpers/git-helper";
import { SessionPage } from "../../pages/session-page";
// Download is wired in file-context-menu.tsx → useFileOperations.downloadFile →
// downloadFileContent → triggerFileDownload (Blob + <a download>). We drive
// the visible flow: right-click a file, pick Download, assert the browser
// download event fires with the right filename and content.
async function setupTask(
testPage: Page,
apiClient: ApiClient,
seedData: { workspaceId: string; workflowId: string; startStepId: string; repositoryId: string },
profileName: string,
taskTitle: string,
) {
async function setupTask({
testPage,
apiClient,
seedData,
profileName,
taskTitle,
requiredPath,
}: {
testPage: Page;
apiClient: ApiClient;
seedData: { workspaceId: string; workflowId: string; startStepId: string; repositoryId: string };
profileName: string;
taskTitle: string;
requiredPath: string;
}) {
const profile = await createStandardProfile(apiClient, profileName);
await apiClient.createTaskWithAgent(seedData.workspaceId, taskTitle, profile.id, {
const task = await apiClient.createTaskWithAgent(seedData.workspaceId, taskTitle, profile.id, {
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
});
const session = await openTaskSession(testPage, taskTitle);
let workspacePath = "";
await expect
.poll(async () => (await apiClient.getTaskEnvironment(task.id))?.status ?? null, {
timeout: 30_000,
message: `Waiting for ${taskTitle} task environment to be ready`,
})
.toBe("ready");
await expect
.poll(
async () => {
const environment = await apiClient.getTaskEnvironment(task.id);
workspacePath = environment?.workspace_path ?? environment?.repos?.[0]?.worktree_path ?? "";
return Boolean(workspacePath && fs.existsSync(path.join(workspacePath, requiredPath)));
},
{ timeout: 60_000, message: `Waiting for ${taskTitle} worktree materialization` },
)
.toBe(true);
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
await session.clickTab("Files");
return session;
}
@@ -49,10 +73,16 @@ test.describe("File tree Download", () => {
git.stageAll();
git.commit("seed download file");
const session = await setupTask(testPage, apiClient, seedData, "ft-dl", "FT Download");
const session = await setupTask({
testPage,
apiClient,
seedData,
profileName: "ft-dl",
taskTitle: "FT Download",
requiredPath: fileName,
});
const node = session.fileTreeNode(fileName);
await expect(node).toBeVisible({ timeout: 15_000 });
const node = await session.fileTree.waitForFileTreeNode(fileName);
await node.click({ button: "right" });
const downloadItem = testPage.getByRole("menuitem", { name: "Download" });
@@ -83,10 +113,16 @@ test.describe("File tree Download", () => {
git.stageAll();
git.commit("seed subdir");
const session = await setupTask(testPage, apiClient, seedData, "ft-dl-dir", "FT Download Dir");
const session = await setupTask({
testPage,
apiClient,
seedData,
profileName: "ft-dl-dir",
taskTitle: "FT Download Dir",
requiredPath: "subdir/inside.txt",
});
const dirNode = session.fileTreeNode("subdir");
await expect(dirNode).toBeVisible({ timeout: 15_000 });
const dirNode = await session.fileTree.waitForFileTreeNode("subdir");
await dirNode.click({ button: "right" });
// The menu itself must render (Delete/Rename are still available), but
apps/web/e2e/tests/task/file-tree-drag-drop.spec.ts
+116 -60
View File
@@ -1,14 +1,10 @@
import { type Page, type Locator } from "@playwright/test";
import { type Page } from "@playwright/test";
import fs from "node:fs";
import path from "node:path";
import { test, expect } from "../../fixtures/test-base";
import type { ApiClient } from "../../helpers/api-client";
import {
GitHelper,
makeGitEnv,
openTaskSession,
createStandardProfile,
} from "../../helpers/git-helper";
import { SessionPage } from "../../pages/session-page";
import { GitHelper, makeGitEnv, createStandardProfile } from "../../helpers/git-helper";
// DnD in file-browser.tsx uses native HTML5 drag events (dragstart, dragover,
// drop) keyed off React's onDragStart/Over/Drop. Playwright's locator.dragTo()
@@ -21,13 +17,21 @@ import {
// workaround for testing HTML5 DnD in Playwright and mirrors what the user
// would do.
async function setupTask(
testPage: Page,
apiClient: ApiClient,
seedData: { workspaceId: string; workflowId: string; startStepId: string; repositoryId: string },
profileName: string,
taskTitle: string,
) {
async function setupTask({
testPage,
apiClient,
seedData,
profileName,
taskTitle,
requiredPath,
}: {
testPage: Page;
apiClient: ApiClient;
seedData: { workspaceId: string; workflowId: string; startStepId: string; repositoryId: string };
profileName: string;
taskTitle: string;
requiredPath: string;
}) {
const profile = await createStandardProfile(apiClient, profileName);
const task = await apiClient.createTaskWithAgent(seedData.workspaceId, taskTitle, profile.id, {
description: "/e2e:simple-message",
@@ -40,6 +44,7 @@ async function setupTask(
// for the environment's durable ready state before the first tree request;
// otherwise the tree can legitimately snapshot the repository while the
// agent session is still being attached to it.
let workspacePath = "";
await expect
.poll(async () => (await apiClient.getTaskEnvironment(task.id))?.status ?? null, {
timeout: 30_000,
@@ -47,46 +52,92 @@ async function setupTask(
})
.toBe("ready");
const session = await openTaskSession(testPage, taskTitle);
// Environment readiness and repository materialization are separate
// transitions. Wait for the exact fixture file in the task worktree before
// the first tree snapshot, otherwise a valid early tree can be retained
// while the checkout is still being populated.
await expect
.poll(
async () => {
const environment = await apiClient.getTaskEnvironment(task.id);
workspacePath = environment?.workspace_path ?? environment?.repos?.[0]?.worktree_path ?? "";
return Boolean(workspacePath && fs.existsSync(path.join(workspacePath, requiredPath)));
},
{ timeout: 60_000, message: `Waiting for ${requiredPath} in the ${taskTitle} worktree` },
)
.toBe(true);
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
await session.clickTab("Files");
return session;
}
async function dispatchHtmlDnd(testPage: Page, source: Locator, target: Locator) {
// Make sure both are attached and visible.
async function dispatchHtmlDnd(testPage: Page, sourcePath: string, targetPath: string) {
// Virtualized trees can unmount the source while the target is revealed.
// Keep the browser DataTransfer on the page between the two scrolls so the
// source and target do not need to be mounted at the same time.
const source = testPage.locator(
`[data-testid="file-tree-node"][data-path=${JSON.stringify(sourcePath)}]:visible`,
);
const target = testPage.locator(
`[data-testid="file-tree-node"][data-path=${JSON.stringify(targetPath)}]:visible`,
);
await expect(source).toBeVisible({ timeout: 30_000 });
await source.scrollIntoViewIfNeeded();
await testPage.evaluate((nodePath) => {
const row = Array.from(document.querySelectorAll('[data-testid="file-tree-node"]')).find(
(element) =>
element.getAttribute("data-path") === nodePath &&
element.getBoundingClientRect().width > 0 &&
element.getBoundingClientRect().height > 0,
);
if (!row) throw new Error(`DnD source is not mounted: ${nodePath}`);
const dataTransfer = new DataTransfer();
const event = new DragEvent("dragstart", {
bubbles: true,
cancelable: true,
composed: true,
dataTransfer,
});
row.dispatchEvent(event);
Object.defineProperty(window, "__kandevE2eDataTransfer", {
configurable: true,
value: dataTransfer,
});
}, sourcePath);
await expect(target).toBeVisible({ timeout: 30_000 });
await target.scrollIntoViewIfNeeded();
const sourceHandle = await source.elementHandle();
const targetHandle = await target.elementHandle();
if (!sourceHandle || !targetHandle) throw new Error("DnD: source/target missing");
// Real browsers reuse the same DataTransfer across dragstart -> drop. We
// do the whole sequence inside one evaluate() so the instance is shared.
// dragover gets explicit preventDefault() because that's the contract
// React's onDragOver fulfils when the drop is valid - without it, the
// browser would interpret the drop as a navigation attempt for any
// text-typed data and unload the page.
await testPage.evaluate(
([src, dst]) => {
const dt = new DataTransfer();
const fireOn = (el: Element, type: string) => {
const ev = new DragEvent(type, {
bubbles: true,
cancelable: true,
composed: true,
dataTransfer: dt,
});
el.dispatchEvent(ev);
return ev;
({ sourcePath, targetPath: nodePath }) => {
const row = Array.from(document.querySelectorAll('[data-testid="file-tree-node"]')).find(
(element) =>
element.getAttribute("data-path") === nodePath &&
element.getBoundingClientRect().width > 0 &&
element.getBoundingClientRect().height > 0,
);
const dataTransfer = (window as Window & { __kandevE2eDataTransfer?: DataTransfer })
.__kandevE2eDataTransfer;
if (!row || !dataTransfer) throw new Error(`DnD target is not mounted: ${nodePath}`);
const fireOn = (element: Element, type: string) => {
element.dispatchEvent(
new DragEvent(type, { bubbles: true, cancelable: true, composed: true, dataTransfer }),
);
};
fireOn(src, "dragstart");
fireOn(dst, "dragenter");
fireOn(dst, "dragover");
fireOn(dst, "drop");
// Source may have been removed from the DOM by the optimistic move -
// guard the dragend call.
if (src.isConnected) fireOn(src, "dragend");
fireOn(row, "dragenter");
fireOn(row, "dragover");
fireOn(row, "drop");
const source = Array.from(document.querySelectorAll('[data-testid="file-tree-node"]')).find(
(element) => element.getAttribute("data-path") === sourcePath,
);
if (source) fireOn(source, "dragend");
document.dispatchEvent(new DragEvent("dragend", { bubbles: true, dataTransfer }));
delete (window as Window & { __kandevE2eDataTransfer?: DataTransfer })
.__kandevE2eDataTransfer;
},
[sourceHandle, targetHandle] as const,
{ sourcePath, targetPath },
);
}
@@ -104,20 +155,25 @@ test.describe("File tree drag and drop", () => {
git.stageAll();
git.commit("seed dnd");
const session = await setupTask(testPage, apiClient, seedData, "ft-dnd-move", "FT DnD Move");
const session = await setupTask({
testPage,
apiClient,
seedData,
profileName: "ft-dnd-move",
taskTitle: "FT DnD Move",
requiredPath: "movable.ts",
});
const file = session.fileTreeNode("movable.ts");
const folder = session.fileTreeNode("target-dir");
await expect(file).toBeVisible({ timeout: 15_000 });
await expect(folder).toBeVisible({ timeout: 15_000 });
await session.fileTree.waitForFileTreeNode("movable.ts");
await session.fileTree.waitForFileTreeNode("target-dir");
await dispatchHtmlDnd(testPage, file, folder);
await dispatchHtmlDnd(testPage, "movable.ts", "target-dir");
// The file is removed from the root immediately (optimistic update).
await expect(session.fileTreeNode("movable.ts")).toHaveCount(0, { timeout: 10_000 });
// Expand the target folder to verify the moved child landed inside.
// moveNodesInTree does not auto-expand the drop target.
await folder.click();
await session.fileTreeNode("target-dir").click();
await expect(session.fileTreeNode("target-dir/movable.ts")).toBeVisible({ timeout: 10_000 });
await expect
@@ -140,27 +196,27 @@ test.describe("File tree drag and drop", () => {
git.stageAll();
git.commit("seed selfdir");
const session = await setupTask(
const session = await setupTask({
testPage,
apiClient,
seedData,
"ft-dnd-self",
"FT DnD Self Reject",
);
profileName: "ft-dnd-self",
taskTitle: "FT DnD Self Reject",
requiredPath: "selfdir/leaf.ts",
});
const folder = session.fileTreeNode("selfdir");
await expect(folder).toBeVisible({ timeout: 15_000 });
await session.fileTree.waitForFileTreeNode("selfdir");
// Drop onto self: handleDragOver short-circuits via isDropInvalid so
// preventDefault is never called, which means the browser would never
// fire drop in real usage. Dispatching events directly bypasses that
// guard, but the drop handler also calls isDropInvalid and bails.
await dispatchHtmlDnd(testPage, folder, folder);
await dispatchHtmlDnd(testPage, "selfdir", "selfdir");
// Tree is unchanged: folder is still at root with its original child.
await expect(session.fileTreeNode("selfdir")).toBeVisible({ timeout: 5_000 });
// Expand and confirm the child is still there.
await folder.click();
await session.fileTreeNode("selfdir").click();
await expect(session.fileTreeNode("selfdir/leaf.ts")).toBeVisible({ timeout: 10_000 });
// Disk untouched - no self-nested directory created.
apps/web/e2e/tests/task/parked-background-work.spec.ts
+26 -12
View File
@@ -105,6 +105,29 @@ async function waitForFirstSessionId(apiClient: ApiClient, taskId: string): Prom
return sessionId;
}
/** Creates a fresh profile after the per-test cleanup has run. The suite
* restart registers the provider, while testPage cleanup removes every
* non-seed profile, so a profile captured during beforeAll may be gone before
* the task is created. */
async function createClaudeAcpProfile(apiClient: ApiClient): Promise<string> {
let agentId = "";
await expect
.poll(
async () => {
const { agents } = await apiClient.listAgents();
agentId = agents.find((agent) => agent.name === "claude-acp")?.id ?? "";
return agentId;
},
{ message: "claude-acp agent should be registered", timeout: 30_000 },
)
.not.toBe("");
const profile = await apiClient.createAgentProfile(agentId, `Parked E2E ${Date.now()}`, {
model: "mock-fast",
});
return profile.id;
}
/** Creates a task whose foreground turn registers a shell-kind detached
* launch after `settleDelay`, scripts its probe to hold "live" indefinitely
* before that settle can happen, then waits for the settle itself. */
@@ -140,22 +163,11 @@ async function createParkedTask(
}
test.describe("Parked on background work", () => {
let claudeAcpProfileId: string;
test.beforeAll(async ({ backend, apiClient }) => {
test.beforeAll(async ({ backend }) => {
await backend.restart({
KANDEV_PARKED_PROBE_INTERVAL: PARKED_PROBE_INTERVAL,
KANDEV_MOCK_PROVIDERS: "claude-acp",
});
const { agents } = await apiClient.listAgents();
const claudeAgent = agents.find((agent) => agent.name === "claude-acp");
const profileId = claudeAgent?.profiles[0]?.id;
if (!profileId) {
throw new Error(
"E2E seed has no claude-acp mock profile after KANDEV_MOCK_PROVIDERS restart",
);
}
claudeAcpProfileId = profileId;
});
test.afterAll(async ({ backend }) => {
@@ -168,6 +180,7 @@ test.describe("Parked on background work", () => {
testPage,
}) => {
test.setTimeout(120_000);
const claudeAcpProfileId = await createClaudeAcpProfile(apiClient);
// Mount the sidebar on a different task first so the client is live
// before the parked task's own WS traffic starts (see
@@ -219,6 +232,7 @@ test.describe("Parked on background work", () => {
testPage,
}) => {
test.setTimeout(120_000);
const claudeAcpProfileId = await createClaudeAcpProfile(apiClient);
// Keep the sidebar mounted before the parked task settles. This lets the
// test observe the task.updated projection instead of depending on a
apps/web/e2e/tests/task/repository-discovery-consent.spec.ts
+1 -1
View File
@@ -37,7 +37,7 @@ test.describe("Desktop repository discovery consent", () => {
]);
expect(chooseBox).not.toBeNull();
expect(refreshBox).not.toBeNull();
expect(refreshBox!.height).toBeCloseTo(chooseBox!.height, 1);
expect(Math.abs(refreshBox!.height - chooseBox!.height)).toBeLessThanOrEqual(1);
expect(refreshBox!.height).toBeLessThanOrEqual(32);
});
apps/web/e2e/tests/task/task-listing-view-preferences.spec.ts
+11
View File
@@ -62,6 +62,17 @@ test.describe("Task listing display preferences", () => {
mergeable_state: "clean",
});
// This test's first assertion is about the default presentation. Set and
// observe the backend value after seeding so an earlier settings update
// cannot win a race with the page's initial boot payload.
await apiClient.saveUserSettings({
workspace_id: seedData.workspaceId,
tasks_list_show_details: false,
});
await expect
.poll(async () => (await apiClient.getUserSettings()).settings.tasks_list_show_details)
.toBe(false);
await testPage.goto("/tasks");
const row = testPage.getByTestId("tasks-list-row").filter({ hasText: RICH_TASK_TITLE });
await expect(row).toBeVisible();
apps/web/e2e/tests/terminal/terminal-dockview-ui.spec.ts
+4 -2
View File
@@ -523,7 +523,9 @@ test.describe("Terminals — dockview UI", () => {
// Create a second terminal so we have a non-default row to click.
await clickNewTerminalInPlusMenu(testPage, session);
await expect(testPage.getByTestId("terminal-tab-seq-2")).toBeVisible({ timeout: 10_000 });
const secondTab = testPage.getByTestId("terminal-tab-seq-2");
await expect(secondTab).toBeVisible({ timeout: 30_000 });
await session.expectTerminalConnected(30_000);
// Count terminal tab content elements before the focus click. Polling the
// count is the wait: it returns as soon as the second tab has rendered
@@ -535,7 +537,7 @@ test.describe("Terminals — dockview UI", () => {
});
await expect
.poll(() => terminalContent.count(), {
timeout: 10_000,
timeout: 30_000,
message: "two terminal tabs before clicking reopen",
})
.toBeGreaterThanOrEqual(2);
apps/web/e2e/tests/workflow/workflow-settings.spec.ts
+5
View File
@@ -166,6 +166,11 @@ test.describe("Workflow settings", () => {
});
await testPage.getByRole("option", { name: /Mock Smart/ }).click({ force: true });
await expect(settings).toContainText("Mock Smart", { timeout: 15_000 });
// The shared picker stays open when dependent options are available. Close
// it explicitly so the next interaction starts from a deterministic state.
if ((await settings.getAttribute("aria-expanded")) === "true") {
await testPage.keyboard.press("Escape");
}
await expect(settings).toHaveAttribute("aria-expanded", "false");
await settings.click();
const effortTrigger = testPage.getByTestId("config-option-trigger-effort");
apps/web/e2e/tests/workflow/workflow-start-step.spec.ts
+5 -3
View File
@@ -238,9 +238,11 @@ test.describe("Workflow start step placement", () => {
});
// Agent responds after the delay
await expect(session.chat.getByText("delayed mock response", { exact: false })).toBeVisible({
timeout: 30_000,
});
const delayedResponse = session.chat
.getByTestId("agent-message-highlight")
.filter({ hasText: "delayed mock response" });
await expect(delayedResponse).toHaveCount(1, { timeout: 30_000 });
await expect(delayedResponse).toBeVisible();
// on_turn_complete fires on In Progress → task moves to Done
await expect(session.stepperStep("Done")).toHaveAttribute("aria-current", "step", {
apps/web/e2e/tests/workflow/workflow-step-progress.spec.ts
+1 -1
View File
@@ -129,7 +129,7 @@ test.describe("Workflow step progress", () => {
await testPage.goto(`/t/${task.task_id}`);
await new SessionPage(testPage).waitForLoad();
const stepper = testPage.getByTestId("workflow-stepper");
const stepper = testPage.locator('[data-testid="workflow-stepper"]:visible').first();
const targetTrigger = stepper.getByTestId(`workflow-step-${targetStep.name}`);
await expect(targetTrigger).toBeVisible();
await expect(stepper.getByTestId("workflow-stepper-minimal")).toHaveCount(0);
apps/web/hooks/domains/session/use-queue.test.ts
+37
View File
@@ -724,6 +724,43 @@ describe("useQueue clearAll", () => {
});
});
describe("useQueue clearAll optimistic state", () => {
beforeEach(() => {
queueApiMock.clearQueue.mockReset();
queueApiMock.clearQueue.mockResolvedValue(undefined);
});
it("clears the visible queue before a slow clear request resolves", async () => {
const pendingClear = Promise.withResolvers<void>();
queueApiMock.clearQueue.mockReturnValueOnce(pendingClear.promise);
const { result } = renderHook(() => useQueue(SESSION_ID));
await waitFor(() => expect(queueApiMock.getQueueStatus).toHaveBeenCalled());
mockState.setQueueEntries.mockClear();
let clearPromise!: Promise<void>;
act(() => {
clearPromise = result.current.clearAll();
});
await waitFor(() =>
expect(mockState.setQueueEntries).toHaveBeenCalledWith(SESSION_ID, [], {
count: 0,
max: 0,
mergeEnabled: true,
autoRun: true,
taskId: TASK_ID,
sessionIncarnationId: IDENTITY.session_incarnation_id,
}),
);
expect(queueApiMock.clearQueue).toHaveBeenCalledWith(IDENTITY);
await act(async () => {
pendingClear.resolve();
await clearPromise;
});
});
});
describe("useQueue removeEntry", () => {
beforeEach(() => {
queueApiMock.removeQueuedEntry.mockReset();
apps/web/hooks/domains/session/use-queue.ts
+13 -9
View File
@@ -280,17 +280,21 @@ function useClearAllAction({
let mutationFailed = false;
let mutationError: unknown;
try {
// Close the panel as soon as the user commits the action. Invalidate
// older refetches first so a response that was already in flight cannot
// restore the pre-clear list while the mutation is pending. The
// authoritative refetch below restores the list if the request fails.
invalidateRefetch(sessionId);
setQueueEntries(sessionId, [], {
count: 0,
max: metaMax ?? 0,
mergeEnabled: metaMergeEnabled ?? true,
autoRun: metaAutoRun ?? true,
taskId: identity.task_id,
sessionIncarnationId: incarnationId,
});
try {
await clearQueue(identity);
invalidateRefetch(sessionId);
setQueueEntries(sessionId, [], {
count: 0,
max: metaMax ?? 0,
mergeEnabled: metaMergeEnabled ?? true,
autoRun: metaAutoRun ?? true,
taskId: identity.task_id,
sessionIncarnationId: incarnationId,
});
} catch (err) {
mutationFailed = true;
mutationError = err;
apps/web/hooks/domains/sidebar/use-sidebar-hover-reveal.test.ts
+14
View File
@@ -208,6 +208,20 @@ it("drops stale focus when navigation replaces the revealed contents", () => {
expect(result.current.revealed).toBe(false);
});
it("requires a fresh pointer entry after a collapse state change", () => {
const { result, rerender } = setup();
act(() => result.current.handlers.onPointerEnter(pointer()));
act(() => vi.advanceTimersByTime(500));
expect(result.current.revealed).toBe(true);
rerender({ collapsed: false, pathname: "/" });
rerender({ collapsed: true, pathname: "/" });
act(() => result.current.handlers.onPointerEnter(pointer()));
act(() => vi.advanceTimersByTime(500));
expect(result.current.revealed).toBe(true);
});
// @covers AC-UI-SIDEBAR-HOVER-002.4
it("honors disabled hover, custom dwell and setting changes", () => {
const { result, rerender } = renderHook(
apps/web/hooks/domains/sidebar/use-sidebar-hover-reveal.ts
+5
View File
@@ -82,6 +82,11 @@ export function useSidebarHoverReveal({
const media = window.matchMedia(HOVER_QUERY);
const reset = (capabilityChanged = false) => {
cancelTimer();
// A collapse, route change, or capability change can remove the element
// under the pointer before the browser dispatches pointerleave. Require
// a fresh entry so that a stale inside flag cannot suppress the next
// hover gesture.
inside.current = false;
const changed =
capabilityChanged ||
previousSettings.current.eligible !== eligible ||
apps/web/lib/state/slices/session/session-slice.ts
+13 -2
View File
@@ -98,6 +98,10 @@ function mergeMessageAtIndex(messages: Message[], message: Message): void {
messages[index] = merged;
}
function isTransientRetryNotice(message: Message): boolean {
return message.type === "status" && message.metadata?.retrying === true;
}
/** Return a new messages array with the message matching `messageId` removed. */
function removeMessageByID(messages: Message[], messageId: string) {
return messages.filter((message) => message.id !== messageId);
@@ -324,8 +328,15 @@ function buildMessageActions(set: ImmerSet) {
updateMessages: (messages: Parameters<SessionSlice["updateMessages"]>[0]) =>
set((draft) => {
for (const message of messages) {
const sessionMessages = draft.messages.bySession[message.session_id];
if (sessionMessages) mergeMessageAtIndex(sessionMessages, message);
let sessionMessages = draft.messages.bySession[message.session_id];
if (!sessionMessages && isTransientRetryNotice(message)) {
sessionMessages = draft.messages.bySession[message.session_id] = [];
}
if (sessionMessages) {
const hasMessage = sessionMessages.some((entry) => entry.id === message.id);
if (hasMessage) mergeMessageAtIndex(sessionMessages, message);
else if (isTransientRetryNotice(message)) sessionMessages.push(message);
}
updatePromptMessage(draft, message);
}
}),
apps/web/lib/state/slices/session/session-slice.update-messages.test.ts
+18
View File
@@ -86,6 +86,24 @@ describe("updateMessages", () => {
expect(store.getState().messages.bySession["session-2"][0]).toBe(otherSessionMessage);
});
it("upserts a retry notice when its reused row is outside the loaded window", () => {
const store = makeStore();
store.getState().updateMessages([
makeMessage("retry", "Model at capacity", SESSION, {
type: "status",
metadata: { retrying: true, attempt: 2 },
}),
]);
expect(store.getState().messages.bySession[SESSION]).toEqual([
expect.objectContaining({
id: "retry",
type: "status",
metadata: { retrying: true, attempt: 2 },
}),
]);
});
it("fans batched updates into the prompt cache", () => {
const store = makeStore();
store.getState().addMessage(
apps/web/lib/ws/handlers/messages.test.ts
+39
View File
@@ -11,6 +11,8 @@ const SESSION_ID = "session-1";
const QUESTION_ID = "question-1";
const QUESTION_CONTENT = "Choose";
const ADD_ACTION = "session.message.added";
const RETRY_MESSAGE_ID = "retry-message";
const RETRY_CONTENT = "Model at capacity, retrying in 10s";
function pendingRevision(sequence: number) {
return { epoch: PROJECTION_EPOCH, sequence };
@@ -211,6 +213,43 @@ describe("session message frame scheduler", () => {
});
});
describe("session message retry notices", () => {
it("updates a retry notice in place when a later attempt arrives", () => {
const { store, addMessage, updateMessages } = makeStore({
[SESSION_ID]: [{ id: RETRY_MESSAGE_ID, updated_at: "2026-08-02T00:00:01.000Z" }],
});
const registration = createMessagesHandlerRegistration(store);
registration.handlers["session.message.updated"]!({
...makeUpdated(SESSION_ID, RETRY_MESSAGE_ID, RETRY_CONTENT),
payload: makePayload(SESSION_ID, RETRY_MESSAGE_ID, RETRY_CONTENT, {
type: "status",
metadata: {
retrying: true,
attempt: 2,
max_attempts: 5,
retry_in_seconds: 10,
retry_at: "2026-08-02T00:00:12.000Z",
provider_name: "codex-acp",
},
updated_at: "2026-08-02T00:00:02.000Z",
}),
});
registration.scheduler.flush();
expect(addMessage).not.toHaveBeenCalled();
expect(updateMessages).toHaveBeenCalledWith([
expect.objectContaining({
id: RETRY_MESSAGE_ID,
type: "status",
content: RETRY_CONTENT,
metadata: expect.objectContaining({ attempt: 2, retry_at: "2026-08-02T00:00:12.000Z" }),
}),
]);
registration.dispose();
});
});
describe("session message prompt_index mapping", () => {
it("maps prompt_index from a session.message.added payload", () => {
const { store, addMessage } = makeStore();
docs/plans/single-capacity-retry-banner/plan.md
+227
View File
@@ -0,0 +1,227 @@
---
created: 2026-09-15
status: implemented
requirements:
- REQ-PLATFORM-PROVIDER-ERROR-RECOVERY-001
system_design:
- ../../specs/platform/system-design/provider-error-recovery.md
legacy_specs: []
---
# Implementation Plan: Single Capacity Retry Banner
## Overview
Reuse one persisted retry notice and update it through successive attempts.
One sequential work order covers persistence, event delivery, and rendered proof.
Platform owns this outcome because it owns the interactive retry lifecycle.
## Evidence and root cause
The supplied screenshot shows attempt 1 and attempt 2 together.
`createTransientRetryStatusMessage` always calls `CreateSessionMessage`.
`nextTransientAttempt` cancels the timer but does not remove the prior message.
`TestResetTransientRetry_ResolvesPersistedNotices` explicitly expects two rows
before cleanup. This source trace explains the screenshot without a runtime launch.
The linked task's filtered history returned no matching messages.
Minimal reproduction: send `/overloaded:9` in an isolated mock session and wait
for attempt 2. Current E2E checks both attempt labels but never asserts one card.
## Scope
### In scope
- Update one message across attempts, including its deadline and Cancel metadata.
- Consolidate legacy duplicates on the next successful notice write.
- Preserve cleanup, authorization, retry policy, and session isolation.
- Prove same-message updates through persistence, WebSocket delivery, and reload.
### Out of scope
- Generic no-output warnings and resumed-agent transcript entries.
- Provider classification, retry limits, scheduling policy, and Office routing.
- A new layout, retry component, or public API.
## Technical approach
Extend `TransientRetryMessageService` with existing task-service `UpdateMessage`.
Keep task-service mutation and event publication as the single persistence path.
Use the selection and error rules in the
[interactive lifecycle design](../../specs/platform/system-design/provider-error-recovery.md#interactive-transient-retry-notice-lifecycle).
Preserve the original ID, timestamp, and turn association. Select the newest
matching notice when repairing legacy duplicates so the live row remains inside
the newest-message hydration window. Replace retry metadata as a unit so
obsolete provider or model fields cannot leak into later attempts. Upsert a
reused retry update in the client when its row is absent from the loaded window.
Retain existing message update handling and countdown rerender behavior.
Audit existing per-session guards before changing event ordering. Do not hold
`taskRuntimeStateMu` across storage operations or introduce recursive guard acquisition.
The earlier [cleanup package](../stale-transient-retry-notice/plan.md) remains
implemented. Its cleanup tests must seed legacy duplicates directly through the
task service once the normal notice writer stops creating duplicates.
Its transport-loss coverage was subsequently changed by the provider projection
contract (.24); this package does not restore automatic retries for that path.
## ASCII UI preview
### UI-01: Task chat during capacity retry
Entry: an open task session on desktop or phone. The chat owns scrolling.
Before (supplied screenshot):
```text
[ Model at capacity Retrying now... Attempt 1 of 5 Cancel ]
[ Model at capacity Retrying in 0:01 Attempt 2 of 5 Cancel ]
```
After (shared desktop and phone composition):
```text
[ ! Model at capacity Retrying in 0:10 Cancel ]
[ Provider: codex-acp ]
[ Attempt 2 of 5 ]
```
The same banner updates from attempt 1 to attempt 2. At zero, the countdown
shows the existing localized “Retrying now...” state. Existing running-state
visibility and terminal cleanup remain unchanged. Cancellation removes the
retry banner and exposes the existing manual recovery actions.
One banner, latest metadata, and one Cancel action are structural requirements.
Spacing and example values are illustrative. Phone uses the existing inline
`TransientRetryNotice` exemplar: reason first, provider second, attempt third.
The short status belongs beside the conversation and needs no new overlay.
Text wraps within the chat width. Cancel remains visible and touch-reachable.
No navigation, safe-area, or scroll-owner change is planned. Existing localized
copy and `role="status"` remain. The mobile rendered check verifies containment
and Cancel behavior. This view covers criteria .9, .10, .11, and .25.
## Tests
- Add `TestTransientRetryStatusMessage_UpdatesSameNotice` in
`event_handlers_transient_resolution_test.go`: attempt 1 then 2 retains one ID,
latest deadline, and one added event followed by an updated event (.25).
- Add cases for legacy duplicates mixed with unrelated status rows and another
session; list/update/delete errors; cancel/write races; and terminal cleanup
(.10, .11, .25). Directly seed duplicates for legacy cleanup tests.
- Existing `action-message.test.tsx` rerender coverage verifies updated attempt
and countdown (.9, .25). Extend `lib/ws/handlers/messages.test.ts` to prove
changed retry metadata updates the existing row rather than appending (.25).
## E2E tests
Strengthen `tests/session/transient-retry.spec.ts` in `chromium` and
`tests/session/mobile-transient-retry.spec.ts` in `mobile-chrome`.
Wait for attempt 2 through the actual event or backend state, then assert one
card with the latest attempt and one Cancel action. Reload and repeat the
assertions, accounting for a later valid attempt. Cancel and assert no retry
card remains and manual recovery is available (.9, .10, .11, .25).
Use causal waits rather than sleeps. Do not use `.last()` to conceal duplicates.
The mobile test also verifies no horizontal overflow and a reachable Cancel action.
## Work orders
- [x] [Task 01: Update one retry notice](task-01-update-one-retry-notice.md)
## Verification
Run sequentially from the repository root. Managed E2E commands rebuild artifacts.
```bash
(cd apps && pnpm install --frozen-lockfile)
(cd apps/backend && go test ./internal/orchestrator -run 'Test.*Transient.*' -count=1)
(cd apps/backend && go test -race ./internal/orchestrator -run 'Test.*Transient.*' -count=1)
(cd apps/web && pnpm exec vitest run components/task/chat/messages/action-message.test.tsx lib/ws/handlers/messages.test.ts lib/state/slices/session/session-slice.update-messages.test.ts)
(cd apps/web && pnpm run typecheck)
(cd apps/web && pnpm e2e:run --project chromium tests/session/transient-retry.spec.ts)
(cd apps/web && pnpm e2e:run --project mobile-chrome tests/session/mobile-transient-retry.spec.ts)
python3 scripts/list-docs.py validate
python3 scripts/lint-spec-files.py --all
git diff --check
```
## Verification results
Implementation and verification completed. The retry writer now updates the
newest matching status message, consolidates duplicates after a successful
update, preserves storage error behavior, and serializes writes with
retirement. Its per-session guard uses reference accounting for concurrent
users and a bounded five-minute retirement fence, so ordinary session churn
and deleted sessions do not retain obsolete state. Prompt evidence opens a
retired lifecycle only after a new execution identity is installed, and retry
timers arm after the failed turn is completed and the session is parked. The
client upserts a reused retry row when it is outside the loaded transcript
window. Desktop and mobile E2E cover attempt advancement, reload, stable
message identity, Cancel cleanup, and mobile document containment.
- Focused Go tests: passed.
- Focused Go tests with `-race`: passed.
- Full `internal/orchestrator` package tests: passed.
- Backend `make build`: passed.
- Backend `make lint`: passed with 0 issues.
- Changed-file Go lint with `--new-from-rev`: passed with 0 issues.
- Focused Vitest: 50 tests passed in 3 files.
- Web lint: passed with 0 issues.
- Web `pnpm run typecheck`: passed.
- Desktop managed E2E: 4 tests passed.
- Mobile managed E2E: 1 test passed.
- Specification catalog validation: 275 decisions and 947 specifications
validated.
- Specification lint: passed.
- `git diff --check`: passed.
- Review regressions: passed for session churn/deletion reclamation, guard
reference accounting, cancellation and terminal late failures, stale-event
identity fencing, new-prompt fence reset, concurrent failures sharing one
notice and timer entry, and client upsert of a reused retry row.
The repository-wide `pnpm run lint:e2e-sleeps` command remains non-clean because
of existing violations in untouched files, including missing ESLint rule
definitions and pre-existing hand-rolled waits. Targeted lint for the changed
E2E files passed.
CI fixup additionally stabilized the existing workflow settings and LSP E2E
surfaces. The workflow picker test dismisses a picker that can remain open
while dependent options are present. The LSP helper searches for exact paths
when virtualization leaves a valid file outside the mounted tree rows; its
archive cleanup test forces that condition with 48 preceding files.
- Workflow settings focused Chromium E2E with retries disabled and
`--repeat-each=3`: 3 passed.
- LSP file intelligence with retries disabled: 13 passed.
- LSP archive cleanup focused E2E with retries disabled: 1 passed with the
virtualized-tree regression fixture.
- Targeted ESLint and web TypeScript typecheck: passed.
The complete E2E blob audit for CI run `35032213664` exposed 13 retry
attempts across 12 tests even though the aggregate workflow was green. The
fixup stabilizes each reported path: mobile branch refresh activation, backend
startup interlock reads, virtualized and delayed rendering, provider readiness,
layout geometry, strict plan-comment selection, mobile containment, and parked
session profile setup. The parked suite now creates its temporary provider
profile after per-test cleanup, which preserves the profile used by task
creation across file ordering and repetition.
- Interlock helper Vitest regression: passed.
- Mobile CI regression batch with retries disabled and `--repeat-each=3`: 9
passed.
- Chromium/tablet regression batch with retries disabled and
`--repeat-each=3`: 15 passed.
- Git plus parked-work sequence with retries disabled: 2 passed.
- Parked-work regression with retries disabled and `--repeat-each=3`: 3
passed; full parked-work file: 2 passed.
## Risks
- A delayed update must not recreate a notice after cancellation or deletion.
- Persistence errors can leave older metadata or legacy duplicates until cleanup succeeds.
- The retained message belongs to an earlier turn; rendered tests must prove it stays current.
- Mock fail-then-success behavior has known lifecycle limitations. Backend tests
provide deterministic success cleanup evidence.
## Documentation impact
This package changes design intent only. Public documentation stays unchanged.
Existing provider-recovery requirements and design remain the authoritative pair.
docs/plans/single-capacity-retry-banner/task-01-update-one-retry-notice.md
+193
View File
@@ -0,0 +1,193 @@
---
id: "01-update-one-retry-notice"
title: "Update one retry notice"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-PLATFORM-PROVIDER-ERROR-RECOVERY-001
acceptance_criteria:
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.9
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.10
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.11
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25
system_design:
- ../../specs/platform/system-design/provider-error-recovery.md
---
# Task 01: Update One Retry Notice
## Summary
Update the existing retry message as each attempt is scheduled.
Prove that connected viewers and reloads show one current banner.
## In scope
- Task-service message updates, duplicate consolidation, and failure handling.
- Existing cancellation and terminal cleanup guarantees.
- Backend, store, component, desktop, and phone regression evidence.
## Out of scope
Generic transcript warnings, policy changes, Office routing, and new UI copy.
## Acceptance
- Consecutive attempts retain one persisted ID with latest metadata and emit
an update event. Legacy duplicates consolidate without changing unrelated rows.
- Store errors do not create extra notices. Cancellation and terminal cleanup
cannot be undone by stale writes. Existing authorization remains first.
- Desktop and phone show one current card after attempt advancement and reload.
Cancel removes it and retains the existing manual recovery actions.
## ASCII UI preview
See the [full before/after view](plan.md#ascii-ui-preview).
UI-01: task chat, desktop and phone, attempt 2 waiting (.9, .25):
```text
[ ! Model at capacity Retrying in 0:10 Cancel ]
[ Provider: codex-acp ]
[ Attempt 2 of 5 ]
```
Reuse the existing inline composition. The chat owns scrolling. Phone text
wraps and Cancel stays reachable. Running and terminal states retain existing
visibility rules (.10, .11). No extra card is appended.
## Implementation sequence
1. Add the same-ID backend regression and demonstrate the two-row failure.
2. Add the narrow update dependency and implement the design's selection rules.
3. Preserve existing guards and cleanup. Add error, mixed-row, and race tests.
4. Seed legacy duplicates directly in cleanup tests. Add store and rendered assertions.
5. Run the exact checks below and record each result in both package files.
## Verification
Run sequentially from the repository root. Use managed E2E builds.
```bash
(cd apps && pnpm install --frozen-lockfile)
(cd apps/backend && go test ./internal/orchestrator -run 'Test.*Transient.*' -count=1)
(cd apps/backend && go test -race ./internal/orchestrator -run 'Test.*Transient.*' -count=1)
(cd apps/web && pnpm exec vitest run components/task/chat/messages/action-message.test.tsx lib/ws/handlers/messages.test.ts)
(cd apps/web && pnpm run typecheck)
(cd apps/web && pnpm e2e:run --project chromium tests/session/transient-retry.spec.ts)
(cd apps/web && pnpm e2e:run --project mobile-chrome tests/session/mobile-transient-retry.spec.ts)
python3 scripts/list-docs.py validate
python3 scripts/lint-spec-files.py --all
git diff --check
```
## Files likely touched
- `apps/backend/internal/orchestrator/service.go`
- `apps/backend/internal/orchestrator/event_handlers_transient.go`
- `apps/backend/internal/orchestrator/event_handlers_transient_resolution_test.go`
- `apps/web/lib/ws/handlers/messages.test.ts`
- `apps/web/e2e/helpers/transient-retry.ts`
- `apps/web/e2e/tests/session/transient-retry.spec.ts`
- `apps/web/e2e/tests/session/mobile-transient-retry.spec.ts`
## Dependencies
None. Reuse the task-service-backed fixture and existing message update pipeline.
## Risks
Guard reentrancy, cancellation races, and old-turn visibility require focused proof.
Do not introduce database I/O under the global runtime-state mutex.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/platform/requirements/provider-error-recovery.md), criteria .9-.11 and .25.
- [Design](../../specs/platform/system-design/provider-error-recovery.md#interactive-transient-retry-notice-lifecycle).
- `newPersistentTransientRetryTestService`, `TransientRetryNotice`, and
`createMessagesHandlerRegistration` in the existing source and tests.
- [Earlier cleanup work order](../stale-transient-retry-notice/task-01-retire-stale-transient-retry-notices.md).
## Results
Implemented the single persisted retry-notice lifecycle. The orchestrator
updates the newest exact task and session match, replaces retry metadata as a
unit, removes legacy duplicates after a successful update, and leaves failed
storage operations eligible for later cleanup. Per-session serialization and a
retirement fence prevent cancellation or terminal cleanup from being undone by
late provider events. The WebSocket update path upserts a reused retry row when
the row is outside the loaded transcript window, and the desktop and mobile E2E
suites verify one current notice through advancement, reload, and Cancel.
The review remediation adds reference-counted guard ownership and a bounded
five-minute retirement fence. Active prompt/retry state remains owned, normal
successful-turn cleanup releases it, and deletion/terminal cleanup keeps only
the short fence needed to reject late provider events. New prompt evidence
opens that fence only after its execution identity is complete. Retry entries
are reserved before completion but armed after the failed turn is completed and
the session is parked. Deterministic backend tests cover session churn,
deletion, concurrent guard users, cancellation and terminal late failures,
stale-event identity fencing, new-prompt fence reset, and concurrent failures
that share one notice and timer entry; frontend tests cover a reused row missing
from the loaded transcript.
- `go test ./internal/orchestrator -run 'Test.*Transient.*' -count=1`: passed.
- `go test -race ./internal/orchestrator -run 'Test.*Transient.*' -count=1`:
passed.
- `go test ./internal/orchestrator -count=1`: passed.
- `make build`: passed.
- `make lint`: passed with 0 issues.
- Changed-file Go lint with `--new-from-rev`: passed with 0 issues.
- Focused Vitest: 50 tests passed in 3 files.
- Web lint: passed with 0 issues.
- `pnpm run typecheck`: passed.
- Managed Chromium E2E: 4 passed.
- Managed mobile-chrome E2E: 1 passed.
- `python3 scripts/list-docs.py validate`: passed, 275 decisions and 947
specifications.
- `python3 scripts/lint-spec-files.py --all`: passed.
- `git diff --check`: passed.
Targeted lint for the changed E2E files passed. The repository-wide
`pnpm run lint:e2e-sleeps` command still reports pre-existing violations in
untouched files.
CI fixup also removed two unrelated E2E blockers exposed by the first PR run.
The workflow settings test now dismisses the shared picker explicitly after a
model selection, because dependent options can keep it open. The LSP file
opening helper now uses exact file search when virtualization leaves a valid
file outside the mounted tree rows. The archive cleanup regression seeds 48
preceding files so that path is exercised deterministically.
- Workflow settings focused Chromium E2E with retries disabled and
`--repeat-each=3`: 3 passed.
- LSP file intelligence focused archive cleanup E2E with retries disabled: 1
passed with the virtualized-tree regression fixture.
- Full LSP file intelligence E2E with retries disabled: 13 passed.
- Targeted ESLint for the three changed E2E files: passed.
- Web TypeScript typecheck: passed.
The subsequent CI blob audit (run `35032213664`) found 13 hidden retry
attempts across 12 tests. The follow-up fixup removed those timing, geometry,
selector, and setup races: mobile branch refresh waits for settled controls and
uses a forceful touch activation, layout checks poll or use their intended
one-pixel tolerance, delayed provider and virtualized content use causal
bounds, and the settings interlock retries only the backend's startup `503`.
The parked-work suite now creates its temporary `claude-acp` profile after
per-test profile cleanup, so a stale deleted profile cannot suppress session
creation.
- Interlock helper Vitest regression: passed.
- Mobile CI regression batch with retries disabled and `--repeat-each=3`: 9
passed.
- Chromium/tablet regression batch with retries disabled and
`--repeat-each=3`: 15 passed.
- Git plus parked-work sequence with retries disabled: 2 passed.
- Parked-work regression with retries disabled and `--repeat-each=3`: 3
passed; full parked-work file: 2 passed.
docs/specs/platform/requirements/provider-error-recovery.md
+3 -1
View File
@@ -2,7 +2,7 @@
status: draft
system: platform
created: 2026-08-08
updated: 2026-09-11
updated: 2026-09-15
owners:
- Kandev
---
@@ -46,6 +46,8 @@ Agent CLIs and providers report equivalent failures through different ACP frames
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.23:** A recorded diagnostic shall authorize pre-result recovery only when its normalized text is contained in the normalized message of the terminal provider failure, in addition to both classifying to the same high-confidence, fallback-eligible semantic code. Normalization shall trim leading and trailing whitespace and collapse internal whitespace runs to a single space, and comparison shall be case-sensitive. An empty normalized diagnostic shall never satisfy containment. The normalized diagnostic text shall be retained with the recorded diagnostic code, shall be cleared whenever that code is cleared, and shall not be overwritten by a later diagnostic in the same generation. A diagnostic that classifies to the terminal code but whose normalized text is not contained in the terminal message shall keep the output fence and expose manual recovery.
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.24:** When a terminal ACP prompt error carries its transient signature only in `RequestError.Data`, Kandev shall present the failure as terminal and expose manual recovery on every occurrence, including repeated failures in the same session, and shall neither show a transient retry notice nor start an automatic retry. This criterion applies to the terminal failure whether or not a diagnostic preceded it, and it does not outrank acceptance criteria `.16` and `.23`: when a diagnostic did precede, those criteria reach the same manual-recovery outcome, because a terminal message that classifies to nothing can match no recorded diagnostic code. The existing session end-to-end specification that drives the mock agent's `/transport-lost` command shall be the surface that observes this outcome: its automatic-retry assertions shall be retargeted to manual recovery, and its retry-cancellation sub-test shall be retired rather than rewritten, because no retry loop exists on this path to cancel. The mock command's envelope shall keep the SDK's own peer-disconnect shape, `Message` `Internal error` with the signature only in `Data`, so the specification keeps exercising a shape that occurs in production. Repository documentation describing that command, or the recovery timeout it drives, shall not state that this path enters the automatic retry ladder. Losing end-to-end coverage of the automatic retry ladder for the transport-loss cause is accepted in this version; the catalogue rule keeps its unit coverage and its remaining surfaces.
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25:** During one interactive transient retry loop, desktop and mobile chat shall show at most one current retry banner. Each subsequent attempt shall update that banner's reason, provider, attempt count, deadline, and Cancel action. Reloading the session shall retain the latest persisted attempt. Existing duplicate retry notices shall consolidate on the next successful retry-notice write. Unrelated messages and other sessions shall remain unchanged. Storage failures shall remain non-fatal and shall not trigger creation of an additional notice when an existing notice cannot be read or updated.
## Out of scope
- Classifier rules keyed on the structured error kind or JSON-RPC code. The metadata is transported and observable in this version; making it outrank the message-text catalogue is a separate contract change, because it would let a gateway authorize dynamic recovery through a field no fixture yet constrains.
docs/specs/platform/system-design/provider-error-recovery.md
+39 -6
View File
@@ -4,7 +4,7 @@ system: platform
requirements:
- REQ-PLATFORM-PROVIDER-ERROR-RECOVERY-001
created: 2026-08-08
updated: 2026-09-03
updated: 2026-09-15
owners:
- Kandev
---
@@ -298,12 +298,43 @@ not write legacy rule shapes.
### Interactive transient retry notice lifecycle
Concrete-profile task chat persists one status message for each scheduled
retry attempt. The message metadata contains `retrying: true`; its visible
Concrete-profile task chat reuses one status message across scheduled
retry attempts (AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.25). The metadata contains `retrying: true`; its visible
content includes the safe reason, provider, attempt ordinal, absolute deadline,
and Cancel action. The backend timer owns the retry. The persisted message is a
transcript projection of that ownership, not an independent retry state.
The proposed attempt-write path lists notices through `TransientRetryMessageService`.
It selects status messages for the exact task and session with boolean
`retrying: true`. With no match, it uses `CreateSessionMessage` once.
With matches, it keeps the newest message by creation time, then ID, so legacy
duplicates leave the active row inside the newest hydration window.
It replaces that message's retry content and metadata through task-service
`UpdateMessage`. Message ID, creation time, and original turn association remain stable.
The existing `session.message.updated` path updates the frontend store by ID. If
a reused row is outside the loaded window, it upserts the retry status update.
After a successful update, the writer removes other matching notices through
`DeleteMessage`. This also repairs duplicates from earlier versions.
List or update errors are logged and swallowed without a fallback insert.
Duplicate deletion errors remain eligible for the next write or terminal cleanup.
No schema migration or separate frontend retry store is required.
Writes and terminal cleanup must preserve session event ordering: a cancelled
or superseded attempt cannot recreate a retired notice. Database operations
must remain outside `taskRuntimeStateMu`.
The process-local notice lifecycle uses one reference-counted guard per session.
Short operations increment its reference count before taking the guard, so a
waiting caller cannot observe a replacement mutex. The entry remains owned
while an accepted prompt or retry exists. Retirement clears that ownership and
arms a five-minute fence for late provider events. A timer reclaims the entry
only after all guard users have released it and the lifecycle is no longer
owned. Prompt evidence uses the guard and opens the fence only after a complete
execution identity. Reserve retry entries under the guard; arm them after failed
turns complete and sessions enter `WAITING_FOR_INPUT`. Deletion performs the
same retirement before removing the row. Tests cover events, fence reset,
concurrency, churn, deletion, and guards.
The orchestrator attempts to resolve every outstanding transient-retry status
message for a session whenever retry ownership ends through success, exhaustion,
a terminal event, coordinator stop, explicit cancellation, an ordinary session
@@ -327,9 +358,11 @@ warning, preserving stale retry text and its Cancel action. The retry attempts
remain observable through agent output and recovery history without retaining
an actionable status projection after ownership ends.
Advancing from attempt N to attempt N+1 only cancels the superseded in-memory
timer. It does not run the retry-ending reset and therefore does not retire the
current attempt's notice prematurely. Durable message operations do not run
Advancing from attempt N to attempt N+1 cancels the superseded in-memory
timer, updates the existing notice, and arms the replacement after its failed
turn completes and the session is parked. It does not run the
retry-ending reset.
Durable message operations do not run
while the orchestrator's runtime-state mutex is held.
`CancelTransientRetry` authorizes the task-session pair before reading or