fix: complete E2E cleanup and quick-chat stabilization (#3102)

This commit is contained in:
Carlos Florêncio
2026-08-28 09:39:01 +00:00
committed by GitHub
parent bf84fb52bd
commit 311ba3ce72
8 changed files with 323 additions and 71 deletions
+8 -10
View File
@@ -264,6 +264,10 @@ func handleE2EReset(
}
var deletedTasks int64
deletedTaskIDs := append([]string(nil), taskIDsForCleanup...)
deletedTaskIDSet := make(map[string]struct{}, len(deletedTaskIDs))
for _, taskID := range deletedTaskIDs {
deletedTaskIDSet[taskID] = struct{}{}
}
for _, t := range tasks {
if err := taskSvc.DeleteTask(ctx, t.ID); err != nil {
// Abort: leaving an undeleted task with its workflow gone
@@ -274,7 +278,8 @@ func handleE2EReset(
return
}
deletedTasks++
if !containsE2ETaskID(deletedTaskIDs, t.ID) {
if _, exists := deletedTaskIDSet[t.ID]; !exists {
deletedTaskIDSet[t.ID] = struct{}{}
deletedTaskIDs = append(deletedTaskIDs, t.ID)
}
}
@@ -283,6 +288,8 @@ func handleE2EReset(
// removes worktrees asynchronously. The next test reuses the worker's
// repository, so returning before those jobs finish lets an old cleanup
// race with the next test's repository setup and file-tree read.
// This wait covers the durable cleanup path. E2E startup wires
// resourceCleanups, so the fire-and-forget fallback is not expected here.
if err := waitForE2ETaskCleanup(ctx, repo.DB(), deletedTaskIDs); err != nil {
log.Error("e2e reset: task cleanup did not finish", zap.Error(err))
c.JSON(http.StatusInternalServerError, gin.H{errKey: err.Error()})
@@ -454,15 +461,6 @@ func listE2ETaskIDs(ctx context.Context, database *sql.DB, workspaceID string) (
return ids, nil
}
func containsE2ETaskID(taskIDs []string, taskID string) bool {
for _, id := range taskIDs {
if id == taskID {
return true
}
}
return false
}
func resetGitHubAppRegistrationsForE2E(
ctx context.Context,
service *github.Service,
@@ -6,6 +6,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
@@ -69,8 +70,15 @@ func TestWaitForE2ETaskCleanupWithReader(t *testing.T) {
firstRead := make(chan struct{})
releaseFirstRead := make(chan struct{})
result := make(chan error, 1)
done := make(chan struct{})
var releaseOnce sync.Once
t.Cleanup(func() {
releaseOnce.Do(func() { close(releaseFirstRead) })
<-done
})
readCount := 0
go func() {
defer close(done)
result <- waitForE2ETaskCleanupWithReader(
context.Background(),
[]string{"task-1"},
@@ -91,7 +99,7 @@ func TestWaitForE2ETaskCleanupWithReader(t *testing.T) {
}()
<-firstRead
close(releaseFirstRead)
releaseOnce.Do(func() { close(releaseFirstRead) })
if err := <-result; err != nil {
t.Fatalf("waitForE2ETaskCleanupWithReader: %v", err)
}
@@ -189,9 +197,18 @@ func TestListE2ETaskIDsIncludesAutomationOwnedTasks(t *testing.T) {
if err != nil {
t.Fatalf("listE2ETaskIDs: %v", err)
}
if len(ids) != 2 || !containsE2ETaskID(ids, "ordinary-task") || !containsE2ETaskID(ids, "automation-task") {
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
seen[id] = struct{}{}
}
if len(ids) != 2 {
t.Fatalf("task IDs = %v, want both ws-1 tasks", ids)
}
for _, id := range []string{"ordinary-task", "automation-task"} {
if _, ok := seen[id]; !ok {
t.Fatalf("task IDs = %v, missing %s", ids, id)
}
}
}
func assertWorkspaceRows(t *testing.T, database *sqlx.DB, table, workspaceID string, want int) {
@@ -299,14 +299,12 @@ func (s *Service) preparedTaskCleanupMutationCommitted(
return false, err
}
taskExists := err == nil && task != nil
if taskResourceCleanupDeletesTask(job.Trigger) {
return !taskExists, nil
}
switch job.Trigger {
case models.TaskResourceCleanupTriggerArchive, models.TaskResourceCleanupTriggerCascadeArchive:
return taskExists && task.ArchivedAt != nil, nil
case models.TaskResourceCleanupTriggerDelete,
models.TaskResourceCleanupTriggerCascadeDelete,
models.TaskResourceCleanupTriggerWorkspaceDelete,
models.TaskResourceCleanupTriggerQuickChatExpire:
return !taskExists, nil
default:
return false, nil
}
@@ -429,7 +427,14 @@ func (s *Service) executeTaskResourceCleanupJob(
return fmt.Errorf("refresh task cleanup runtime inventory: %w", err)
}
s.registerTaskRuntimeStopOwners(targets, true)
stopOutcome := s.stopTaskRuntimeTargets(ctx, job.TaskID, targets, taskResourceCleanupStopReason(job.Trigger), "task cleanup runtime stop failed")
stopOutcome := s.stopTaskRuntimeTargetsWithTaskDeleted(
ctx,
job.TaskID,
targets,
taskResourceCleanupStopReason(job.Trigger),
"task cleanup runtime stop failed",
taskResourceCleanupDeletesTask(job.Trigger),
)
failedStops := stopOutcome.failed
if cancelled, err := s.cancelIfTaskUnarchived(ctx, job); err != nil || cancelled {
return err
@@ -468,6 +473,18 @@ func (s *Service) hasLegacyWorktreeCleanup() bool {
return !isProvider
}
func taskResourceCleanupDeletesTask(trigger models.TaskResourceCleanupTrigger) bool {
switch trigger {
case models.TaskResourceCleanupTriggerDelete,
models.TaskResourceCleanupTriggerCascadeDelete,
models.TaskResourceCleanupTriggerWorkspaceDelete,
models.TaskResourceCleanupTriggerQuickChatExpire:
return true
default:
return false
}
}
func taskResourceCleanupStopReason(trigger models.TaskResourceCleanupTrigger) string {
switch trigger {
case models.TaskResourceCleanupTriggerArchive:
@@ -452,7 +452,7 @@ func (s *Service) runWorkspaceDeleteTaskCleanup(cleanup workspaceDeleteTaskClean
return
}
envCleanup := taskEnvironmentCleanup{env: cleanup.taskEnv, deleteRow: false}
s.runTaskCleanup(cleanup.task.ID, cleanup.sessions, cleanup.worktrees, cleanup.stopTargets, envCleanup,
s.runTaskCleanup(cleanup.task.ID, cleanup.sessions, cleanup.worktrees, cleanup.stopTargets, envCleanup, true,
"task deleted", "failed to stop session on task delete", "task cleanup completed")
}
@@ -2094,7 +2094,7 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error {
zap.String("job_id", cleanupJob.ID), zap.String("task_id", id), zap.Error(err))
}
} else if len(stopTargets) > 0 || s.worktreeCleanup != nil || len(sessions) > 0 || taskEnv != nil {
s.runAsyncTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup,
s.runAsyncTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup, false,
"task archived", "failed to stop session on task archive", "task archive cleanup completed")
}
@@ -2327,7 +2327,7 @@ func (s *Service) deleteTaskWithReasonAndDBDelete(
zap.String("job_id", cleanupJob.ID), zap.String("task_id", id), zap.Error(err))
}
} else if hasCleanup {
s.runAsyncTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup,
s.runAsyncTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup, true,
"task deleted", "failed to stop session on task delete", "task cleanup completed")
}
@@ -2367,6 +2367,7 @@ func (s *Service) deleteTaskStopTargets(ctx context.Context, id string) ([]taskS
// for delete cascade, false for archive — archive preserves the row). Runtime
// inventory failures abort cleanup so durable stop handles remain retryable.
func (s *Service) CleanupTaskResources(ctx context.Context, taskID string, deleteEnvRow bool) {
taskDeleted := deleteEnvRow
if deleteEnvRow && s.attachmentSvc != nil {
if err := s.attachmentSvc.DeleteByTask(context.WithoutCancel(ctx), taskID); err != nil {
s.logger.Warn("failed to remove task attachment bytes during resource cleanup",
@@ -2426,7 +2427,7 @@ func (s *Service) CleanupTaskResources(ctx context.Context, taskID string, delet
if deleteEnvRow {
reason = "cascade delete"
}
s.runAsyncTaskCleanup(taskID, sessions, worktrees, stopTargets, envCleanup,
s.runAsyncTaskCleanup(taskID, sessions, worktrees, stopTargets, envCleanup, taskDeleted,
reason, "failed to stop session on cascade cleanup", "cascade cleanup completed")
}
@@ -2473,9 +2474,10 @@ func (s *Service) runAsyncTaskCleanup(
worktrees []*worktree.Worktree,
stopTargets []taskStopTarget,
envCleanup taskEnvironmentCleanup,
taskDeleted bool,
stopReason, stopFailMsg, cleanupMsg string,
) {
go s.runTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup, stopReason, stopFailMsg, cleanupMsg)
go s.runTaskCleanup(id, sessions, worktrees, stopTargets, envCleanup, taskDeleted, stopReason, stopFailMsg, cleanupMsg)
}
func (s *Service) runTaskCleanup(
@@ -2484,6 +2486,7 @@ func (s *Service) runTaskCleanup(
worktrees []*worktree.Worktree,
stopTargets []taskStopTarget,
envCleanup taskEnvironmentCleanup,
taskDeleted bool,
stopReason, stopFailMsg, cleanupMsg string,
) {
cleanupStart := time.Now()
@@ -2500,7 +2503,7 @@ func (s *Service) runTaskCleanup(
stopTargets = refreshedTargets
s.registerTaskRuntimeStopOwners(stopTargets, true)
stopOutcome := s.stopTaskRuntimeTargets(cleanupCtx, id, stopTargets, stopReason, stopFailMsg)
stopOutcome := s.stopTaskRuntimeTargetsWithTaskDeleted(cleanupCtx, id, stopTargets, stopReason, stopFailMsg, taskDeleted)
cleanupErrors := s.performTaskCleanup(cleanupCtx, id, sessions, worktrees, stopTargets, envCleanup,
taskCleanupPreserveRows(stopOutcome))
@@ -2704,6 +2707,16 @@ func taskCleanupPreserveRows(outcome taskRuntimeStopOutcome) map[string]struct{}
}
func (s *Service) stopTaskRuntimeTargets(ctx context.Context, taskID string, stopTargets []taskStopTarget, stopReason, stopFailMsg string) taskRuntimeStopOutcome {
return s.stopTaskRuntimeTargetsWithTaskDeleted(ctx, taskID, stopTargets, stopReason, stopFailMsg, false)
}
func (s *Service) stopTaskRuntimeTargetsWithTaskDeleted(
ctx context.Context,
taskID string,
stopTargets []taskStopTarget,
stopReason, stopFailMsg string,
taskDeleted bool,
) taskRuntimeStopOutcome {
outcome := taskRuntimeStopOutcome{
failed: make(map[string]struct{}),
preserve: make(map[string]struct{}),
@@ -2715,56 +2728,156 @@ func (s *Service) stopTaskRuntimeTargets(ctx context.Context, taskID string, sto
if context.Cause(ctx) != nil {
return outcome
}
if target.executionID != "" {
if err := s.executionStopper.StopExecution(ctx, target.executionID, stopReason, true); err != nil {
if runtimeStopAlreadyComplete(err) {
continue
}
if target.terminal {
s.logger.Debug("stop failed for terminal session execution (expected), proceeding with cleanup",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
continue
}
outcome.failed[target.sessionID] = struct{}{}
s.logger.Warn(stopFailMsg,
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.String("execution_id", target.executionID),
zap.Error(err))
}
continue
}
if err := s.executionStopper.StopSession(ctx, target.sessionID, stopReason, true); err != nil {
if target.terminal {
s.logger.Debug("stop failed for terminal session (expected), proceeding with cleanup",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
continue
}
// A session-level not-found is retryable by default (the execution may
// simply not be registered yet). But when the owned row is a
// confirmed-dead LOCAL runtime, the runtime really is gone — treat the
// stop as already complete so the durable cleanup job stops retrying a
// runtime that will never come back.
if runtimeStopAlreadyComplete(err) {
if running := s.confirmedDeadLocalRow(ctx, target.sessionID); running != nil {
s.reconcileConfirmedDeadRow(ctx, taskID, target.sessionID, running, &outcome)
continue
}
}
outcome.failed[target.sessionID] = struct{}{}
s.logger.Warn(stopFailMsg,
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
}
s.stopTaskRuntimeTarget(ctx, taskID, target, stopReason, stopFailMsg, taskDeleted, &outcome)
}
return outcome
}
func (s *Service) stopTaskRuntimeTarget(
ctx context.Context,
taskID string,
target taskStopTarget,
stopReason, stopFailMsg string,
taskDeleted bool,
outcome *taskRuntimeStopOutcome,
) {
if target.executionID != "" {
s.stopTaskRuntimeExecution(ctx, taskID, target, stopReason, stopFailMsg, outcome)
return
}
s.stopTaskRuntimeSession(ctx, taskID, target, stopReason, stopFailMsg, taskDeleted, outcome)
}
func (s *Service) stopTaskRuntimeExecution(
ctx context.Context,
taskID string,
target taskStopTarget,
stopReason, stopFailMsg string,
outcome *taskRuntimeStopOutcome,
) {
err := s.executionStopper.StopExecution(ctx, target.executionID, stopReason, true)
if err == nil || runtimeStopAlreadyComplete(err) {
return
}
if target.terminal {
s.logger.Debug("stop failed for terminal session execution (expected), proceeding with cleanup",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
return
}
outcome.failed[target.sessionID] = struct{}{}
s.logger.Warn(stopFailMsg,
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.String("execution_id", target.executionID),
zap.Error(err))
}
func (s *Service) stopTaskRuntimeSession(
ctx context.Context,
taskID string,
target taskStopTarget,
stopReason, stopFailMsg string,
taskDeleted bool,
outcome *taskRuntimeStopOutcome,
) {
err := s.executionStopper.StopSession(ctx, target.sessionID, stopReason, true)
if err == nil {
return
}
if target.terminal {
s.logger.Debug("stop failed for terminal session (expected), proceeding with cleanup",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
return
}
// A session-level not-found is retryable by default (the execution may
// simply not be registered yet). But when the owned row is a
// confirmed-dead LOCAL runtime, the runtime really is gone — treat the
// stop as already complete so the durable cleanup job stops retrying a
// runtime that will never come back.
if runtimeStopAlreadyComplete(err) {
if taskDeleted && s.stopDeletedSessionRuntime(ctx, taskID, target, stopReason) {
return
}
if running := s.confirmedDeadLocalRow(ctx, target.sessionID); running != nil {
s.reconcileConfirmedDeadRow(ctx, taskID, target.sessionID, running, outcome)
return
}
}
outcome.failed[target.sessionID] = struct{}{}
s.logger.Warn(stopFailMsg,
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
}
// stopDeletedSessionRuntime handles a session stop that raced with task deletion.
// A missing session is complete only when no executor row remains. If a late row
// has an exact execution ID, stop that execution before cleanup removes the row.
// When the row has no execution ID, return false so the caller can apply the
// confirmed-dead local probe before marking the session for retry.
func (s *Service) stopDeletedSessionRuntime(
ctx context.Context,
taskID string,
target taskStopTarget,
stopReason string,
) bool {
if s.executors == nil || s.executionStopper == nil {
return false
}
running, err := s.executors.GetExecutorRunningBySessionID(ctx, target.sessionID)
if err != nil {
if errors.Is(err, models.ErrExecutorRunningNotFound) {
s.logger.Debug("deleted task session has no registered runtime",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID))
return true
}
s.logger.Warn("failed to inspect deleted task session runtime",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.Error(err))
return false
}
if running == nil {
return true
}
executionID := strings.TrimSpace(running.AgentExecutionID)
if executionID == "" {
return false
}
s.executionStopper.RegisterExecutionStopOwner(target.sessionID, executionID, true)
if err := s.executionStopper.StopExecution(ctx, executionID, stopReason, true); err != nil {
if runtimeStopAlreadyComplete(err) {
s.logger.Debug("late deleted task execution was already stopped",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.String("execution_id", executionID),
zap.Error(err))
return true
}
if target.terminal {
s.logger.Debug("stop failed for late terminal task execution (expected), proceeding with cleanup",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.String("execution_id", executionID),
zap.Error(err))
return true
}
s.logger.Warn("failed to stop late deleted task execution",
zap.String("task_id", taskID),
zap.String("session_id", target.sessionID),
zap.String("execution_id", executionID),
zap.Error(err))
return false
}
return true
}
// reconcileConfirmedDeadRow applies the resume-safety deletion invariant to a
// confirmed-dead local row whose stop reported not-found. A row holding a
// resume_token or backing a resumable session is repaired in place (never
@@ -229,6 +229,7 @@ type stubStopper struct {
stopExecutionErr error
stopSessionCalls int
stopExecutionCalls int
onStopSession func()
}
func (s *stubStopper) StopTask(_ context.Context, _, _ string, _ bool) error { return nil }
@@ -239,6 +240,9 @@ func (s *stubStopper) StopExecution(_ context.Context, _, _ string, _ bool) erro
func (s *stubStopper) RegisterExecutionStopOwner(string, string, bool) {}
func (s *stubStopper) StopSession(_ context.Context, _, _ string, _ bool) error {
s.stopSessionCalls++
if s.onStopSession != nil {
s.onStopSession()
}
return s.stopSessionErr
}
@@ -320,6 +324,107 @@ func TestStopTaskRuntimeTargets_SessionRuntimeAbsenceRemainsRetryable(t *testing
}
}
func TestStopTaskRuntimeTargetsAfterTaskDeletionWithoutRuntimeIsComplete(t *testing.T) {
svc, _, _ := createTestService(t)
svc.executors = &stubExecutors{}
stopper := &stubStopper{
stopSessionErr: fmt.Errorf("session deleted: %w", runtimeapi.ErrNotFound),
}
svc.executionStopper = stopper
outcome := svc.stopTaskRuntimeTargetsWithTaskDeleted(
context.Background(),
"task-deleted",
[]taskStopTarget{{sessionID: "sess-deleted"}},
"task deleted",
"stop failed",
true,
)
if len(outcome.failed) != 0 {
t.Fatalf("deleted task with no runtime must not retry cleanup: %v", outcome.failed)
}
if stopper.stopSessionCalls != 1 {
t.Fatalf("stop session calls = %d, want 1", stopper.stopSessionCalls)
}
}
func TestStopTaskRuntimeTargetsAfterTaskDeletionKeepsUnidentifiedRuntimeRetryable(t *testing.T) {
svc, _, _ := createTestService(t)
svc.executors = &stubExecutors{
runningBySession: &models.ExecutorRunning{SessionID: "sess-unknown"},
}
svc.executionStopper = &stubStopper{
stopSessionErr: fmt.Errorf("session deleted: %w", runtimeapi.ErrNotFound),
}
outcome := svc.stopTaskRuntimeTargetsWithTaskDeleted(
context.Background(),
"task-deleted",
[]taskStopTarget{{sessionID: "sess-unknown"}},
"task deleted",
"stop failed",
true,
)
if _, ok := outcome.failed["sess-unknown"]; !ok {
t.Fatalf("unidentified runtime row must remain retryable: %v", outcome.failed)
}
}
func TestStopTaskRuntimeTargetsAfterTaskDeletionStopsLateExactRuntime(t *testing.T) {
svc, _, _ := createTestService(t)
execs := &stubExecutors{}
stopper := &stubStopper{
stopSessionErr: fmt.Errorf("session deleted: %w", runtimeapi.ErrNotFound),
onStopSession: func() {
execs.runningBySession = &models.ExecutorRunning{
SessionID: "sess-late",
AgentExecutionID: "exec-late",
}
},
}
svc.executors = execs
svc.executionStopper = stopper
outcome := svc.stopTaskRuntimeTargetsWithTaskDeleted(
context.Background(),
"task-deleted",
[]taskStopTarget{{sessionID: "sess-late"}},
"task deleted",
"stop failed",
true,
)
if len(outcome.failed) != 0 {
t.Fatalf("late exact runtime should be stopped, not retried: %v", outcome.failed)
}
if stopper.stopExecutionCalls != 1 {
t.Fatalf("stop execution calls = %d, want 1", stopper.stopExecutionCalls)
}
}
func TestTaskResourceCleanupDeletesTask(t *testing.T) {
tests := []struct {
trigger models.TaskResourceCleanupTrigger
want bool
}{
{models.TaskResourceCleanupTriggerArchive, false},
{models.TaskResourceCleanupTriggerDelete, true},
{models.TaskResourceCleanupTriggerCascadeArchive, false},
{models.TaskResourceCleanupTriggerCascadeDelete, true},
{models.TaskResourceCleanupTriggerWorkspaceDelete, true},
{models.TaskResourceCleanupTriggerQuickChatExpire, true},
{models.TaskResourceCleanupTriggerReconcile, false},
{models.TaskResourceCleanupTrigger("unknown"), false},
}
for _, tt := range tests {
t.Run(string(tt.trigger), func(t *testing.T) {
if got := taskResourceCleanupDeletesTask(tt.trigger); got != tt.want {
t.Fatalf("taskResourceCleanupDeletesTask(%q) = %t, want %t", tt.trigger, got, tt.want)
}
})
}
}
// stubRowLivenessProber is a minimal TaskRowLivenessProber for tests.
type stubRowLivenessProber struct {
liveness models.ProcessLiveness
+1
View File
@@ -498,6 +498,7 @@ test.beforeEach(async ({ apiClient, seedData }) => {
lsp_auto_install_languages: [],
lsp_server_configs: {},
kanban_view_mode: "",
tasks_list_show_details: false,
startup_page: "task_overview",
show_anchored_prompt_bar: false,
show_scroll_to_last_prompt: true,
@@ -34,7 +34,8 @@ test.describe("Tool completion on turn end", () => {
// The composer can become idle before the final tool-state reconciliation
// reaches the rendered message. Give that bounded update its own wait.
const spinners = session.chat.locator('[role="status"][aria-label="Loading"]');
const toolGroups = session.chat.getByRole("button", { name: /Terminal/ }).locator("..");
const spinners = toolGroups.locator('[role="status"][aria-label="Loading"]');
await expect(spinners).toHaveCount(0, { timeout: 30_000 });
});
});