From 9514ab28e4e0831e0e09c1e4523fd50738cfa1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Salavisa?= Date: Fri, 7 Aug 2026 20:48:31 +0200 Subject: [PATCH] fix(tasks): clear the error icon once the agent recovers (#2391) Co-authored-by: Kandev Agent Co-authored-by: Claude Opus 5 (1M context) --- .../orchestrator/event_handlers_agent.go | 56 ++++++ .../event_handlers_streaming_test.go | 73 ++++++++ apps/backend/internal/task/models/models.go | 4 +- .../task/repository/sqlite/base_migrations.go | 107 +++++++++++ .../repository/sqlite/postgres_schema_test.go | 109 ++++++++++++ .../task/repository/sqlite/session_test.go | 98 ++++++++++ .../service_status_summary_rebuild_test.go | 47 +++++ .../internal/task/statussummary/projector.go | 64 +++++-- .../task/statussummary/projector_helpers.go | 42 ++++- .../task/statussummary/projector_test.go | 168 ++++++++++++++++++ .../internal/task/statussummary/rebuild.go | 21 +-- 11 files changed, 755 insertions(+), 34 deletions(-) diff --git a/apps/backend/internal/orchestrator/event_handlers_agent.go b/apps/backend/internal/orchestrator/event_handlers_agent.go index 913e07e09b..b00b568fe6 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent.go @@ -1054,6 +1054,10 @@ func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.A s.scheduler.HandleTaskCompleted(data.TaskID, true) s.scheduler.RemoveTask(data.TaskID) + // The agent finished a turn, so any stored failure no longer describes the + // session. `session` was read above, so the guard costs nothing. + s.clearRecoveredAgentError(context.WithoutCancel(ctx), data.TaskID, session) + s.completeTurnForSession(context.WithoutCancel(ctx), data.SessionID) if s.sessionHasPendingClarification(ctx, data.SessionID) { @@ -1503,6 +1507,58 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE } } +// clearRecoveredAgentError drops a session's stored agent failure once the agent +// completes a turn, and publishes the inactive error event so open clients drop +// the red error affordance. +// +// persistLastAgentError deliberately keeps the record across a successful turn +// as an investigation breadcrumb, but nothing ever retired it, so a failure the +// agent recovered from weeks ago still read as live: every path that re-derives +// task status from session metadata (a status-summary rebuild, a backend +// restart, a later session event) put it straight back. The failure also lands +// in the transcript as a recovery message, which is where an investigation +// actually looks, so the metadata copy is not the durable record. +// +// Writes JSON null rather than a delete: LoadLastAgentError already treats that +// as absent, so no new repository surface is needed. +func (s *Service) clearRecoveredAgentError(ctx context.Context, taskID string, session *models.TaskSession) { + if session == nil || session.ID == "" { + return + } + if _, ok := models.LoadLastAgentError(session.Metadata); !ok { + return + } + if err := s.repo.SetSessionMetadataKey( + ctx, session.ID, models.SessionMetaKeyLastAgentError, nil, + ); err != nil { + s.logger.Warn("failed to clear recovered agent error", + zap.String("task_id", taskID), + zap.String("session_id", session.ID), + zap.Error(err)) + return + } + // Keep the in-memory copy in step: the session-state publish below reads + // its `session_metadata` straight off this object. + delete(session.Metadata, models.SessionMetaKeyLastAgentError) + if s.eventBus == nil { + return + } + if err := s.eventBus.Publish(ctx, events.TaskSessionErrorChanged, bus.NewEvent( + events.TaskSessionErrorChanged, + "orchestrator", + map[string]interface{}{ + "task_id": taskID, + "session_id": session.ID, + "active": false, + }, + )); err != nil { + s.logger.Warn("failed to publish recovered task session error event", + zap.String("task_id", taskID), + zap.String("session_id", session.ID), + zap.Error(err)) + } +} + // createRecoveryStatusMessage builds and persists the ActionMessage shown // in the kanban chat surface after a recoverable agent failure. Must only // be called for non-office sessions (office sessions render their own error UI). diff --git a/apps/backend/internal/orchestrator/event_handlers_streaming_test.go b/apps/backend/internal/orchestrator/event_handlers_streaming_test.go index 3600b24b18..35c6e62fbe 100644 --- a/apps/backend/internal/orchestrator/event_handlers_streaming_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_streaming_test.go @@ -2191,6 +2191,79 @@ func TestSessionStartClearsInterruptedMarker(t *testing.T) { }) } +// Nothing else ever retires a stored agent failure, so without this clear every +// path that re-derives task status from session metadata resurrects it and the +// error icon never goes away. +func TestClearRecoveredAgentErrorOnTurnCompletion(t *testing.T) { + ctx := context.Background() + + seedError := func(t *testing.T, repo *sqliterepo.Repository) { + t.Helper() + require.NoError(t, repo.SetSessionMetadataKey( + ctx, "s1", models.SessionMetaKeyLastAgentError, + models.LastAgentError{Message: "agent crashed", OccurredAt: time.Now().UTC().Add(-time.Hour)}, + )) + } + + errorEvents := func(eb *recordingEventBus) []recordedEvent { + var out []recordedEvent + for _, recorded := range eb.events { + if recorded.event != nil && recorded.event.Type == events.TaskSessionErrorChanged { + out = append(out, recorded) + } + } + return out + } + + newService := func(repo *sqliterepo.Repository) (*Service, *recordingEventBus) { + eb := &recordingEventBus{} + svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo()) + svc.eventBus = eb + return svc, eb + } + + t.Run("clears the record and publishes it inactive", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + seedError(t, repo) + svc, eb := newService(repo) + + session, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + svc.clearRecoveredAgentError(ctx, "t1", session) + + stored, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + _, ok := models.LoadLastAgentError(stored.Metadata) + require.False(t, ok, "a recovered failure must not keep driving the error icon") + + // The session-state publish that follows reads session_metadata straight + // off this object, so a stale copy would re-arm the icon immediately. + _, ok = models.LoadLastAgentError(session.Metadata) + require.False(t, ok, "the in-memory copy must be cleared too") + + published := errorEvents(eb) + require.Len(t, published, 1, "clients need one event to drop the icon") + data, ok := published[0].event.Data.(map[string]interface{}) + require.True(t, ok) + require.Equal(t, false, data["active"]) + require.Equal(t, "s1", data["session_id"]) + }) + + t.Run("stays silent when the session has no stored error", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + svc, eb := newService(repo) + + session, err := repo.GetTaskSession(ctx, "s1") + require.NoError(t, err) + svc.clearRecoveredAgentError(ctx, "t1", session) + + require.Empty(t, errorEvents(eb), + "every later completion must stay silent so turns do not churn the row") + }) +} + func TestSetSessionStartingRejectsTerminalSession(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index c5a41ca09f..9d1044ebe1 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -662,8 +662,8 @@ const ( // Workflow represents a task workflow type Workflow struct { - ID string `json:"id"` - WorkspaceID string `json:"workspace_id"` + ID string `json:"id"` + WorkspaceID string `json:"workspace_id"` Name string `json:"name"` Description string `json:"description"` // Prompt is optional workflow-level agent instructions prepended at diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 2d85694fb9..57fedacc46 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -237,9 +237,116 @@ func (r *Repository) runMigrations() error { CREATE INDEX IF NOT EXISTS idx_task_status_summaries_workspace ON task_status_summaries(workspace_id)`) + if err := r.clearRecoveredAgentErrors(); err != nil { + return err + } + return nil } +// clearRecoveredAgentErrors repairs sessions whose stored agent failure was +// overtaken by successful work before the orchestrator learned to clear it. +// +// Nothing used to retire `last_agent_error`, so a failure the agent recovered +// from weeks ago still reads as live and keeps a red error icon on the task list +// forever. The orchestrator now clears the record on turn completion; this +// applies the same rule to history. +// +// Deliberately narrow: a record is cleared only when the session has an ordinary +// agent message newer than the failure — the same "the agent has produced good +// output since" signal. A failure with no successful work after it is current +// and is left alone. +func (r *Repository) clearRecoveredAgentErrors() error { + postgres := dialect.IsPostgres(r.db.DriverName()) + + sessionError := func(key string) string { + return jsonText(postgres, "s.metadata", "last_agent_error", key) + } + recoveredSessions := ` + SELECT s.id + FROM task_sessions s + WHERE ` + sessionError("message") + ` IS NOT NULL + AND EXISTS ( + SELECT 1 FROM task_session_messages m + WHERE m.task_session_id = s.id + AND m.author_type = 'agent' + AND m.type NOT IN ('error', 'status') + AND ` + timestampColumn(postgres, "m.created_at") + ` > ` + + timestampFromText(postgres, sessionError("occurred_at")) + ` + )` + + r.migrate.Apply("task_sessions.last_agent_error.recovered_cleanup", + `UPDATE task_sessions SET metadata = `+jsonRemoveKey(postgres, "metadata", "last_agent_error")+ + ` WHERE id IN (`+recoveredSessions+`)`) + + // The summary row caches the derived error, so clearing the session record + // alone would leave the icon up until that session next emitted an event. + // A cached error naming a session that has no stored failure is stale by + // definition, which also sweeps up any earlier drift. + r.migrate.Apply("task_status_summaries.active_error.recovered_cleanup", ` + UPDATE task_status_summaries + SET summary = `+jsonRemoveKey(postgres, "summary", "active_error")+` + WHERE `+jsonText(postgres, "summary", "active_error", "session_id")+` IN ( + SELECT s.id FROM task_sessions s + WHERE `+sessionError("message")+` IS NULL + )`) + return nil +} + +// jsonColumn makes a TEXT-typed JSON column safe to parse. Both dialects raise +// on an empty or malformed document — Postgres on the `::jsonb` cast, SQLite in +// `json_extract` — and these statements scan every row before filtering, so a +// single such row aborts the whole migration. The repository writes ” and +// 'null' for "no metadata", so both are normalized the way the rest of the +// package already does. +func jsonColumn(postgres bool, column string) string { + empty, open := "'{}'", "" + if postgres { + empty, open = "'{}'::jsonb", "::jsonb" + } + return "(CASE WHEN " + column + " IS NULL OR " + column + " = 'null' OR " + column + + " = '' THEN " + empty + " ELSE " + column + open + " END)" +} + +// jsonText extracts a nested JSON text value from a TEXT-typed JSON column. +func jsonText(postgres bool, column, parent, key string) string { + base := jsonColumn(postgres, column) + if postgres { + return "(" + base + " #>> '{" + parent + "," + key + "}')" + } + return "json_extract(" + base + ", '$." + parent + "." + key + "')" +} + +// timestampColumn normalizes a timestamp column for comparison. SQLite stores +// timestamps as text and would compare them lexically, so it needs Julian days; +// on Postgres the column is already a timestamp. +func timestampColumn(postgres bool, column string) string { + if postgres { + return "(" + column + ")::timestamptz" + } + return "julianday(" + column + ")" +} + +// timestampFromText normalizes a timestamp extracted from JSON so it compares +// against timestampColumn — the two carry different text shapes +// ('2026-08-01 10:00:00+00:00' vs RFC3339 '2026-08-01T10:00:00Z'). NULLIF keeps +// an empty stored timestamp from raising on the Postgres cast; it then compares +// as NULL, so the row simply does not match. +func timestampFromText(postgres bool, expression string) string { + if postgres { + return "(NULLIF(" + expression + ", '')::timestamptz)" + } + return "julianday(" + expression + ")" +} + +func jsonRemoveKey(postgres bool, column, key string) string { + base := jsonColumn(postgres, column) + if postgres { + return "(" + base + " - '" + key + "')::text" + } + return "json_remove(" + base + ", '$." + key + "')" +} + // ensureImproveKandevWorkflowTemplateUniqueness removes the broad index from // the initial implementation, reconciles legacy bootstrap duplicates, then // enforces uniqueness only for the two hidden workflows created by this flow. diff --git a/apps/backend/internal/task/repository/sqlite/postgres_schema_test.go b/apps/backend/internal/task/repository/sqlite/postgres_schema_test.go index 187231e328..92675e0877 100644 --- a/apps/backend/internal/task/repository/sqlite/postgres_schema_test.go +++ b/apps/backend/internal/task/repository/sqlite/postgres_schema_test.go @@ -696,3 +696,112 @@ func TestPostgresSetSessionPrimary_ConcurrentPromotionsLeaveExactlyOnePrimary(t t.Errorf("expected exactly 1 primary session after %d concurrent promotions, got %d — FOR UPDATE lock not serializing across connections", concurrency, primaryCount) } } + +// TestPostgresClearRecoveredAgentErrors is the Postgres counterpart to +// TestClearRecoveredAgentErrorsBackfill. clearRecoveredAgentErrors is built from +// three dialect-sensitive helpers (jsonColumn, jsonText/timestamp*, +// jsonRemoveKey), so ADR 0027 asks for env-gated Postgres behavior coverage — +// schema replay would not exercise the jsonb operators at all. +// +// migrate.Apply swallows SQL errors, so the assertions below are the only proof +// the statements ran: a dialect mistake leaves the recovered row untouched +// rather than returning an error. Skips unless KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresClearRecoveredAgentErrors(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + + occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC) + if _, err := db.Exec(db.Rebind(` + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), "task-pg-recovered", "ws-pg-recovered", "Task", occurredAt, occurredAt); err != nil { + t.Fatalf("seed task: %v", err) + } + + // recovered: agent output after the failure. current: none since. + // blank/nulled: rows the repository wrote as "no metadata" — both dialects + // raise on parsing those, and the statements scan every row before + // filtering, so an unguarded cast aborts the migration for everyone. + for _, sessionID := range []string{"pg-recovered", "pg-current", "pg-blank", "pg-nulled"} { + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: sessionID, TaskID: "task-pg-recovered", State: models.TaskSessionStateWaitingForInput, + }); err != nil { + t.Fatalf("CreateTaskSession %s: %v", sessionID, err) + } + } + // Seeded with plain SQL on purpose: SetSessionMetadataKey is SQLite-only + // (json_set/json()), so using it here would fail on the seed rather than + // exercise the migration under test. + lastAgentError := `{"last_agent_error":{"message":"agent crashed","occurred_at":"` + + occurredAt.Format(time.RFC3339Nano) + `"}}` + for _, sessionID := range []string{"pg-recovered", "pg-current"} { + if _, err := db.Exec(db.Rebind( + `UPDATE task_sessions SET metadata = ? WHERE id = ?`), lastAgentError, sessionID); err != nil { + t.Fatalf("seed last agent error on %s: %v", sessionID, err) + } + } + for value, sessionID := range map[string]string{"": "pg-blank", "null": "pg-nulled"} { + if _, err := db.Exec(db.Rebind( + `UPDATE task_sessions SET metadata = ? WHERE id = ?`), value, sessionID); err != nil { + t.Fatalf("seed %q metadata: %v", value, err) + } + } + + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `), "pg-turn", "pg-recovered", "task-pg-recovered", occurredAt, occurredAt, occurredAt); err != nil { + t.Fatalf("seed turn: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at) + VALUES (?, ?, '', ?, 'agent', 'back on track', 'message', '{}', ?) + `), "pg-msg", "pg-recovered", "pg-turn", occurredAt.Add(time.Hour)); err != nil { + t.Fatalf("seed agent message: %v", err) + } + + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at) + VALUES (?, ?, 4, ?, ?) + `), "task-pg-recovered", "ws-pg-recovered", + `{"active_error":{"session_id":"pg-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`, + occurredAt); err != nil { + t.Fatalf("seed status summary: %v", err) + } + + if err := repo.clearRecoveredAgentErrors(); err != nil { + t.Fatalf("clearRecoveredAgentErrors: %v", err) + } + + hasError := func(sessionID string) bool { + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("get session %s: %v", sessionID, err) + } + _, ok := models.LoadLastAgentError(session.Metadata) + return ok + } + if hasError("pg-recovered") { + t.Fatal("a failure the agent recovered from must be cleared on Postgres") + } + if !hasError("pg-current") { + t.Fatal("a failure with no successful work after it must be left alone") + } + + var summary string + if err := db.Get(&summary, db.Rebind( + `SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-pg-recovered"); err != nil { + t.Fatalf("read status summary: %v", err) + } + if strings.Contains(summary, "active_error") { + t.Fatalf("summary = %s, want the cached error cleared", summary) + } + if !strings.Contains(summary, "permission") { + t.Fatalf("summary = %s, want the rest of the projection preserved", summary) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/session_test.go b/apps/backend/internal/task/repository/sqlite/session_test.go index b073320fa5..abd6d403c3 100644 --- a/apps/backend/internal/task/repository/sqlite/session_test.go +++ b/apps/backend/internal/task/repository/sqlite/session_test.go @@ -6,7 +6,9 @@ import ( "context" "database/sql" "errors" + "fmt" "path/filepath" + "strconv" "strings" "sync" "testing" @@ -1494,6 +1496,102 @@ func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) { } } +// Nothing used to retire a stored agent failure, so installations carry records +// whose errors the agent recovered from long ago — each one still painting a red +// icon on the task list. +func TestClearRecoveredAgentErrorsBackfill(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC) + seedStale := func(t *testing.T, taskID, sessionID, turnID string) { + t.Helper() + seedForMsgTest(t, repo, taskID, sessionID, turnID) + if err := repo.SetSessionMetadataKey(ctx, sessionID, models.SessionMetaKeyLastAgentError, + models.LastAgentError{Message: "agent crashed", OccurredAt: occurredAt}); err != nil { + t.Fatalf("seed last agent error on %s: %v", sessionID, err) + } + } + + // Recovered: an ordinary agent message lands after the failure. + seedStale(t, "task-recovered", "sess-recovered", "turn-recovered") + insertAgentMsg(t, repo, "msg-recovered", "sess-recovered", "turn-recovered", + "agent", "back on track", occurredAt.Add(time.Hour)) + + // Still failing: nothing successful has happened since. + seedStale(t, "task-current", "sess-current", "turn-current") + + // Recovery that predates the failure must not count. + seedStale(t, "task-older", "sess-older", "turn-older") + insertAgentMsg(t, repo, "msg-older", "sess-older", "turn-older", + "agent", "earlier output", occurredAt.Add(-time.Hour)) + + // A user message after the failure is not the agent producing good output. + seedStale(t, "task-user", "sess-user", "turn-user") + insertAgentMsg(t, repo, "msg-user", "sess-user", "turn-user", + "user", "are you stuck?", occurredAt.Add(time.Hour)) + + // These statements scan every session before filtering, so a row the + // repository wrote as "no metadata" must not abort them. migrate.Apply + // swallows SQL errors, so the proof is that the recovered session below is + // still cleared with these rows present — an aborted statement would leave + // it untouched. + for index, blank := range []string{"", "null"} { + id := fmt.Sprintf("sess-blank-%d", index) + seedForMsgTest(t, repo, "task-blank-"+strconv.Itoa(index), id, "turn-blank-"+strconv.Itoa(index)) + if _, err := repo.db.Exec(repo.db.Rebind( + `UPDATE task_sessions SET metadata = ? WHERE id = ?`), blank, id); err != nil { + t.Fatalf("seed %q metadata: %v", blank, err) + } + } + + // The cached summary keeps the icon up until it is cleared too. + if _, err := repo.db.Exec(repo.db.Rebind(` + INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at) + VALUES (?, '', 4, ?, ?) + `), "task-recovered", + `{"active_error":{"session_id":"sess-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`, + time.Now().UTC()); err != nil { + t.Fatalf("seed status summary: %v", err) + } + + if err := repo.clearRecoveredAgentErrors(); err != nil { + t.Fatalf("clearRecoveredAgentErrors: %v", err) + } + + hasError := func(t *testing.T, sessionID string) bool { + t.Helper() + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("get session %s: %v", sessionID, err) + } + _, ok := models.LoadLastAgentError(session.Metadata) + return ok + } + + if hasError(t, "sess-recovered") { + t.Fatalf("a failure the agent recovered from must be cleared") + } + for _, sessionID := range []string{"sess-current", "sess-older", "sess-user"} { + if !hasError(t, sessionID) { + t.Fatalf("%s has no successful agent work after the failure; it must stay", sessionID) + } + } + + var summary string + if err := repo.db.QueryRow(repo.db.Rebind( + `SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-recovered", + ).Scan(&summary); err != nil { + t.Fatalf("read status summary: %v", err) + } + if strings.Contains(summary, "active_error") { + t.Fatalf("summary = %s, want the cached error cleared", summary) + } + if !strings.Contains(summary, "permission") { + t.Fatalf("summary = %s, want the rest of the projection preserved", summary) + } +} + func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID string) (string, sql.NullTime, time.Time) { t.Helper() var errorMessage string diff --git a/apps/backend/internal/task/service/service_status_summary_rebuild_test.go b/apps/backend/internal/task/service/service_status_summary_rebuild_test.go index 05d8afd823..41f3ec0513 100644 --- a/apps/backend/internal/task/service/service_status_summary_rebuild_test.go +++ b/apps/backend/internal/task/service/service_status_summary_rebuild_test.go @@ -151,3 +151,50 @@ func TestHydrateMissingTaskStatusSummariesRepairsExistingTaskOnce(t *testing.T) t.Fatalf("PR reader calls after existing summary = %d, want one", prReader.calls) } } + +// The rebuild path is what runs after a restart, so it is where a stale record +// used to come back. A session whose error was cleared must rebuild clean. +func TestHydrateMissingTaskStatusSummariesSkipsClearedAgentErrors(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + createTaskWithoutRepositories(t, ctx, repo) + + session := &models.TaskSession{ + ID: "session-1", + TaskID: "task-1", + State: models.TaskSessionStateWaitingForInput, + IsPrimary: true, + } + if err := repo.CreateTaskSession(ctx, session); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + // The orchestrator writes JSON null on turn completion rather than deleting + // the key, so the rebuild must treat that as "no failure". + if err := repo.SetSessionMetadataKey(ctx, session.ID, models.SessionMetaKeyLastAgentError, nil); err != nil { + t.Fatalf("clear last agent error: %v", err) + } + stored, err := repo.GetTaskSession(ctx, session.ID) + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + + svc.statusSummaries = repo + task := &models.Task{ID: "task-1", WorkspaceID: "ws-1"} + got, err := svc.HydrateMissingTaskStatusSummaries( + ctx, + []*models.Task{task}, + map[string][]*models.TaskSession{task.ID: {stored}}, + map[string]models.TaskPendingAction{}, + map[string]*statussummary.TaskStatusSummary{}, + ) + if err != nil { + t.Fatalf("HydrateMissingTaskStatusSummaries: %v", err) + } + summary := got[task.ID] + if summary == nil { + t.Fatal("repaired summary missing from returned map") + } + if summary.ActiveError != nil { + t.Fatalf("active error = %+v, want a cleared record to stay invisible", summary.ActiveError) + } +} diff --git a/apps/backend/internal/task/statussummary/projector.go b/apps/backend/internal/task/statussummary/projector.go index 58bd55d9c9..8f38e7831f 100644 --- a/apps/backend/internal/task/statussummary/projector.go +++ b/apps/backend/internal/task/statussummary/projector.go @@ -101,13 +101,17 @@ type projectionState struct { pendingObserved bool activityObserved bool errors map[string]*ActiveErrorSummary - errorsObserved bool - git map[string]GitSummary - gitBaseline *GitSummary - gitObserved bool - prs map[string]pullRequestObservation - prBaseline *PullRequestSummary - prObserved bool + // clearedErrorStamps records, per session, the stamp of the last error this + // projection cleared, so a durable breadcrumb replayed on a later session + // event cannot re-arm an error affordance the agent already recovered from. + clearedErrorStamps map[string]string + errorsObserved bool + git map[string]GitSummary + gitBaseline *GitSummary + gitObserved bool + prs map[string]pullRequestObservation + prBaseline *PullRequestSummary + prObserved bool } type sessionObservation struct { @@ -463,11 +467,12 @@ func updateSessionPendingLocked(state *projectionState, sessionID, action string func newProjectionState() *projectionState { return &projectionState{ - sessions: make(map[string]sessionObservation), - pending: make(map[string]string), - errors: make(map[string]*ActiveErrorSummary), - git: make(map[string]GitSummary), - prs: make(map[string]pullRequestObservation), + sessions: make(map[string]sessionObservation), + pending: make(map[string]string), + errors: make(map[string]*ActiveErrorSummary), + clearedErrorStamps: make(map[string]string), + git: make(map[string]GitSummary), + prs: make(map[string]pullRequestObservation), } } @@ -591,15 +596,36 @@ func (p *Projector) applySessionEventLocked(state *projectionState, data map[str changed := true if metadata, ok := data["session_metadata"].(map[string]interface{}); ok { - if errSummary, present := errorFromMetadata(p.now().UTC(), sessionID, metadata); present { - state.errorsObserved = true - changed = !errorEqual(state.errors[sessionID], errSummary) || changed - state.errors[sessionID] = errSummary - } + p.applySessionMetadataErrorLocked(state, sessionID, metadata) } return changed } +// applySessionMetadataErrorLocked folds the session's durable error record into +// the projection. A dismissed or superseded record clears the affordance, and a +// record this projection already cleared is not re-applied — session metadata +// keeps failures as a breadcrumb, so replaying it verbatim would re-arm errors +// the agent has since recovered from. +func (p *Projector) applySessionMetadataErrorLocked( + state *projectionState, + sessionID string, + metadata map[string]interface{}, +) { + errSummary, observed := errorFromMetadata(p.now().UTC(), sessionID, metadata) + if !observed { + return + } + state.errorsObserved = true + if errSummary == nil { + p.clearErrorLocked(state, sessionID) + return + } + if state.clearedErrorStamps[sessionID] == errSummary.Stamp { + return + } + state.errors[sessionID] = errSummary +} + func (p *Projector) applyActivityEventLocked(state *projectionState, data map[string]interface{}) bool { sessionID := stringField(data, "session_id") if sessionID == "" { @@ -635,6 +661,10 @@ func (p *Projector) applyErrorEventLocked(state *projectionState, data map[strin if errorEqual(state.errors[sessionID], errSummary) { return false } + // An explicit error event is authoritative: it re-arms even a stamp this + // projection cleared earlier, which is what makes an identical failure + // recurring after a recovery visible again. + delete(state.clearedErrorStamps, sessionID) state.errors[sessionID] = errSummary return true } diff --git a/apps/backend/internal/task/statussummary/projector_helpers.go b/apps/backend/internal/task/statussummary/projector_helpers.go index e609932104..f5f4a509b8 100644 --- a/apps/backend/internal/task/statussummary/projector_helpers.go +++ b/apps/backend/internal/task/statussummary/projector_helpers.go @@ -37,19 +37,51 @@ func recomputeTaskPending(state *projectionState) { func (p *Projector) clearErrorLocked(state *projectionState, sessionID string) bool { state.errorsObserved = true - if _, ok := state.errors[sessionID]; !ok { + existing, ok := state.errors[sessionID] + if !ok { return false } + // Remember which error was cleared. Session events carry a `session_metadata` + // snapshot taken when the publisher read the session, which can predate the + // clear — without this, such an event puts the very same error straight back. + // A genuinely new failure has a different stamp and is still applied. + if existing != nil && existing.Stamp != "" { + state.clearedErrorStamps[sessionID] = existing.Stamp + } delete(state.errors, sessionID) return true } -func errorFromMetadata(now time.Time, sessionID string, metadata map[string]interface{}) (*ActiveErrorSummary, bool) { - raw, ok := metadata["last_agent_error"].(map[string]interface{}) - if !ok { +// errorFromMetadata reads the session's stored error record. `observed` is true +// whenever the `last_agent_error` key is present at all: a dismissed record, or +// the JSON null the orchestrator writes when a turn completes, is an +// authoritative "no active error" for that session rather than an absence of +// information, so the caller must clear rather than ignore it. +// +// The key must be tested for presence before its type: a JSON null arrives as a +// nil value, and asserting `.(map[string]interface{})` on that is +// indistinguishable from the key being missing entirely. Reading it as missing +// would leave a restored error armed forever on any session whose record was +// cleared. +func errorFromMetadata( + now time.Time, + sessionID string, + metadata map[string]interface{}, +) (summary *ActiveErrorSummary, observed bool) { + value, present := metadata["last_agent_error"] + if !present { return nil, false } - return errorFromMap(now, sessionID, raw) + raw, ok := value.(map[string]interface{}) + if !ok { + // Null or malformed: the key exists, so the session has no active error. + return nil, true + } + active, ok := errorFromMap(now, sessionID, raw) + if !ok { + return nil, true + } + return active, true } func errorFromMap(now time.Time, sessionID string, data map[string]interface{}) (*ActiveErrorSummary, bool) { diff --git a/apps/backend/internal/task/statussummary/projector_test.go b/apps/backend/internal/task/statussummary/projector_test.go index 7ebd030e04..36a0b13bb5 100644 --- a/apps/backend/internal/task/statussummary/projector_test.go +++ b/apps/backend/internal/task/statussummary/projector_test.go @@ -279,6 +279,174 @@ func TestProjectorDerivesBoundedStatusAcrossSources(t *testing.T) { } } +// Session events carry a `session_metadata` snapshot taken when the publisher +// read the session, so one taken before the error was cleared must not re-arm an +// error this projection already cleared. +func TestProjectorDoesNotResurrectClearedErrorFromSessionMetadata(t *testing.T) { + _, store, eventBus, _, _ := newProjectorTest(t) + const taskID = "task-error-replay" + const sessionID = "session-error-replay" + + occurredAt := "2026-08-01T18:01:00.000000000Z" + breadcrumb := map[string]interface{}{ + "last_agent_error": map[string]interface{}{ + "message": "agent crashed", + "occurred_at": occurredAt, + }, + } + + publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{ + "session_metadata": breadcrumb, + }) + if got := store.summary(taskID); got == nil || got.ActiveError == nil { + t.Fatalf("active error after failure = %+v", got) + } + + // The agent recovers and posts an ordinary message. + publishProjectorEvent(t, eventBus, events.MessageAdded, events.MessageAdded, map[string]interface{}{ + "task_id": taskID, + "session_id": sessionID, + "author_type": "agent", + "type": "text", + "content": "recovered", + }) + if got := store.summary(taskID); got.ActiveError != nil { + t.Fatalf("active error after recovery = %+v, want cleared", got.ActiveError) + } + + // Turn end republishes the session, metadata breadcrumb and all. + publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{ + "new_state": "WAITING_FOR_INPUT", + "session_metadata": breadcrumb, + }) + if got := store.summary(taskID); got.ActiveError != nil { + t.Fatalf("active error after metadata replay = %+v, want it to stay cleared", got.ActiveError) + } + + // A genuinely new failure is still authoritative. + publishProjectorEvent(t, eventBus, events.TaskSessionErrorChanged, events.TaskSessionErrorChanged, map[string]interface{}{ + "task_id": taskID, + "session_id": sessionID, + "active": true, + "message": "agent crashed again", + "occurred_at": "2026-08-01T18:05:00.000000000Z", + "stamp": "error-stamp-2", + }) + got := store.summary(taskID) + if got.ActiveError == nil || got.ActiveError.Stamp != "error-stamp-2" { + t.Fatalf("active error after a new failure = %+v, want the new error", got.ActiveError) + } +} + +// A retired record is an authoritative "no active error" for the session, so it +// must clear an error the projection restored from its persisted row rather than +// being read as an absence of information. +func TestProjectorClearsErrorWhenSessionMetadataIsRetired(t *testing.T) { + store := newProjectorTestStore() + storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC) + store.rows["task-superseded"] = &StoredTaskStatusSummary{ + TaskID: "task-superseded", + WorkspaceID: "workspace-1", + Summary: TaskStatusSummary{ + Revision: 3, + UpdatedAt: storedAt, + PrimarySession: &PrimarySessionSummary{ID: "session-superseded", State: "RUNNING"}, + ActiveError: &ActiveErrorSummary{ + SessionID: "session-superseded", + Stamp: "error-stored", + OccurredAt: storedAt, + Preview: "agent crashed", + }, + }, + } + + projector := NewProjector(ProjectorConfig{ + Store: store, + ResolveWorkspace: func(context.Context, string) (string, error) { + return "workspace-1", nil + }, + Now: func() time.Time { return storedAt.Add(time.Minute) }, + }) + + if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{ + "task_id": "task-superseded", + "workspace_id": "workspace-1", + "session_id": "session-superseded", + "new_state": "WAITING_FOR_INPUT", + "is_primary": true, + "session_metadata": map[string]interface{}{ + "last_agent_error": map[string]interface{}{ + "message": "agent crashed", + "occurred_at": storedAt.Format(time.RFC3339Nano), + "dismissed_at": storedAt.Add(30 * time.Second).Format(time.RFC3339Nano), + }, + }, + })); err != nil { + t.Fatalf("replay session event: %v", err) + } + + got := store.summary("task-superseded") + if got == nil { + t.Fatal("summary disappeared") + } + if got.ActiveError != nil { + t.Fatalf("active error after retired metadata = %+v, want cleared", got.ActiveError) + } +} + +// The orchestrator clears a recovered failure by writing JSON null, which +// round-trips as a nil value under an existing key. Reading that as "no +// information" instead of "no active error" would leave a restored summary's +// error armed forever — the restart path this whole fix is about. +func TestProjectorClearsErrorWhenSessionMetadataIsNull(t *testing.T) { + store := newProjectorTestStore() + storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC) + store.rows["task-null"] = &StoredTaskStatusSummary{ + TaskID: "task-null", + WorkspaceID: "workspace-1", + Summary: TaskStatusSummary{ + Revision: 3, + UpdatedAt: storedAt, + PrimarySession: &PrimarySessionSummary{ID: "session-null", State: "RUNNING"}, + ActiveError: &ActiveErrorSummary{ + SessionID: "session-null", + Stamp: "error-stored", + OccurredAt: storedAt, + Preview: "agent crashed", + }, + }, + } + + projector := NewProjector(ProjectorConfig{ + Store: store, + ResolveWorkspace: func(context.Context, string) (string, error) { + return "workspace-1", nil + }, + Now: func() time.Time { return storedAt.Add(time.Minute) }, + }) + + if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{ + "task_id": "task-null", + "workspace_id": "workspace-1", + "session_id": "session-null", + "new_state": "WAITING_FOR_INPUT", + "is_primary": true, + "session_metadata": map[string]interface{}{ + "last_agent_error": nil, + }, + })); err != nil { + t.Fatalf("replay session event: %v", err) + } + + got := store.summary("task-null") + if got == nil { + t.Fatal("summary disappeared") + } + if got.ActiveError != nil { + t.Fatalf("active error after null metadata = %+v, want cleared", got.ActiveError) + } +} + func TestProjectorConvergesConcurrentGitObservations(t *testing.T) { projector, store, _, _, _ := newProjectorTest(t) const taskID = "task-concurrent-git" diff --git a/apps/backend/internal/task/statussummary/rebuild.go b/apps/backend/internal/task/statussummary/rebuild.go index e7ac9e5e39..43186d05db 100644 --- a/apps/backend/internal/task/statussummary/rebuild.go +++ b/apps/backend/internal/task/statussummary/rebuild.go @@ -67,16 +67,17 @@ type RebuildInput struct { // replaying any session stream. func BuildFromAuthoritative(input RebuildInput) TaskStatusSummary { state := &projectionState{ - sessions: make(map[string]sessionObservation, len(input.Sessions)), - pending: make(map[string]string, len(input.PendingActions)), - errors: make(map[string]*ActiveErrorSummary), - git: make(map[string]GitSummary, len(input.Git)), - prs: make(map[string]pullRequestObservation, len(input.PullRequests)), - pendingObserved: true, - activityObserved: input.ActivityObserved, - errorsObserved: true, - gitObserved: input.GitObserved, - prObserved: input.PRObserved, + sessions: make(map[string]sessionObservation, len(input.Sessions)), + pending: make(map[string]string, len(input.PendingActions)), + errors: make(map[string]*ActiveErrorSummary), + clearedErrorStamps: make(map[string]string), + git: make(map[string]GitSummary, len(input.Git)), + prs: make(map[string]pullRequestObservation, len(input.PullRequests)), + pendingObserved: true, + activityObserved: input.ActivityObserved, + errorsObserved: true, + gitObserved: input.GitObserved, + prObserved: input.PRObserved, } for _, inputSession := range input.Sessions { if strings.TrimSpace(inputSession.ID) == "" {