fix(tasks): clear the error icon once the agent recovers (#2391)

Co-authored-by: Kandev Agent <agent@kandev.dev>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
João Salavisa
2026-08-07 20:48:31 +02:00
committed by GitHub
parent 13c0621b0c
commit 9514ab28e4
11 changed files with 755 additions and 34 deletions
@@ -1054,6 +1054,10 @@ func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.A
s.scheduler.HandleTaskCompleted(data.TaskID, true)
s.scheduler.RemoveTask(data.TaskID)
// The agent finished a turn, so any stored failure no longer describes the
// session. `session` was read above, so the guard costs nothing.
s.clearRecoveredAgentError(context.WithoutCancel(ctx), data.TaskID, session)
s.completeTurnForSession(context.WithoutCancel(ctx), data.SessionID)
if s.sessionHasPendingClarification(ctx, data.SessionID) {
@@ -1503,6 +1507,58 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE
}
}
// clearRecoveredAgentError drops a session's stored agent failure once the agent
// completes a turn, and publishes the inactive error event so open clients drop
// the red error affordance.
//
// persistLastAgentError deliberately keeps the record across a successful turn
// as an investigation breadcrumb, but nothing ever retired it, so a failure the
// agent recovered from weeks ago still read as live: every path that re-derives
// task status from session metadata (a status-summary rebuild, a backend
// restart, a later session event) put it straight back. The failure also lands
// in the transcript as a recovery message, which is where an investigation
// actually looks, so the metadata copy is not the durable record.
//
// Writes JSON null rather than a delete: LoadLastAgentError already treats that
// as absent, so no new repository surface is needed.
func (s *Service) clearRecoveredAgentError(ctx context.Context, taskID string, session *models.TaskSession) {
if session == nil || session.ID == "" {
return
}
if _, ok := models.LoadLastAgentError(session.Metadata); !ok {
return
}
if err := s.repo.SetSessionMetadataKey(
ctx, session.ID, models.SessionMetaKeyLastAgentError, nil,
); err != nil {
s.logger.Warn("failed to clear recovered agent error",
zap.String("task_id", taskID),
zap.String("session_id", session.ID),
zap.Error(err))
return
}
// Keep the in-memory copy in step: the session-state publish below reads
// its `session_metadata` straight off this object.
delete(session.Metadata, models.SessionMetaKeyLastAgentError)
if s.eventBus == nil {
return
}
if err := s.eventBus.Publish(ctx, events.TaskSessionErrorChanged, bus.NewEvent(
events.TaskSessionErrorChanged,
"orchestrator",
map[string]interface{}{
"task_id": taskID,
"session_id": session.ID,
"active": false,
},
)); err != nil {
s.logger.Warn("failed to publish recovered task session error event",
zap.String("task_id", taskID),
zap.String("session_id", session.ID),
zap.Error(err))
}
}
// createRecoveryStatusMessage builds and persists the ActionMessage shown
// in the kanban chat surface after a recoverable agent failure. Must only
// be called for non-office sessions (office sessions render their own error UI).
@@ -2191,6 +2191,79 @@ func TestSessionStartClearsInterruptedMarker(t *testing.T) {
})
}
// Nothing else ever retires a stored agent failure, so without this clear every
// path that re-derives task status from session metadata resurrects it and the
// error icon never goes away.
func TestClearRecoveredAgentErrorOnTurnCompletion(t *testing.T) {
ctx := context.Background()
seedError := func(t *testing.T, repo *sqliterepo.Repository) {
t.Helper()
require.NoError(t, repo.SetSessionMetadataKey(
ctx, "s1", models.SessionMetaKeyLastAgentError,
models.LastAgentError{Message: "agent crashed", OccurredAt: time.Now().UTC().Add(-time.Hour)},
))
}
errorEvents := func(eb *recordingEventBus) []recordedEvent {
var out []recordedEvent
for _, recorded := range eb.events {
if recorded.event != nil && recorded.event.Type == events.TaskSessionErrorChanged {
out = append(out, recorded)
}
}
return out
}
newService := func(repo *sqliterepo.Repository) (*Service, *recordingEventBus) {
eb := &recordingEventBus{}
svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo())
svc.eventBus = eb
return svc, eb
}
t.Run("clears the record and publishes it inactive", func(t *testing.T) {
repo := setupTestRepo(t)
seedSession(t, repo, "t1", "s1", "step1")
seedError(t, repo)
svc, eb := newService(repo)
session, err := repo.GetTaskSession(ctx, "s1")
require.NoError(t, err)
svc.clearRecoveredAgentError(ctx, "t1", session)
stored, err := repo.GetTaskSession(ctx, "s1")
require.NoError(t, err)
_, ok := models.LoadLastAgentError(stored.Metadata)
require.False(t, ok, "a recovered failure must not keep driving the error icon")
// The session-state publish that follows reads session_metadata straight
// off this object, so a stale copy would re-arm the icon immediately.
_, ok = models.LoadLastAgentError(session.Metadata)
require.False(t, ok, "the in-memory copy must be cleared too")
published := errorEvents(eb)
require.Len(t, published, 1, "clients need one event to drop the icon")
data, ok := published[0].event.Data.(map[string]interface{})
require.True(t, ok)
require.Equal(t, false, data["active"])
require.Equal(t, "s1", data["session_id"])
})
t.Run("stays silent when the session has no stored error", func(t *testing.T) {
repo := setupTestRepo(t)
seedSession(t, repo, "t1", "s1", "step1")
svc, eb := newService(repo)
session, err := repo.GetTaskSession(ctx, "s1")
require.NoError(t, err)
svc.clearRecoveredAgentError(ctx, "t1", session)
require.Empty(t, errorEvents(eb),
"every later completion must stay silent so turns do not churn the row")
})
}
func TestSetSessionStartingRejectsTerminalSession(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
+2 -2
View File
@@ -662,8 +662,8 @@ const (
// Workflow represents a task workflow
type Workflow struct {
ID string `json:"id"`
WorkspaceID string `json:"workspace_id"`
ID string `json:"id"`
WorkspaceID string `json:"workspace_id"`
Name string `json:"name"`
Description string `json:"description"`
// Prompt is optional workflow-level agent instructions prepended at
@@ -237,9 +237,116 @@ func (r *Repository) runMigrations() error {
CREATE INDEX IF NOT EXISTS idx_task_status_summaries_workspace
ON task_status_summaries(workspace_id)`)
if err := r.clearRecoveredAgentErrors(); err != nil {
return err
}
return nil
}
// clearRecoveredAgentErrors repairs sessions whose stored agent failure was
// overtaken by successful work before the orchestrator learned to clear it.
//
// Nothing used to retire `last_agent_error`, so a failure the agent recovered
// from weeks ago still reads as live and keeps a red error icon on the task list
// forever. The orchestrator now clears the record on turn completion; this
// applies the same rule to history.
//
// Deliberately narrow: a record is cleared only when the session has an ordinary
// agent message newer than the failure — the same "the agent has produced good
// output since" signal. A failure with no successful work after it is current
// and is left alone.
func (r *Repository) clearRecoveredAgentErrors() error {
postgres := dialect.IsPostgres(r.db.DriverName())
sessionError := func(key string) string {
return jsonText(postgres, "s.metadata", "last_agent_error", key)
}
recoveredSessions := `
SELECT s.id
FROM task_sessions s
WHERE ` + sessionError("message") + ` IS NOT NULL
AND EXISTS (
SELECT 1 FROM task_session_messages m
WHERE m.task_session_id = s.id
AND m.author_type = 'agent'
AND m.type NOT IN ('error', 'status')
AND ` + timestampColumn(postgres, "m.created_at") + ` > ` +
timestampFromText(postgres, sessionError("occurred_at")) + `
)`
r.migrate.Apply("task_sessions.last_agent_error.recovered_cleanup",
`UPDATE task_sessions SET metadata = `+jsonRemoveKey(postgres, "metadata", "last_agent_error")+
` WHERE id IN (`+recoveredSessions+`)`)
// The summary row caches the derived error, so clearing the session record
// alone would leave the icon up until that session next emitted an event.
// A cached error naming a session that has no stored failure is stale by
// definition, which also sweeps up any earlier drift.
r.migrate.Apply("task_status_summaries.active_error.recovered_cleanup", `
UPDATE task_status_summaries
SET summary = `+jsonRemoveKey(postgres, "summary", "active_error")+`
WHERE `+jsonText(postgres, "summary", "active_error", "session_id")+` IN (
SELECT s.id FROM task_sessions s
WHERE `+sessionError("message")+` IS NULL
)`)
return nil
}
// jsonColumn makes a TEXT-typed JSON column safe to parse. Both dialects raise
// on an empty or malformed document — Postgres on the `::jsonb` cast, SQLite in
// `json_extract` — and these statements scan every row before filtering, so a
// single such row aborts the whole migration. The repository writes ” and
// 'null' for "no metadata", so both are normalized the way the rest of the
// package already does.
func jsonColumn(postgres bool, column string) string {
empty, open := "'{}'", ""
if postgres {
empty, open = "'{}'::jsonb", "::jsonb"
}
return "(CASE WHEN " + column + " IS NULL OR " + column + " = 'null' OR " + column +
" = '' THEN " + empty + " ELSE " + column + open + " END)"
}
// jsonText extracts a nested JSON text value from a TEXT-typed JSON column.
func jsonText(postgres bool, column, parent, key string) string {
base := jsonColumn(postgres, column)
if postgres {
return "(" + base + " #>> '{" + parent + "," + key + "}')"
}
return "json_extract(" + base + ", '$." + parent + "." + key + "')"
}
// timestampColumn normalizes a timestamp column for comparison. SQLite stores
// timestamps as text and would compare them lexically, so it needs Julian days;
// on Postgres the column is already a timestamp.
func timestampColumn(postgres bool, column string) string {
if postgres {
return "(" + column + ")::timestamptz"
}
return "julianday(" + column + ")"
}
// timestampFromText normalizes a timestamp extracted from JSON so it compares
// against timestampColumn — the two carry different text shapes
// ('2026-08-01 10:00:00+00:00' vs RFC3339 '2026-08-01T10:00:00Z'). NULLIF keeps
// an empty stored timestamp from raising on the Postgres cast; it then compares
// as NULL, so the row simply does not match.
func timestampFromText(postgres bool, expression string) string {
if postgres {
return "(NULLIF(" + expression + ", '')::timestamptz)"
}
return "julianday(" + expression + ")"
}
func jsonRemoveKey(postgres bool, column, key string) string {
base := jsonColumn(postgres, column)
if postgres {
return "(" + base + " - '" + key + "')::text"
}
return "json_remove(" + base + ", '$." + key + "')"
}
// ensureImproveKandevWorkflowTemplateUniqueness removes the broad index from
// the initial implementation, reconciles legacy bootstrap duplicates, then
// enforces uniqueness only for the two hidden workflows created by this flow.
@@ -696,3 +696,112 @@ func TestPostgresSetSessionPrimary_ConcurrentPromotionsLeaveExactlyOnePrimary(t
t.Errorf("expected exactly 1 primary session after %d concurrent promotions, got %d — FOR UPDATE lock not serializing across connections", concurrency, primaryCount)
}
}
// TestPostgresClearRecoveredAgentErrors is the Postgres counterpart to
// TestClearRecoveredAgentErrorsBackfill. clearRecoveredAgentErrors is built from
// three dialect-sensitive helpers (jsonColumn, jsonText/timestamp*,
// jsonRemoveKey), so ADR 0027 asks for env-gated Postgres behavior coverage —
// schema replay would not exercise the jsonb operators at all.
//
// migrate.Apply swallows SQL errors, so the assertions below are the only proof
// the statements ran: a dialect mistake leaves the recovered row untouched
// rather than returning an error. Skips unless KANDEV_TEST_POSTGRES_DSN is set.
func TestPostgresClearRecoveredAgentErrors(t *testing.T) {
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
repo, err := NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init postgres schema: %v", err)
}
ctx := context.Background()
occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC)
if _, err := db.Exec(db.Rebind(`
INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
VALUES (?, ?, ?, ?, ?)
`), "task-pg-recovered", "ws-pg-recovered", "Task", occurredAt, occurredAt); err != nil {
t.Fatalf("seed task: %v", err)
}
// recovered: agent output after the failure. current: none since.
// blank/nulled: rows the repository wrote as "no metadata" — both dialects
// raise on parsing those, and the statements scan every row before
// filtering, so an unguarded cast aborts the migration for everyone.
for _, sessionID := range []string{"pg-recovered", "pg-current", "pg-blank", "pg-nulled"} {
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: sessionID, TaskID: "task-pg-recovered", State: models.TaskSessionStateWaitingForInput,
}); err != nil {
t.Fatalf("CreateTaskSession %s: %v", sessionID, err)
}
}
// Seeded with plain SQL on purpose: SetSessionMetadataKey is SQLite-only
// (json_set/json()), so using it here would fail on the seed rather than
// exercise the migration under test.
lastAgentError := `{"last_agent_error":{"message":"agent crashed","occurred_at":"` +
occurredAt.Format(time.RFC3339Nano) + `"}}`
for _, sessionID := range []string{"pg-recovered", "pg-current"} {
if _, err := db.Exec(db.Rebind(
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), lastAgentError, sessionID); err != nil {
t.Fatalf("seed last agent error on %s: %v", sessionID, err)
}
}
for value, sessionID := range map[string]string{"": "pg-blank", "null": "pg-nulled"} {
if _, err := db.Exec(db.Rebind(
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), value, sessionID); err != nil {
t.Fatalf("seed %q metadata: %v", value, err)
}
}
if _, err := db.Exec(db.Rebind(`
INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
`), "pg-turn", "pg-recovered", "task-pg-recovered", occurredAt, occurredAt, occurredAt); err != nil {
t.Fatalf("seed turn: %v", err)
}
if _, err := db.Exec(db.Rebind(`
INSERT INTO task_session_messages
(id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at)
VALUES (?, ?, '', ?, 'agent', 'back on track', 'message', '{}', ?)
`), "pg-msg", "pg-recovered", "pg-turn", occurredAt.Add(time.Hour)); err != nil {
t.Fatalf("seed agent message: %v", err)
}
if _, err := db.Exec(db.Rebind(`
INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at)
VALUES (?, ?, 4, ?, ?)
`), "task-pg-recovered", "ws-pg-recovered",
`{"active_error":{"session_id":"pg-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`,
occurredAt); err != nil {
t.Fatalf("seed status summary: %v", err)
}
if err := repo.clearRecoveredAgentErrors(); err != nil {
t.Fatalf("clearRecoveredAgentErrors: %v", err)
}
hasError := func(sessionID string) bool {
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("get session %s: %v", sessionID, err)
}
_, ok := models.LoadLastAgentError(session.Metadata)
return ok
}
if hasError("pg-recovered") {
t.Fatal("a failure the agent recovered from must be cleared on Postgres")
}
if !hasError("pg-current") {
t.Fatal("a failure with no successful work after it must be left alone")
}
var summary string
if err := db.Get(&summary, db.Rebind(
`SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-pg-recovered"); err != nil {
t.Fatalf("read status summary: %v", err)
}
if strings.Contains(summary, "active_error") {
t.Fatalf("summary = %s, want the cached error cleared", summary)
}
if !strings.Contains(summary, "permission") {
t.Fatalf("summary = %s, want the rest of the projection preserved", summary)
}
}
@@ -6,7 +6,9 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
@@ -1494,6 +1496,102 @@ func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) {
}
}
// Nothing used to retire a stored agent failure, so installations carry records
// whose errors the agent recovered from long ago — each one still painting a red
// icon on the task list.
func TestClearRecoveredAgentErrorsBackfill(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC)
seedStale := func(t *testing.T, taskID, sessionID, turnID string) {
t.Helper()
seedForMsgTest(t, repo, taskID, sessionID, turnID)
if err := repo.SetSessionMetadataKey(ctx, sessionID, models.SessionMetaKeyLastAgentError,
models.LastAgentError{Message: "agent crashed", OccurredAt: occurredAt}); err != nil {
t.Fatalf("seed last agent error on %s: %v", sessionID, err)
}
}
// Recovered: an ordinary agent message lands after the failure.
seedStale(t, "task-recovered", "sess-recovered", "turn-recovered")
insertAgentMsg(t, repo, "msg-recovered", "sess-recovered", "turn-recovered",
"agent", "back on track", occurredAt.Add(time.Hour))
// Still failing: nothing successful has happened since.
seedStale(t, "task-current", "sess-current", "turn-current")
// Recovery that predates the failure must not count.
seedStale(t, "task-older", "sess-older", "turn-older")
insertAgentMsg(t, repo, "msg-older", "sess-older", "turn-older",
"agent", "earlier output", occurredAt.Add(-time.Hour))
// A user message after the failure is not the agent producing good output.
seedStale(t, "task-user", "sess-user", "turn-user")
insertAgentMsg(t, repo, "msg-user", "sess-user", "turn-user",
"user", "are you stuck?", occurredAt.Add(time.Hour))
// These statements scan every session before filtering, so a row the
// repository wrote as "no metadata" must not abort them. migrate.Apply
// swallows SQL errors, so the proof is that the recovered session below is
// still cleared with these rows present — an aborted statement would leave
// it untouched.
for index, blank := range []string{"", "null"} {
id := fmt.Sprintf("sess-blank-%d", index)
seedForMsgTest(t, repo, "task-blank-"+strconv.Itoa(index), id, "turn-blank-"+strconv.Itoa(index))
if _, err := repo.db.Exec(repo.db.Rebind(
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), blank, id); err != nil {
t.Fatalf("seed %q metadata: %v", blank, err)
}
}
// The cached summary keeps the icon up until it is cleared too.
if _, err := repo.db.Exec(repo.db.Rebind(`
INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at)
VALUES (?, '', 4, ?, ?)
`), "task-recovered",
`{"active_error":{"session_id":"sess-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`,
time.Now().UTC()); err != nil {
t.Fatalf("seed status summary: %v", err)
}
if err := repo.clearRecoveredAgentErrors(); err != nil {
t.Fatalf("clearRecoveredAgentErrors: %v", err)
}
hasError := func(t *testing.T, sessionID string) bool {
t.Helper()
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("get session %s: %v", sessionID, err)
}
_, ok := models.LoadLastAgentError(session.Metadata)
return ok
}
if hasError(t, "sess-recovered") {
t.Fatalf("a failure the agent recovered from must be cleared")
}
for _, sessionID := range []string{"sess-current", "sess-older", "sess-user"} {
if !hasError(t, sessionID) {
t.Fatalf("%s has no successful agent work after the failure; it must stay", sessionID)
}
}
var summary string
if err := repo.db.QueryRow(repo.db.Rebind(
`SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-recovered",
).Scan(&summary); err != nil {
t.Fatalf("read status summary: %v", err)
}
if strings.Contains(summary, "active_error") {
t.Fatalf("summary = %s, want the cached error cleared", summary)
}
if !strings.Contains(summary, "permission") {
t.Fatalf("summary = %s, want the rest of the projection preserved", summary)
}
}
func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID string) (string, sql.NullTime, time.Time) {
t.Helper()
var errorMessage string
@@ -151,3 +151,50 @@ func TestHydrateMissingTaskStatusSummariesRepairsExistingTaskOnce(t *testing.T)
t.Fatalf("PR reader calls after existing summary = %d, want one", prReader.calls)
}
}
// The rebuild path is what runs after a restart, so it is where a stale record
// used to come back. A session whose error was cleared must rebuild clean.
func TestHydrateMissingTaskStatusSummariesSkipsClearedAgentErrors(t *testing.T) {
svc, _, repo := createTestService(t)
ctx := context.Background()
createTaskWithoutRepositories(t, ctx, repo)
session := &models.TaskSession{
ID: "session-1",
TaskID: "task-1",
State: models.TaskSessionStateWaitingForInput,
IsPrimary: true,
}
if err := repo.CreateTaskSession(ctx, session); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
// The orchestrator writes JSON null on turn completion rather than deleting
// the key, so the rebuild must treat that as "no failure".
if err := repo.SetSessionMetadataKey(ctx, session.ID, models.SessionMetaKeyLastAgentError, nil); err != nil {
t.Fatalf("clear last agent error: %v", err)
}
stored, err := repo.GetTaskSession(ctx, session.ID)
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
svc.statusSummaries = repo
task := &models.Task{ID: "task-1", WorkspaceID: "ws-1"}
got, err := svc.HydrateMissingTaskStatusSummaries(
ctx,
[]*models.Task{task},
map[string][]*models.TaskSession{task.ID: {stored}},
map[string]models.TaskPendingAction{},
map[string]*statussummary.TaskStatusSummary{},
)
if err != nil {
t.Fatalf("HydrateMissingTaskStatusSummaries: %v", err)
}
summary := got[task.ID]
if summary == nil {
t.Fatal("repaired summary missing from returned map")
}
if summary.ActiveError != nil {
t.Fatalf("active error = %+v, want a cleared record to stay invisible", summary.ActiveError)
}
}
@@ -101,13 +101,17 @@ type projectionState struct {
pendingObserved bool
activityObserved bool
errors map[string]*ActiveErrorSummary
errorsObserved bool
git map[string]GitSummary
gitBaseline *GitSummary
gitObserved bool
prs map[string]pullRequestObservation
prBaseline *PullRequestSummary
prObserved bool
// clearedErrorStamps records, per session, the stamp of the last error this
// projection cleared, so a durable breadcrumb replayed on a later session
// event cannot re-arm an error affordance the agent already recovered from.
clearedErrorStamps map[string]string
errorsObserved bool
git map[string]GitSummary
gitBaseline *GitSummary
gitObserved bool
prs map[string]pullRequestObservation
prBaseline *PullRequestSummary
prObserved bool
}
type sessionObservation struct {
@@ -463,11 +467,12 @@ func updateSessionPendingLocked(state *projectionState, sessionID, action string
func newProjectionState() *projectionState {
return &projectionState{
sessions: make(map[string]sessionObservation),
pending: make(map[string]string),
errors: make(map[string]*ActiveErrorSummary),
git: make(map[string]GitSummary),
prs: make(map[string]pullRequestObservation),
sessions: make(map[string]sessionObservation),
pending: make(map[string]string),
errors: make(map[string]*ActiveErrorSummary),
clearedErrorStamps: make(map[string]string),
git: make(map[string]GitSummary),
prs: make(map[string]pullRequestObservation),
}
}
@@ -591,15 +596,36 @@ func (p *Projector) applySessionEventLocked(state *projectionState, data map[str
changed := true
if metadata, ok := data["session_metadata"].(map[string]interface{}); ok {
if errSummary, present := errorFromMetadata(p.now().UTC(), sessionID, metadata); present {
state.errorsObserved = true
changed = !errorEqual(state.errors[sessionID], errSummary) || changed
state.errors[sessionID] = errSummary
}
p.applySessionMetadataErrorLocked(state, sessionID, metadata)
}
return changed
}
// applySessionMetadataErrorLocked folds the session's durable error record into
// the projection. A dismissed or superseded record clears the affordance, and a
// record this projection already cleared is not re-applied — session metadata
// keeps failures as a breadcrumb, so replaying it verbatim would re-arm errors
// the agent has since recovered from.
func (p *Projector) applySessionMetadataErrorLocked(
state *projectionState,
sessionID string,
metadata map[string]interface{},
) {
errSummary, observed := errorFromMetadata(p.now().UTC(), sessionID, metadata)
if !observed {
return
}
state.errorsObserved = true
if errSummary == nil {
p.clearErrorLocked(state, sessionID)
return
}
if state.clearedErrorStamps[sessionID] == errSummary.Stamp {
return
}
state.errors[sessionID] = errSummary
}
func (p *Projector) applyActivityEventLocked(state *projectionState, data map[string]interface{}) bool {
sessionID := stringField(data, "session_id")
if sessionID == "" {
@@ -635,6 +661,10 @@ func (p *Projector) applyErrorEventLocked(state *projectionState, data map[strin
if errorEqual(state.errors[sessionID], errSummary) {
return false
}
// An explicit error event is authoritative: it re-arms even a stamp this
// projection cleared earlier, which is what makes an identical failure
// recurring after a recovery visible again.
delete(state.clearedErrorStamps, sessionID)
state.errors[sessionID] = errSummary
return true
}
@@ -37,19 +37,51 @@ func recomputeTaskPending(state *projectionState) {
func (p *Projector) clearErrorLocked(state *projectionState, sessionID string) bool {
state.errorsObserved = true
if _, ok := state.errors[sessionID]; !ok {
existing, ok := state.errors[sessionID]
if !ok {
return false
}
// Remember which error was cleared. Session events carry a `session_metadata`
// snapshot taken when the publisher read the session, which can predate the
// clear — without this, such an event puts the very same error straight back.
// A genuinely new failure has a different stamp and is still applied.
if existing != nil && existing.Stamp != "" {
state.clearedErrorStamps[sessionID] = existing.Stamp
}
delete(state.errors, sessionID)
return true
}
func errorFromMetadata(now time.Time, sessionID string, metadata map[string]interface{}) (*ActiveErrorSummary, bool) {
raw, ok := metadata["last_agent_error"].(map[string]interface{})
if !ok {
// errorFromMetadata reads the session's stored error record. `observed` is true
// whenever the `last_agent_error` key is present at all: a dismissed record, or
// the JSON null the orchestrator writes when a turn completes, is an
// authoritative "no active error" for that session rather than an absence of
// information, so the caller must clear rather than ignore it.
//
// The key must be tested for presence before its type: a JSON null arrives as a
// nil value, and asserting `.(map[string]interface{})` on that is
// indistinguishable from the key being missing entirely. Reading it as missing
// would leave a restored error armed forever on any session whose record was
// cleared.
func errorFromMetadata(
now time.Time,
sessionID string,
metadata map[string]interface{},
) (summary *ActiveErrorSummary, observed bool) {
value, present := metadata["last_agent_error"]
if !present {
return nil, false
}
return errorFromMap(now, sessionID, raw)
raw, ok := value.(map[string]interface{})
if !ok {
// Null or malformed: the key exists, so the session has no active error.
return nil, true
}
active, ok := errorFromMap(now, sessionID, raw)
if !ok {
return nil, true
}
return active, true
}
func errorFromMap(now time.Time, sessionID string, data map[string]interface{}) (*ActiveErrorSummary, bool) {
@@ -279,6 +279,174 @@ func TestProjectorDerivesBoundedStatusAcrossSources(t *testing.T) {
}
}
// Session events carry a `session_metadata` snapshot taken when the publisher
// read the session, so one taken before the error was cleared must not re-arm an
// error this projection already cleared.
func TestProjectorDoesNotResurrectClearedErrorFromSessionMetadata(t *testing.T) {
_, store, eventBus, _, _ := newProjectorTest(t)
const taskID = "task-error-replay"
const sessionID = "session-error-replay"
occurredAt := "2026-08-01T18:01:00.000000000Z"
breadcrumb := map[string]interface{}{
"last_agent_error": map[string]interface{}{
"message": "agent crashed",
"occurred_at": occurredAt,
},
}
publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{
"session_metadata": breadcrumb,
})
if got := store.summary(taskID); got == nil || got.ActiveError == nil {
t.Fatalf("active error after failure = %+v", got)
}
// The agent recovers and posts an ordinary message.
publishProjectorEvent(t, eventBus, events.MessageAdded, events.MessageAdded, map[string]interface{}{
"task_id": taskID,
"session_id": sessionID,
"author_type": "agent",
"type": "text",
"content": "recovered",
})
if got := store.summary(taskID); got.ActiveError != nil {
t.Fatalf("active error after recovery = %+v, want cleared", got.ActiveError)
}
// Turn end republishes the session, metadata breadcrumb and all.
publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{
"new_state": "WAITING_FOR_INPUT",
"session_metadata": breadcrumb,
})
if got := store.summary(taskID); got.ActiveError != nil {
t.Fatalf("active error after metadata replay = %+v, want it to stay cleared", got.ActiveError)
}
// A genuinely new failure is still authoritative.
publishProjectorEvent(t, eventBus, events.TaskSessionErrorChanged, events.TaskSessionErrorChanged, map[string]interface{}{
"task_id": taskID,
"session_id": sessionID,
"active": true,
"message": "agent crashed again",
"occurred_at": "2026-08-01T18:05:00.000000000Z",
"stamp": "error-stamp-2",
})
got := store.summary(taskID)
if got.ActiveError == nil || got.ActiveError.Stamp != "error-stamp-2" {
t.Fatalf("active error after a new failure = %+v, want the new error", got.ActiveError)
}
}
// A retired record is an authoritative "no active error" for the session, so it
// must clear an error the projection restored from its persisted row rather than
// being read as an absence of information.
func TestProjectorClearsErrorWhenSessionMetadataIsRetired(t *testing.T) {
store := newProjectorTestStore()
storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC)
store.rows["task-superseded"] = &StoredTaskStatusSummary{
TaskID: "task-superseded",
WorkspaceID: "workspace-1",
Summary: TaskStatusSummary{
Revision: 3,
UpdatedAt: storedAt,
PrimarySession: &PrimarySessionSummary{ID: "session-superseded", State: "RUNNING"},
ActiveError: &ActiveErrorSummary{
SessionID: "session-superseded",
Stamp: "error-stored",
OccurredAt: storedAt,
Preview: "agent crashed",
},
},
}
projector := NewProjector(ProjectorConfig{
Store: store,
ResolveWorkspace: func(context.Context, string) (string, error) {
return "workspace-1", nil
},
Now: func() time.Time { return storedAt.Add(time.Minute) },
})
if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{
"task_id": "task-superseded",
"workspace_id": "workspace-1",
"session_id": "session-superseded",
"new_state": "WAITING_FOR_INPUT",
"is_primary": true,
"session_metadata": map[string]interface{}{
"last_agent_error": map[string]interface{}{
"message": "agent crashed",
"occurred_at": storedAt.Format(time.RFC3339Nano),
"dismissed_at": storedAt.Add(30 * time.Second).Format(time.RFC3339Nano),
},
},
})); err != nil {
t.Fatalf("replay session event: %v", err)
}
got := store.summary("task-superseded")
if got == nil {
t.Fatal("summary disappeared")
}
if got.ActiveError != nil {
t.Fatalf("active error after retired metadata = %+v, want cleared", got.ActiveError)
}
}
// The orchestrator clears a recovered failure by writing JSON null, which
// round-trips as a nil value under an existing key. Reading that as "no
// information" instead of "no active error" would leave a restored summary's
// error armed forever — the restart path this whole fix is about.
func TestProjectorClearsErrorWhenSessionMetadataIsNull(t *testing.T) {
store := newProjectorTestStore()
storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC)
store.rows["task-null"] = &StoredTaskStatusSummary{
TaskID: "task-null",
WorkspaceID: "workspace-1",
Summary: TaskStatusSummary{
Revision: 3,
UpdatedAt: storedAt,
PrimarySession: &PrimarySessionSummary{ID: "session-null", State: "RUNNING"},
ActiveError: &ActiveErrorSummary{
SessionID: "session-null",
Stamp: "error-stored",
OccurredAt: storedAt,
Preview: "agent crashed",
},
},
}
projector := NewProjector(ProjectorConfig{
Store: store,
ResolveWorkspace: func(context.Context, string) (string, error) {
return "workspace-1", nil
},
Now: func() time.Time { return storedAt.Add(time.Minute) },
})
if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{
"task_id": "task-null",
"workspace_id": "workspace-1",
"session_id": "session-null",
"new_state": "WAITING_FOR_INPUT",
"is_primary": true,
"session_metadata": map[string]interface{}{
"last_agent_error": nil,
},
})); err != nil {
t.Fatalf("replay session event: %v", err)
}
got := store.summary("task-null")
if got == nil {
t.Fatal("summary disappeared")
}
if got.ActiveError != nil {
t.Fatalf("active error after null metadata = %+v, want cleared", got.ActiveError)
}
}
func TestProjectorConvergesConcurrentGitObservations(t *testing.T) {
projector, store, _, _, _ := newProjectorTest(t)
const taskID = "task-concurrent-git"
@@ -67,16 +67,17 @@ type RebuildInput struct {
// replaying any session stream.
func BuildFromAuthoritative(input RebuildInput) TaskStatusSummary {
state := &projectionState{
sessions: make(map[string]sessionObservation, len(input.Sessions)),
pending: make(map[string]string, len(input.PendingActions)),
errors: make(map[string]*ActiveErrorSummary),
git: make(map[string]GitSummary, len(input.Git)),
prs: make(map[string]pullRequestObservation, len(input.PullRequests)),
pendingObserved: true,
activityObserved: input.ActivityObserved,
errorsObserved: true,
gitObserved: input.GitObserved,
prObserved: input.PRObserved,
sessions: make(map[string]sessionObservation, len(input.Sessions)),
pending: make(map[string]string, len(input.PendingActions)),
errors: make(map[string]*ActiveErrorSummary),
clearedErrorStamps: make(map[string]string),
git: make(map[string]GitSummary, len(input.Git)),
prs: make(map[string]pullRequestObservation, len(input.PullRequests)),
pendingObserved: true,
activityObserved: input.ActivityObserved,
errorsObserved: true,
gitObserved: input.GitObserved,
prObserved: input.PRObserved,
}
for _, inputSession := range input.Sessions {
if strings.TrimSpace(inputSession.ID) == "" {