fix(tasks): clear the error icon once the agent recovers (#2391)
Co-authored-by: Kandev Agent <agent@kandev.dev> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1054,6 +1054,10 @@ func (s *Service) handleAgentCompletedLocked(ctx context.Context, data watcher.A
|
||||
s.scheduler.HandleTaskCompleted(data.TaskID, true)
|
||||
s.scheduler.RemoveTask(data.TaskID)
|
||||
|
||||
// The agent finished a turn, so any stored failure no longer describes the
|
||||
// session. `session` was read above, so the guard costs nothing.
|
||||
s.clearRecoveredAgentError(context.WithoutCancel(ctx), data.TaskID, session)
|
||||
|
||||
s.completeTurnForSession(context.WithoutCancel(ctx), data.SessionID)
|
||||
|
||||
if s.sessionHasPendingClarification(ctx, data.SessionID) {
|
||||
@@ -1503,6 +1507,58 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE
|
||||
}
|
||||
}
|
||||
|
||||
// clearRecoveredAgentError drops a session's stored agent failure once the agent
|
||||
// completes a turn, and publishes the inactive error event so open clients drop
|
||||
// the red error affordance.
|
||||
//
|
||||
// persistLastAgentError deliberately keeps the record across a successful turn
|
||||
// as an investigation breadcrumb, but nothing ever retired it, so a failure the
|
||||
// agent recovered from weeks ago still read as live: every path that re-derives
|
||||
// task status from session metadata (a status-summary rebuild, a backend
|
||||
// restart, a later session event) put it straight back. The failure also lands
|
||||
// in the transcript as a recovery message, which is where an investigation
|
||||
// actually looks, so the metadata copy is not the durable record.
|
||||
//
|
||||
// Writes JSON null rather than a delete: LoadLastAgentError already treats that
|
||||
// as absent, so no new repository surface is needed.
|
||||
func (s *Service) clearRecoveredAgentError(ctx context.Context, taskID string, session *models.TaskSession) {
|
||||
if session == nil || session.ID == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := models.LoadLastAgentError(session.Metadata); !ok {
|
||||
return
|
||||
}
|
||||
if err := s.repo.SetSessionMetadataKey(
|
||||
ctx, session.ID, models.SessionMetaKeyLastAgentError, nil,
|
||||
); err != nil {
|
||||
s.logger.Warn("failed to clear recovered agent error",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
// Keep the in-memory copy in step: the session-state publish below reads
|
||||
// its `session_metadata` straight off this object.
|
||||
delete(session.Metadata, models.SessionMetaKeyLastAgentError)
|
||||
if s.eventBus == nil {
|
||||
return
|
||||
}
|
||||
if err := s.eventBus.Publish(ctx, events.TaskSessionErrorChanged, bus.NewEvent(
|
||||
events.TaskSessionErrorChanged,
|
||||
"orchestrator",
|
||||
map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"session_id": session.ID,
|
||||
"active": false,
|
||||
},
|
||||
)); err != nil {
|
||||
s.logger.Warn("failed to publish recovered task session error event",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// createRecoveryStatusMessage builds and persists the ActionMessage shown
|
||||
// in the kanban chat surface after a recoverable agent failure. Must only
|
||||
// be called for non-office sessions (office sessions render their own error UI).
|
||||
|
||||
@@ -2191,6 +2191,79 @@ func TestSessionStartClearsInterruptedMarker(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// Nothing else ever retires a stored agent failure, so without this clear every
|
||||
// path that re-derives task status from session metadata resurrects it and the
|
||||
// error icon never goes away.
|
||||
func TestClearRecoveredAgentErrorOnTurnCompletion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
seedError := func(t *testing.T, repo *sqliterepo.Repository) {
|
||||
t.Helper()
|
||||
require.NoError(t, repo.SetSessionMetadataKey(
|
||||
ctx, "s1", models.SessionMetaKeyLastAgentError,
|
||||
models.LastAgentError{Message: "agent crashed", OccurredAt: time.Now().UTC().Add(-time.Hour)},
|
||||
))
|
||||
}
|
||||
|
||||
errorEvents := func(eb *recordingEventBus) []recordedEvent {
|
||||
var out []recordedEvent
|
||||
for _, recorded := range eb.events {
|
||||
if recorded.event != nil && recorded.event.Type == events.TaskSessionErrorChanged {
|
||||
out = append(out, recorded)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
newService := func(repo *sqliterepo.Repository) (*Service, *recordingEventBus) {
|
||||
eb := &recordingEventBus{}
|
||||
svc := createTestService(repo, newMockStepGetter(), newMockTaskRepo())
|
||||
svc.eventBus = eb
|
||||
return svc, eb
|
||||
}
|
||||
|
||||
t.Run("clears the record and publishes it inactive", func(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "t1", "s1", "step1")
|
||||
seedError(t, repo)
|
||||
svc, eb := newService(repo)
|
||||
|
||||
session, err := repo.GetTaskSession(ctx, "s1")
|
||||
require.NoError(t, err)
|
||||
svc.clearRecoveredAgentError(ctx, "t1", session)
|
||||
|
||||
stored, err := repo.GetTaskSession(ctx, "s1")
|
||||
require.NoError(t, err)
|
||||
_, ok := models.LoadLastAgentError(stored.Metadata)
|
||||
require.False(t, ok, "a recovered failure must not keep driving the error icon")
|
||||
|
||||
// The session-state publish that follows reads session_metadata straight
|
||||
// off this object, so a stale copy would re-arm the icon immediately.
|
||||
_, ok = models.LoadLastAgentError(session.Metadata)
|
||||
require.False(t, ok, "the in-memory copy must be cleared too")
|
||||
|
||||
published := errorEvents(eb)
|
||||
require.Len(t, published, 1, "clients need one event to drop the icon")
|
||||
data, ok := published[0].event.Data.(map[string]interface{})
|
||||
require.True(t, ok)
|
||||
require.Equal(t, false, data["active"])
|
||||
require.Equal(t, "s1", data["session_id"])
|
||||
})
|
||||
|
||||
t.Run("stays silent when the session has no stored error", func(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "t1", "s1", "step1")
|
||||
svc, eb := newService(repo)
|
||||
|
||||
session, err := repo.GetTaskSession(ctx, "s1")
|
||||
require.NoError(t, err)
|
||||
svc.clearRecoveredAgentError(ctx, "t1", session)
|
||||
|
||||
require.Empty(t, errorEvents(eb),
|
||||
"every later completion must stay silent so turns do not churn the row")
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetSessionStartingRejectsTerminalSession(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
|
||||
@@ -662,8 +662,8 @@ const (
|
||||
|
||||
// Workflow represents a task workflow
|
||||
type Workflow struct {
|
||||
ID string `json:"id"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ID string `json:"id"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
// Prompt is optional workflow-level agent instructions prepended at
|
||||
|
||||
@@ -237,9 +237,116 @@ func (r *Repository) runMigrations() error {
|
||||
CREATE INDEX IF NOT EXISTS idx_task_status_summaries_workspace
|
||||
ON task_status_summaries(workspace_id)`)
|
||||
|
||||
if err := r.clearRecoveredAgentErrors(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// clearRecoveredAgentErrors repairs sessions whose stored agent failure was
|
||||
// overtaken by successful work before the orchestrator learned to clear it.
|
||||
//
|
||||
// Nothing used to retire `last_agent_error`, so a failure the agent recovered
|
||||
// from weeks ago still reads as live and keeps a red error icon on the task list
|
||||
// forever. The orchestrator now clears the record on turn completion; this
|
||||
// applies the same rule to history.
|
||||
//
|
||||
// Deliberately narrow: a record is cleared only when the session has an ordinary
|
||||
// agent message newer than the failure — the same "the agent has produced good
|
||||
// output since" signal. A failure with no successful work after it is current
|
||||
// and is left alone.
|
||||
func (r *Repository) clearRecoveredAgentErrors() error {
|
||||
postgres := dialect.IsPostgres(r.db.DriverName())
|
||||
|
||||
sessionError := func(key string) string {
|
||||
return jsonText(postgres, "s.metadata", "last_agent_error", key)
|
||||
}
|
||||
recoveredSessions := `
|
||||
SELECT s.id
|
||||
FROM task_sessions s
|
||||
WHERE ` + sessionError("message") + ` IS NOT NULL
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM task_session_messages m
|
||||
WHERE m.task_session_id = s.id
|
||||
AND m.author_type = 'agent'
|
||||
AND m.type NOT IN ('error', 'status')
|
||||
AND ` + timestampColumn(postgres, "m.created_at") + ` > ` +
|
||||
timestampFromText(postgres, sessionError("occurred_at")) + `
|
||||
)`
|
||||
|
||||
r.migrate.Apply("task_sessions.last_agent_error.recovered_cleanup",
|
||||
`UPDATE task_sessions SET metadata = `+jsonRemoveKey(postgres, "metadata", "last_agent_error")+
|
||||
` WHERE id IN (`+recoveredSessions+`)`)
|
||||
|
||||
// The summary row caches the derived error, so clearing the session record
|
||||
// alone would leave the icon up until that session next emitted an event.
|
||||
// A cached error naming a session that has no stored failure is stale by
|
||||
// definition, which also sweeps up any earlier drift.
|
||||
r.migrate.Apply("task_status_summaries.active_error.recovered_cleanup", `
|
||||
UPDATE task_status_summaries
|
||||
SET summary = `+jsonRemoveKey(postgres, "summary", "active_error")+`
|
||||
WHERE `+jsonText(postgres, "summary", "active_error", "session_id")+` IN (
|
||||
SELECT s.id FROM task_sessions s
|
||||
WHERE `+sessionError("message")+` IS NULL
|
||||
)`)
|
||||
return nil
|
||||
}
|
||||
|
||||
// jsonColumn makes a TEXT-typed JSON column safe to parse. Both dialects raise
|
||||
// on an empty or malformed document — Postgres on the `::jsonb` cast, SQLite in
|
||||
// `json_extract` — and these statements scan every row before filtering, so a
|
||||
// single such row aborts the whole migration. The repository writes ” and
|
||||
// 'null' for "no metadata", so both are normalized the way the rest of the
|
||||
// package already does.
|
||||
func jsonColumn(postgres bool, column string) string {
|
||||
empty, open := "'{}'", ""
|
||||
if postgres {
|
||||
empty, open = "'{}'::jsonb", "::jsonb"
|
||||
}
|
||||
return "(CASE WHEN " + column + " IS NULL OR " + column + " = 'null' OR " + column +
|
||||
" = '' THEN " + empty + " ELSE " + column + open + " END)"
|
||||
}
|
||||
|
||||
// jsonText extracts a nested JSON text value from a TEXT-typed JSON column.
|
||||
func jsonText(postgres bool, column, parent, key string) string {
|
||||
base := jsonColumn(postgres, column)
|
||||
if postgres {
|
||||
return "(" + base + " #>> '{" + parent + "," + key + "}')"
|
||||
}
|
||||
return "json_extract(" + base + ", '$." + parent + "." + key + "')"
|
||||
}
|
||||
|
||||
// timestampColumn normalizes a timestamp column for comparison. SQLite stores
|
||||
// timestamps as text and would compare them lexically, so it needs Julian days;
|
||||
// on Postgres the column is already a timestamp.
|
||||
func timestampColumn(postgres bool, column string) string {
|
||||
if postgres {
|
||||
return "(" + column + ")::timestamptz"
|
||||
}
|
||||
return "julianday(" + column + ")"
|
||||
}
|
||||
|
||||
// timestampFromText normalizes a timestamp extracted from JSON so it compares
|
||||
// against timestampColumn — the two carry different text shapes
|
||||
// ('2026-08-01 10:00:00+00:00' vs RFC3339 '2026-08-01T10:00:00Z'). NULLIF keeps
|
||||
// an empty stored timestamp from raising on the Postgres cast; it then compares
|
||||
// as NULL, so the row simply does not match.
|
||||
func timestampFromText(postgres bool, expression string) string {
|
||||
if postgres {
|
||||
return "(NULLIF(" + expression + ", '')::timestamptz)"
|
||||
}
|
||||
return "julianday(" + expression + ")"
|
||||
}
|
||||
|
||||
func jsonRemoveKey(postgres bool, column, key string) string {
|
||||
base := jsonColumn(postgres, column)
|
||||
if postgres {
|
||||
return "(" + base + " - '" + key + "')::text"
|
||||
}
|
||||
return "json_remove(" + base + ", '$." + key + "')"
|
||||
}
|
||||
|
||||
// ensureImproveKandevWorkflowTemplateUniqueness removes the broad index from
|
||||
// the initial implementation, reconciles legacy bootstrap duplicates, then
|
||||
// enforces uniqueness only for the two hidden workflows created by this flow.
|
||||
|
||||
@@ -696,3 +696,112 @@ func TestPostgresSetSessionPrimary_ConcurrentPromotionsLeaveExactlyOnePrimary(t
|
||||
t.Errorf("expected exactly 1 primary session after %d concurrent promotions, got %d — FOR UPDATE lock not serializing across connections", concurrency, primaryCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostgresClearRecoveredAgentErrors is the Postgres counterpart to
|
||||
// TestClearRecoveredAgentErrorsBackfill. clearRecoveredAgentErrors is built from
|
||||
// three dialect-sensitive helpers (jsonColumn, jsonText/timestamp*,
|
||||
// jsonRemoveKey), so ADR 0027 asks for env-gated Postgres behavior coverage —
|
||||
// schema replay would not exercise the jsonb operators at all.
|
||||
//
|
||||
// migrate.Apply swallows SQL errors, so the assertions below are the only proof
|
||||
// the statements ran: a dialect mistake leaves the recovered row untouched
|
||||
// rather than returning an error. Skips unless KANDEV_TEST_POSTGRES_DSN is set.
|
||||
func TestPostgresClearRecoveredAgentErrors(t *testing.T) {
|
||||
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
|
||||
repo, err := NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init postgres schema: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC)
|
||||
if _, err := db.Exec(db.Rebind(`
|
||||
INSERT INTO tasks (id, workspace_id, title, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`), "task-pg-recovered", "ws-pg-recovered", "Task", occurredAt, occurredAt); err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
// recovered: agent output after the failure. current: none since.
|
||||
// blank/nulled: rows the repository wrote as "no metadata" — both dialects
|
||||
// raise on parsing those, and the statements scan every row before
|
||||
// filtering, so an unguarded cast aborts the migration for everyone.
|
||||
for _, sessionID := range []string{"pg-recovered", "pg-current", "pg-blank", "pg-nulled"} {
|
||||
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
|
||||
ID: sessionID, TaskID: "task-pg-recovered", State: models.TaskSessionStateWaitingForInput,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession %s: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
// Seeded with plain SQL on purpose: SetSessionMetadataKey is SQLite-only
|
||||
// (json_set/json()), so using it here would fail on the seed rather than
|
||||
// exercise the migration under test.
|
||||
lastAgentError := `{"last_agent_error":{"message":"agent crashed","occurred_at":"` +
|
||||
occurredAt.Format(time.RFC3339Nano) + `"}}`
|
||||
for _, sessionID := range []string{"pg-recovered", "pg-current"} {
|
||||
if _, err := db.Exec(db.Rebind(
|
||||
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), lastAgentError, sessionID); err != nil {
|
||||
t.Fatalf("seed last agent error on %s: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
for value, sessionID := range map[string]string{"": "pg-blank", "null": "pg-nulled"} {
|
||||
if _, err := db.Exec(db.Rebind(
|
||||
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), value, sessionID); err != nil {
|
||||
t.Fatalf("seed %q metadata: %v", value, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := db.Exec(db.Rebind(`
|
||||
INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`), "pg-turn", "pg-recovered", "task-pg-recovered", occurredAt, occurredAt, occurredAt); err != nil {
|
||||
t.Fatalf("seed turn: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(db.Rebind(`
|
||||
INSERT INTO task_session_messages
|
||||
(id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at)
|
||||
VALUES (?, ?, '', ?, 'agent', 'back on track', 'message', '{}', ?)
|
||||
`), "pg-msg", "pg-recovered", "pg-turn", occurredAt.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("seed agent message: %v", err)
|
||||
}
|
||||
|
||||
if _, err := db.Exec(db.Rebind(`
|
||||
INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at)
|
||||
VALUES (?, ?, 4, ?, ?)
|
||||
`), "task-pg-recovered", "ws-pg-recovered",
|
||||
`{"active_error":{"session_id":"pg-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`,
|
||||
occurredAt); err != nil {
|
||||
t.Fatalf("seed status summary: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.clearRecoveredAgentErrors(); err != nil {
|
||||
t.Fatalf("clearRecoveredAgentErrors: %v", err)
|
||||
}
|
||||
|
||||
hasError := func(sessionID string) bool {
|
||||
session, err := repo.GetTaskSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("get session %s: %v", sessionID, err)
|
||||
}
|
||||
_, ok := models.LoadLastAgentError(session.Metadata)
|
||||
return ok
|
||||
}
|
||||
if hasError("pg-recovered") {
|
||||
t.Fatal("a failure the agent recovered from must be cleared on Postgres")
|
||||
}
|
||||
if !hasError("pg-current") {
|
||||
t.Fatal("a failure with no successful work after it must be left alone")
|
||||
}
|
||||
|
||||
var summary string
|
||||
if err := db.Get(&summary, db.Rebind(
|
||||
`SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-pg-recovered"); err != nil {
|
||||
t.Fatalf("read status summary: %v", err)
|
||||
}
|
||||
if strings.Contains(summary, "active_error") {
|
||||
t.Fatalf("summary = %s, want the cached error cleared", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "permission") {
|
||||
t.Fatalf("summary = %s, want the rest of the projection preserved", summary)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -1494,6 +1496,102 @@ func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing used to retire a stored agent failure, so installations carry records
|
||||
// whose errors the agent recovered from long ago — each one still painting a red
|
||||
// icon on the task list.
|
||||
func TestClearRecoveredAgentErrorsBackfill(t *testing.T) {
|
||||
repo := newRepoForSessionTests(t)
|
||||
ctx := context.Background()
|
||||
|
||||
occurredAt := time.Date(2026, 6, 14, 10, 0, 0, 0, time.UTC)
|
||||
seedStale := func(t *testing.T, taskID, sessionID, turnID string) {
|
||||
t.Helper()
|
||||
seedForMsgTest(t, repo, taskID, sessionID, turnID)
|
||||
if err := repo.SetSessionMetadataKey(ctx, sessionID, models.SessionMetaKeyLastAgentError,
|
||||
models.LastAgentError{Message: "agent crashed", OccurredAt: occurredAt}); err != nil {
|
||||
t.Fatalf("seed last agent error on %s: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovered: an ordinary agent message lands after the failure.
|
||||
seedStale(t, "task-recovered", "sess-recovered", "turn-recovered")
|
||||
insertAgentMsg(t, repo, "msg-recovered", "sess-recovered", "turn-recovered",
|
||||
"agent", "back on track", occurredAt.Add(time.Hour))
|
||||
|
||||
// Still failing: nothing successful has happened since.
|
||||
seedStale(t, "task-current", "sess-current", "turn-current")
|
||||
|
||||
// Recovery that predates the failure must not count.
|
||||
seedStale(t, "task-older", "sess-older", "turn-older")
|
||||
insertAgentMsg(t, repo, "msg-older", "sess-older", "turn-older",
|
||||
"agent", "earlier output", occurredAt.Add(-time.Hour))
|
||||
|
||||
// A user message after the failure is not the agent producing good output.
|
||||
seedStale(t, "task-user", "sess-user", "turn-user")
|
||||
insertAgentMsg(t, repo, "msg-user", "sess-user", "turn-user",
|
||||
"user", "are you stuck?", occurredAt.Add(time.Hour))
|
||||
|
||||
// These statements scan every session before filtering, so a row the
|
||||
// repository wrote as "no metadata" must not abort them. migrate.Apply
|
||||
// swallows SQL errors, so the proof is that the recovered session below is
|
||||
// still cleared with these rows present — an aborted statement would leave
|
||||
// it untouched.
|
||||
for index, blank := range []string{"", "null"} {
|
||||
id := fmt.Sprintf("sess-blank-%d", index)
|
||||
seedForMsgTest(t, repo, "task-blank-"+strconv.Itoa(index), id, "turn-blank-"+strconv.Itoa(index))
|
||||
if _, err := repo.db.Exec(repo.db.Rebind(
|
||||
`UPDATE task_sessions SET metadata = ? WHERE id = ?`), blank, id); err != nil {
|
||||
t.Fatalf("seed %q metadata: %v", blank, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The cached summary keeps the icon up until it is cleared too.
|
||||
if _, err := repo.db.Exec(repo.db.Rebind(`
|
||||
INSERT INTO task_status_summaries (task_id, workspace_id, revision, summary, updated_at)
|
||||
VALUES (?, '', 4, ?, ?)
|
||||
`), "task-recovered",
|
||||
`{"active_error":{"session_id":"sess-recovered","stamp":"s","preview":"agent crashed"},"pending_action":"permission"}`,
|
||||
time.Now().UTC()); err != nil {
|
||||
t.Fatalf("seed status summary: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.clearRecoveredAgentErrors(); err != nil {
|
||||
t.Fatalf("clearRecoveredAgentErrors: %v", err)
|
||||
}
|
||||
|
||||
hasError := func(t *testing.T, sessionID string) bool {
|
||||
t.Helper()
|
||||
session, err := repo.GetTaskSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("get session %s: %v", sessionID, err)
|
||||
}
|
||||
_, ok := models.LoadLastAgentError(session.Metadata)
|
||||
return ok
|
||||
}
|
||||
|
||||
if hasError(t, "sess-recovered") {
|
||||
t.Fatalf("a failure the agent recovered from must be cleared")
|
||||
}
|
||||
for _, sessionID := range []string{"sess-current", "sess-older", "sess-user"} {
|
||||
if !hasError(t, sessionID) {
|
||||
t.Fatalf("%s has no successful agent work after the failure; it must stay", sessionID)
|
||||
}
|
||||
}
|
||||
|
||||
var summary string
|
||||
if err := repo.db.QueryRow(repo.db.Rebind(
|
||||
`SELECT summary FROM task_status_summaries WHERE task_id = ?`), "task-recovered",
|
||||
).Scan(&summary); err != nil {
|
||||
t.Fatalf("read status summary: %v", err)
|
||||
}
|
||||
if strings.Contains(summary, "active_error") {
|
||||
t.Fatalf("summary = %s, want the cached error cleared", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "permission") {
|
||||
t.Fatalf("summary = %s, want the rest of the projection preserved", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID string) (string, sql.NullTime, time.Time) {
|
||||
t.Helper()
|
||||
var errorMessage string
|
||||
|
||||
@@ -151,3 +151,50 @@ func TestHydrateMissingTaskStatusSummariesRepairsExistingTaskOnce(t *testing.T)
|
||||
t.Fatalf("PR reader calls after existing summary = %d, want one", prReader.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// The rebuild path is what runs after a restart, so it is where a stale record
|
||||
// used to come back. A session whose error was cleared must rebuild clean.
|
||||
func TestHydrateMissingTaskStatusSummariesSkipsClearedAgentErrors(t *testing.T) {
|
||||
svc, _, repo := createTestService(t)
|
||||
ctx := context.Background()
|
||||
createTaskWithoutRepositories(t, ctx, repo)
|
||||
|
||||
session := &models.TaskSession{
|
||||
ID: "session-1",
|
||||
TaskID: "task-1",
|
||||
State: models.TaskSessionStateWaitingForInput,
|
||||
IsPrimary: true,
|
||||
}
|
||||
if err := repo.CreateTaskSession(ctx, session); err != nil {
|
||||
t.Fatalf("CreateTaskSession: %v", err)
|
||||
}
|
||||
// The orchestrator writes JSON null on turn completion rather than deleting
|
||||
// the key, so the rebuild must treat that as "no failure".
|
||||
if err := repo.SetSessionMetadataKey(ctx, session.ID, models.SessionMetaKeyLastAgentError, nil); err != nil {
|
||||
t.Fatalf("clear last agent error: %v", err)
|
||||
}
|
||||
stored, err := repo.GetTaskSession(ctx, session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession: %v", err)
|
||||
}
|
||||
|
||||
svc.statusSummaries = repo
|
||||
task := &models.Task{ID: "task-1", WorkspaceID: "ws-1"}
|
||||
got, err := svc.HydrateMissingTaskStatusSummaries(
|
||||
ctx,
|
||||
[]*models.Task{task},
|
||||
map[string][]*models.TaskSession{task.ID: {stored}},
|
||||
map[string]models.TaskPendingAction{},
|
||||
map[string]*statussummary.TaskStatusSummary{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("HydrateMissingTaskStatusSummaries: %v", err)
|
||||
}
|
||||
summary := got[task.ID]
|
||||
if summary == nil {
|
||||
t.Fatal("repaired summary missing from returned map")
|
||||
}
|
||||
if summary.ActiveError != nil {
|
||||
t.Fatalf("active error = %+v, want a cleared record to stay invisible", summary.ActiveError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,13 +101,17 @@ type projectionState struct {
|
||||
pendingObserved bool
|
||||
activityObserved bool
|
||||
errors map[string]*ActiveErrorSummary
|
||||
errorsObserved bool
|
||||
git map[string]GitSummary
|
||||
gitBaseline *GitSummary
|
||||
gitObserved bool
|
||||
prs map[string]pullRequestObservation
|
||||
prBaseline *PullRequestSummary
|
||||
prObserved bool
|
||||
// clearedErrorStamps records, per session, the stamp of the last error this
|
||||
// projection cleared, so a durable breadcrumb replayed on a later session
|
||||
// event cannot re-arm an error affordance the agent already recovered from.
|
||||
clearedErrorStamps map[string]string
|
||||
errorsObserved bool
|
||||
git map[string]GitSummary
|
||||
gitBaseline *GitSummary
|
||||
gitObserved bool
|
||||
prs map[string]pullRequestObservation
|
||||
prBaseline *PullRequestSummary
|
||||
prObserved bool
|
||||
}
|
||||
|
||||
type sessionObservation struct {
|
||||
@@ -463,11 +467,12 @@ func updateSessionPendingLocked(state *projectionState, sessionID, action string
|
||||
|
||||
func newProjectionState() *projectionState {
|
||||
return &projectionState{
|
||||
sessions: make(map[string]sessionObservation),
|
||||
pending: make(map[string]string),
|
||||
errors: make(map[string]*ActiveErrorSummary),
|
||||
git: make(map[string]GitSummary),
|
||||
prs: make(map[string]pullRequestObservation),
|
||||
sessions: make(map[string]sessionObservation),
|
||||
pending: make(map[string]string),
|
||||
errors: make(map[string]*ActiveErrorSummary),
|
||||
clearedErrorStamps: make(map[string]string),
|
||||
git: make(map[string]GitSummary),
|
||||
prs: make(map[string]pullRequestObservation),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -591,15 +596,36 @@ func (p *Projector) applySessionEventLocked(state *projectionState, data map[str
|
||||
|
||||
changed := true
|
||||
if metadata, ok := data["session_metadata"].(map[string]interface{}); ok {
|
||||
if errSummary, present := errorFromMetadata(p.now().UTC(), sessionID, metadata); present {
|
||||
state.errorsObserved = true
|
||||
changed = !errorEqual(state.errors[sessionID], errSummary) || changed
|
||||
state.errors[sessionID] = errSummary
|
||||
}
|
||||
p.applySessionMetadataErrorLocked(state, sessionID, metadata)
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// applySessionMetadataErrorLocked folds the session's durable error record into
|
||||
// the projection. A dismissed or superseded record clears the affordance, and a
|
||||
// record this projection already cleared is not re-applied — session metadata
|
||||
// keeps failures as a breadcrumb, so replaying it verbatim would re-arm errors
|
||||
// the agent has since recovered from.
|
||||
func (p *Projector) applySessionMetadataErrorLocked(
|
||||
state *projectionState,
|
||||
sessionID string,
|
||||
metadata map[string]interface{},
|
||||
) {
|
||||
errSummary, observed := errorFromMetadata(p.now().UTC(), sessionID, metadata)
|
||||
if !observed {
|
||||
return
|
||||
}
|
||||
state.errorsObserved = true
|
||||
if errSummary == nil {
|
||||
p.clearErrorLocked(state, sessionID)
|
||||
return
|
||||
}
|
||||
if state.clearedErrorStamps[sessionID] == errSummary.Stamp {
|
||||
return
|
||||
}
|
||||
state.errors[sessionID] = errSummary
|
||||
}
|
||||
|
||||
func (p *Projector) applyActivityEventLocked(state *projectionState, data map[string]interface{}) bool {
|
||||
sessionID := stringField(data, "session_id")
|
||||
if sessionID == "" {
|
||||
@@ -635,6 +661,10 @@ func (p *Projector) applyErrorEventLocked(state *projectionState, data map[strin
|
||||
if errorEqual(state.errors[sessionID], errSummary) {
|
||||
return false
|
||||
}
|
||||
// An explicit error event is authoritative: it re-arms even a stamp this
|
||||
// projection cleared earlier, which is what makes an identical failure
|
||||
// recurring after a recovery visible again.
|
||||
delete(state.clearedErrorStamps, sessionID)
|
||||
state.errors[sessionID] = errSummary
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -37,19 +37,51 @@ func recomputeTaskPending(state *projectionState) {
|
||||
|
||||
func (p *Projector) clearErrorLocked(state *projectionState, sessionID string) bool {
|
||||
state.errorsObserved = true
|
||||
if _, ok := state.errors[sessionID]; !ok {
|
||||
existing, ok := state.errors[sessionID]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
// Remember which error was cleared. Session events carry a `session_metadata`
|
||||
// snapshot taken when the publisher read the session, which can predate the
|
||||
// clear — without this, such an event puts the very same error straight back.
|
||||
// A genuinely new failure has a different stamp and is still applied.
|
||||
if existing != nil && existing.Stamp != "" {
|
||||
state.clearedErrorStamps[sessionID] = existing.Stamp
|
||||
}
|
||||
delete(state.errors, sessionID)
|
||||
return true
|
||||
}
|
||||
|
||||
func errorFromMetadata(now time.Time, sessionID string, metadata map[string]interface{}) (*ActiveErrorSummary, bool) {
|
||||
raw, ok := metadata["last_agent_error"].(map[string]interface{})
|
||||
if !ok {
|
||||
// errorFromMetadata reads the session's stored error record. `observed` is true
|
||||
// whenever the `last_agent_error` key is present at all: a dismissed record, or
|
||||
// the JSON null the orchestrator writes when a turn completes, is an
|
||||
// authoritative "no active error" for that session rather than an absence of
|
||||
// information, so the caller must clear rather than ignore it.
|
||||
//
|
||||
// The key must be tested for presence before its type: a JSON null arrives as a
|
||||
// nil value, and asserting `.(map[string]interface{})` on that is
|
||||
// indistinguishable from the key being missing entirely. Reading it as missing
|
||||
// would leave a restored error armed forever on any session whose record was
|
||||
// cleared.
|
||||
func errorFromMetadata(
|
||||
now time.Time,
|
||||
sessionID string,
|
||||
metadata map[string]interface{},
|
||||
) (summary *ActiveErrorSummary, observed bool) {
|
||||
value, present := metadata["last_agent_error"]
|
||||
if !present {
|
||||
return nil, false
|
||||
}
|
||||
return errorFromMap(now, sessionID, raw)
|
||||
raw, ok := value.(map[string]interface{})
|
||||
if !ok {
|
||||
// Null or malformed: the key exists, so the session has no active error.
|
||||
return nil, true
|
||||
}
|
||||
active, ok := errorFromMap(now, sessionID, raw)
|
||||
if !ok {
|
||||
return nil, true
|
||||
}
|
||||
return active, true
|
||||
}
|
||||
|
||||
func errorFromMap(now time.Time, sessionID string, data map[string]interface{}) (*ActiveErrorSummary, bool) {
|
||||
|
||||
@@ -279,6 +279,174 @@ func TestProjectorDerivesBoundedStatusAcrossSources(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Session events carry a `session_metadata` snapshot taken when the publisher
|
||||
// read the session, so one taken before the error was cleared must not re-arm an
|
||||
// error this projection already cleared.
|
||||
func TestProjectorDoesNotResurrectClearedErrorFromSessionMetadata(t *testing.T) {
|
||||
_, store, eventBus, _, _ := newProjectorTest(t)
|
||||
const taskID = "task-error-replay"
|
||||
const sessionID = "session-error-replay"
|
||||
|
||||
occurredAt := "2026-08-01T18:01:00.000000000Z"
|
||||
breadcrumb := map[string]interface{}{
|
||||
"last_agent_error": map[string]interface{}{
|
||||
"message": "agent crashed",
|
||||
"occurred_at": occurredAt,
|
||||
},
|
||||
}
|
||||
|
||||
publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{
|
||||
"session_metadata": breadcrumb,
|
||||
})
|
||||
if got := store.summary(taskID); got == nil || got.ActiveError == nil {
|
||||
t.Fatalf("active error after failure = %+v", got)
|
||||
}
|
||||
|
||||
// The agent recovers and posts an ordinary message.
|
||||
publishProjectorEvent(t, eventBus, events.MessageAdded, events.MessageAdded, map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"session_id": sessionID,
|
||||
"author_type": "agent",
|
||||
"type": "text",
|
||||
"content": "recovered",
|
||||
})
|
||||
if got := store.summary(taskID); got.ActiveError != nil {
|
||||
t.Fatalf("active error after recovery = %+v, want cleared", got.ActiveError)
|
||||
}
|
||||
|
||||
// Turn end republishes the session, metadata breadcrumb and all.
|
||||
publishSessionState(t, eventBus, taskID, sessionID, map[string]interface{}{
|
||||
"new_state": "WAITING_FOR_INPUT",
|
||||
"session_metadata": breadcrumb,
|
||||
})
|
||||
if got := store.summary(taskID); got.ActiveError != nil {
|
||||
t.Fatalf("active error after metadata replay = %+v, want it to stay cleared", got.ActiveError)
|
||||
}
|
||||
|
||||
// A genuinely new failure is still authoritative.
|
||||
publishProjectorEvent(t, eventBus, events.TaskSessionErrorChanged, events.TaskSessionErrorChanged, map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"session_id": sessionID,
|
||||
"active": true,
|
||||
"message": "agent crashed again",
|
||||
"occurred_at": "2026-08-01T18:05:00.000000000Z",
|
||||
"stamp": "error-stamp-2",
|
||||
})
|
||||
got := store.summary(taskID)
|
||||
if got.ActiveError == nil || got.ActiveError.Stamp != "error-stamp-2" {
|
||||
t.Fatalf("active error after a new failure = %+v, want the new error", got.ActiveError)
|
||||
}
|
||||
}
|
||||
|
||||
// A retired record is an authoritative "no active error" for the session, so it
|
||||
// must clear an error the projection restored from its persisted row rather than
|
||||
// being read as an absence of information.
|
||||
func TestProjectorClearsErrorWhenSessionMetadataIsRetired(t *testing.T) {
|
||||
store := newProjectorTestStore()
|
||||
storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC)
|
||||
store.rows["task-superseded"] = &StoredTaskStatusSummary{
|
||||
TaskID: "task-superseded",
|
||||
WorkspaceID: "workspace-1",
|
||||
Summary: TaskStatusSummary{
|
||||
Revision: 3,
|
||||
UpdatedAt: storedAt,
|
||||
PrimarySession: &PrimarySessionSummary{ID: "session-superseded", State: "RUNNING"},
|
||||
ActiveError: &ActiveErrorSummary{
|
||||
SessionID: "session-superseded",
|
||||
Stamp: "error-stored",
|
||||
OccurredAt: storedAt,
|
||||
Preview: "agent crashed",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
projector := NewProjector(ProjectorConfig{
|
||||
Store: store,
|
||||
ResolveWorkspace: func(context.Context, string) (string, error) {
|
||||
return "workspace-1", nil
|
||||
},
|
||||
Now: func() time.Time { return storedAt.Add(time.Minute) },
|
||||
})
|
||||
|
||||
if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{
|
||||
"task_id": "task-superseded",
|
||||
"workspace_id": "workspace-1",
|
||||
"session_id": "session-superseded",
|
||||
"new_state": "WAITING_FOR_INPUT",
|
||||
"is_primary": true,
|
||||
"session_metadata": map[string]interface{}{
|
||||
"last_agent_error": map[string]interface{}{
|
||||
"message": "agent crashed",
|
||||
"occurred_at": storedAt.Format(time.RFC3339Nano),
|
||||
"dismissed_at": storedAt.Add(30 * time.Second).Format(time.RFC3339Nano),
|
||||
},
|
||||
},
|
||||
})); err != nil {
|
||||
t.Fatalf("replay session event: %v", err)
|
||||
}
|
||||
|
||||
got := store.summary("task-superseded")
|
||||
if got == nil {
|
||||
t.Fatal("summary disappeared")
|
||||
}
|
||||
if got.ActiveError != nil {
|
||||
t.Fatalf("active error after retired metadata = %+v, want cleared", got.ActiveError)
|
||||
}
|
||||
}
|
||||
|
||||
// The orchestrator clears a recovered failure by writing JSON null, which
|
||||
// round-trips as a nil value under an existing key. Reading that as "no
|
||||
// information" instead of "no active error" would leave a restored summary's
|
||||
// error armed forever — the restart path this whole fix is about.
|
||||
func TestProjectorClearsErrorWhenSessionMetadataIsNull(t *testing.T) {
|
||||
store := newProjectorTestStore()
|
||||
storedAt := time.Date(2026, 8, 1, 17, 59, 0, 0, time.UTC)
|
||||
store.rows["task-null"] = &StoredTaskStatusSummary{
|
||||
TaskID: "task-null",
|
||||
WorkspaceID: "workspace-1",
|
||||
Summary: TaskStatusSummary{
|
||||
Revision: 3,
|
||||
UpdatedAt: storedAt,
|
||||
PrimarySession: &PrimarySessionSummary{ID: "session-null", State: "RUNNING"},
|
||||
ActiveError: &ActiveErrorSummary{
|
||||
SessionID: "session-null",
|
||||
Stamp: "error-stored",
|
||||
OccurredAt: storedAt,
|
||||
Preview: "agent crashed",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
projector := NewProjector(ProjectorConfig{
|
||||
Store: store,
|
||||
ResolveWorkspace: func(context.Context, string) (string, error) {
|
||||
return "workspace-1", nil
|
||||
},
|
||||
Now: func() time.Time { return storedAt.Add(time.Minute) },
|
||||
})
|
||||
|
||||
if err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{
|
||||
"task_id": "task-null",
|
||||
"workspace_id": "workspace-1",
|
||||
"session_id": "session-null",
|
||||
"new_state": "WAITING_FOR_INPUT",
|
||||
"is_primary": true,
|
||||
"session_metadata": map[string]interface{}{
|
||||
"last_agent_error": nil,
|
||||
},
|
||||
})); err != nil {
|
||||
t.Fatalf("replay session event: %v", err)
|
||||
}
|
||||
|
||||
got := store.summary("task-null")
|
||||
if got == nil {
|
||||
t.Fatal("summary disappeared")
|
||||
}
|
||||
if got.ActiveError != nil {
|
||||
t.Fatalf("active error after null metadata = %+v, want cleared", got.ActiveError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectorConvergesConcurrentGitObservations(t *testing.T) {
|
||||
projector, store, _, _, _ := newProjectorTest(t)
|
||||
const taskID = "task-concurrent-git"
|
||||
|
||||
@@ -67,16 +67,17 @@ type RebuildInput struct {
|
||||
// replaying any session stream.
|
||||
func BuildFromAuthoritative(input RebuildInput) TaskStatusSummary {
|
||||
state := &projectionState{
|
||||
sessions: make(map[string]sessionObservation, len(input.Sessions)),
|
||||
pending: make(map[string]string, len(input.PendingActions)),
|
||||
errors: make(map[string]*ActiveErrorSummary),
|
||||
git: make(map[string]GitSummary, len(input.Git)),
|
||||
prs: make(map[string]pullRequestObservation, len(input.PullRequests)),
|
||||
pendingObserved: true,
|
||||
activityObserved: input.ActivityObserved,
|
||||
errorsObserved: true,
|
||||
gitObserved: input.GitObserved,
|
||||
prObserved: input.PRObserved,
|
||||
sessions: make(map[string]sessionObservation, len(input.Sessions)),
|
||||
pending: make(map[string]string, len(input.PendingActions)),
|
||||
errors: make(map[string]*ActiveErrorSummary),
|
||||
clearedErrorStamps: make(map[string]string),
|
||||
git: make(map[string]GitSummary, len(input.Git)),
|
||||
prs: make(map[string]pullRequestObservation, len(input.PullRequests)),
|
||||
pendingObserved: true,
|
||||
activityObserved: input.ActivityObserved,
|
||||
errorsObserved: true,
|
||||
gitObserved: input.GitObserved,
|
||||
prObserved: input.PRObserved,
|
||||
}
|
||||
for _, inputSession := range input.Sessions {
|
||||
if strings.TrimSpace(inputSession.ID) == "" {
|
||||
|
||||
Reference in New Issue
Block a user