diff --git a/apps/backend/internal/agent/runtime/agentctl/availability.go b/apps/backend/internal/agent/runtime/agentctl/availability.go new file mode 100644 index 0000000000..fa3946e96b --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/availability.go @@ -0,0 +1,115 @@ +package client + +import ( + "context" + "sync" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "go.uber.org/zap" +) + +const ( + AvailabilityStatusAvailable = "available" + AvailabilityStatusUnavailable = "unavailable" + AvailabilityReasonAgentctlExited = "agentctl_exited" +) + +// AvailabilitySnapshot is the sanitized, install-wide agent runtime state +// exposed to authenticated clients. It intentionally contains no process or +// launcher details. +type AvailabilitySnapshot struct { + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + OccurredAt *time.Time `json:"occurred_at,omitempty"` +} + +// Availability owns the monotonic runtime availability state. A runtime is +// unpublished until its health check and authentication handshake succeed; +// after that point, an unexpected launcher exit is a terminal unavailable +// state for the lifetime of the backend process. +type Availability struct { + mu sync.RWMutex + publishMu sync.Mutex + eventBus bus.EventBus + logger *logger.Logger + snapshot AvailabilitySnapshot + published bool +} + +// NewAvailability creates an unpublished runtime availability tracker. +func NewAvailability(eventBus bus.EventBus, log *logger.Logger) *Availability { + return &Availability{eventBus: eventBus, logger: log} +} + +// Snapshot returns a copy of the current published state. +func (a *Availability) Snapshot() (AvailabilitySnapshot, bool) { + a.mu.RLock() + defer a.mu.RUnlock() + if !a.published { + return AvailabilitySnapshot{}, false + } + + snapshot := a.snapshot + if a.snapshot.OccurredAt != nil { + occurredAt := *a.snapshot.OccurredAt + snapshot.OccurredAt = &occurredAt + } + return snapshot, true +} + +// MarkAvailable publishes the first healthy-and-authenticated state. +func (a *Availability) MarkAvailable() { + a.publishMu.Lock() + defer a.publishMu.Unlock() + + a.mu.Lock() + if a.published { + a.mu.Unlock() + return + } + a.snapshot = AvailabilitySnapshot{Status: AvailabilityStatusAvailable} + a.published = true + snapshot := a.snapshot + a.mu.Unlock() + + a.publish(snapshot) +} + +// MarkUnavailable records an unexpected agentctl exit. The transition is +// intentionally fail-closed and cannot be reversed until the backend restarts. +func (a *Availability) MarkUnavailable() { + a.publishMu.Lock() + defer a.publishMu.Unlock() + + a.mu.Lock() + if !a.published || a.snapshot.Status == AvailabilityStatusUnavailable { + a.mu.Unlock() + return + } + now := time.Now().UTC() + a.snapshot = AvailabilitySnapshot{ + Status: AvailabilityStatusUnavailable, + Reason: AvailabilityReasonAgentctlExited, + OccurredAt: &now, + } + snapshot := a.snapshot + a.mu.Unlock() + + a.publish(snapshot) +} + +func (a *Availability) publish(snapshot AvailabilitySnapshot) { + if a.eventBus == nil { + return + } + if err := a.eventBus.Publish( + context.Background(), + events.AgentRuntimeAvailabilityChanged, + bus.NewEvent(events.AgentRuntimeAvailabilityChanged, "agentctl", snapshot), + ); err != nil && a.logger != nil { + a.logger.Warn("failed to publish agent runtime availability", zap.Error(err)) + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/availability_test.go b/apps/backend/internal/agent/runtime/agentctl/availability_test.go new file mode 100644 index 0000000000..8e01f41ff4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/availability_test.go @@ -0,0 +1,162 @@ +package client + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +func TestAvailabilityPublishesSanitizedMonotonicSnapshots(t *testing.T) { + eventBus := bus.NewMemoryEventBus(newTestLogger()) + t.Cleanup(eventBus.Close) + var published []*bus.Event + _, err := eventBus.Subscribe(events.AgentRuntimeAvailabilityChanged, func(_ context.Context, event *bus.Event) error { + published = append(published, event) + return nil + }) + if err != nil { + t.Fatalf("subscribe availability events: %v", err) + } + + availability := NewAvailability(eventBus, newTestLogger()) + if _, ok := availability.Snapshot(); ok { + t.Fatal("startup availability must not be published before health and auth") + } + + availability.MarkAvailable() + snapshot, ok := availability.Snapshot() + if !ok || snapshot.Status != AvailabilityStatusAvailable { + t.Fatalf("available snapshot = %+v, published=%v", snapshot, ok) + } + if snapshot.Reason != "" || snapshot.OccurredAt != nil { + t.Fatalf("available snapshot exposes failure details: %+v", snapshot) + } + + availability.MarkUnavailable() + unavailable, ok := availability.Snapshot() + if !ok || unavailable.Status != AvailabilityStatusUnavailable { + t.Fatalf("unavailable snapshot = %+v, published=%v", unavailable, ok) + } + if unavailable.Reason != AvailabilityReasonAgentctlExited || unavailable.OccurredAt == nil { + t.Fatalf("unavailable snapshot = %+v", unavailable) + } + occurredAt := *unavailable.OccurredAt + + availability.MarkUnavailable() + availability.MarkAvailable() + final, ok := availability.Snapshot() + if !ok || final.Status != AvailabilityStatusUnavailable || final.Reason != AvailabilityReasonAgentctlExited { + t.Fatalf("availability was not monotonic: %+v, published=%v", final, ok) + } + if final.OccurredAt == nil || !final.OccurredAt.Equal(occurredAt) { + t.Fatalf("unavailable occurrence changed: got %v, want %v", final.OccurredAt, occurredAt) + } + if len(published) != 2 { + t.Fatalf("published event count = %d, want available and unavailable", len(published)) + } +} + +func TestAvailabilitySnapshotReadsAreSafeDuringTransition(t *testing.T) { + availability := NewAvailability(nil, newTestLogger()) + availability.MarkAvailable() + + start := make(chan struct{}) + var wg sync.WaitGroup + for index := 0; index < 16; index++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for count := 0; count < 1000; count++ { + _, _ = availability.Snapshot() + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + availability.MarkUnavailable() + }() + close(start) + wg.Wait() + + snapshot, ok := availability.Snapshot() + if !ok || snapshot.Status != AvailabilityStatusUnavailable { + t.Fatalf("final snapshot = %+v, published=%v", snapshot, ok) + } +} + +func TestAvailabilityPublishesTransitionsInCommitOrder(t *testing.T) { + eventBus := bus.NewMemoryEventBus(newTestLogger()) + t.Cleanup(eventBus.Close) + var published []AvailabilitySnapshot + var publishedMu sync.Mutex + firstPublishStarted := make(chan struct{}) + releaseFirstPublish := make(chan struct{}) + var publishCalls atomic.Int32 + _, err := eventBus.Subscribe(events.AgentRuntimeAvailabilityChanged, func(_ context.Context, event *bus.Event) error { + if event.Data == nil { + return nil + } + if publishCalls.Add(1) == 1 { + close(firstPublishStarted) + <-releaseFirstPublish + } + snapshot, ok := event.Data.(AvailabilitySnapshot) + if !ok { + t.Fatalf("availability event data = %T, want AvailabilitySnapshot", event.Data) + } + publishedMu.Lock() + published = append(published, snapshot) + publishedMu.Unlock() + return nil + }) + if err != nil { + t.Fatalf("subscribe availability events: %v", err) + } + + availability := NewAvailability(eventBus, newTestLogger()) + availableDone := make(chan struct{}) + go func() { + availability.MarkAvailable() + close(availableDone) + }() + <-firstPublishStarted + + unavailableDone := make(chan struct{}) + go func() { + availability.MarkUnavailable() + close(unavailableDone) + }() + + select { + case <-unavailableDone: + t.Fatal("unavailable transition completed before available publication was released") + case <-time.After(25 * time.Millisecond): + } + + close(releaseFirstPublish) + select { + case <-availableDone: + case <-time.After(time.Second): + t.Fatal("available transition did not complete") + } + select { + case <-unavailableDone: + case <-time.After(time.Second): + t.Fatal("unavailable transition did not complete") + } + + publishedMu.Lock() + defer publishedMu.Unlock() + if len(published) != 2 || published[0].Status != AvailabilityStatusAvailable || + published[1].Status != AvailabilityStatusUnavailable { + t.Fatalf("published availability transitions = %+v, want available then unavailable", published) + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go index ad514cad75..8b580885bf 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go @@ -25,10 +25,11 @@ import ( // Launcher manages an agentctl subprocess. type Launcher struct { - binaryPath string - host string - port int - logger *logger.Logger + binaryPath string + host string + port int + logger *logger.Logger + onUnexpectedExit func() cmd *exec.Cmd exited chan struct{} @@ -59,9 +60,10 @@ type Launcher struct { // Config holds configuration for the launcher. type Config struct { - BinaryPath string // Path to agentctl binary (auto-detected if empty) - Host string // Host to bind to (default: localhost) - Port int // Control port (default: 39429) + BinaryPath string // Path to agentctl binary (auto-detected if empty) + Host string // Host to bind to (default: localhost) + Port int // Control port (default: 39429) + OnUnexpectedExit func() // Called once when the child exits without Stop. } // New creates a new Launcher. @@ -77,11 +79,12 @@ func New(cfg Config, log *logger.Logger) *Launcher { } return &Launcher{ - binaryPath: cfg.BinaryPath, - host: cfg.Host, - port: cfg.Port, - logger: log.WithFields(zap.String("component", "agentctl-launcher")), - exited: make(chan struct{}), + binaryPath: cfg.BinaryPath, + host: cfg.Host, + port: cfg.Port, + onUnexpectedExit: cfg.OnUnexpectedExit, + logger: log.WithFields(zap.String("component", "agentctl-launcher")), + exited: make(chan struct{}), } } @@ -484,6 +487,9 @@ func (l *Launcher) monitorExit() { zap.Int("pid", l.cmd.Process.Pid), zap.Int("exit_code", l.cmd.ProcessState.ExitCode())) } + if !stopping && l.onUnexpectedExit != nil { + l.onUnexpectedExit() + } close(l.exited) } diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go index 1f1979712b..0025b97a2f 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go @@ -1,10 +1,97 @@ package launcher import ( + "fmt" + "io" "os" + "os/exec" + "reflect" + "runtime" + "strconv" "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" ) +func TestLauncherConfigSupportsUnexpectedExitCallback(t *testing.T) { + field, ok := reflect.TypeOf(Config{}).FieldByName("OnUnexpectedExit") + if !ok { + t.Fatal("launcher config has no unexpected-exit callback") + } + if field.Type != reflect.TypeOf((func())(nil)) { + t.Fatalf("unexpected-exit callback type = %v, want func()", field.Type) + } +} + +func TestLauncherUnexpectedExitCallbackRunsForCleanAndFailedExit(t *testing.T) { + for _, exitCode := range []int{0, 7} { + t.Run(fmt.Sprintf("exit-%d", exitCode), func(t *testing.T) { + called := startLauncherMonitorTest(t, exitCode, false) + if !called { + t.Fatalf("unexpected-exit callback was not called for exit code %d", exitCode) + } + }) + } +} + +func TestLauncherUnexpectedExitCallbackSkipsIntentionalStop(t *testing.T) { + if called := startLauncherMonitorTest(t, 7, true); called { + t.Fatal("unexpected-exit callback ran during intentional stop") + } +} + +func startLauncherMonitorTest(t *testing.T, exitCode int, stopping bool) bool { + t.Helper() + cmd := launcherExitCommand(exitCode) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + if err := cmd.Start(); err != nil { + t.Fatalf("start launcher child: %v", err) + } + + callback := make(chan struct{}, 1) + launcher := &Launcher{ + cmd: cmd, + exited: make(chan struct{}), + logger: newUnexpectedExitTestLogger(t), + onUnexpectedExit: func() { callback <- struct{}{} }, + } + launcher.mu.Lock() + launcher.stopping = stopping + launcher.mu.Unlock() + go launcher.monitorExit() + + select { + case <-launcher.exited: + case <-time.After(time.Second): + t.Fatal("launcher monitor did not finish") + } + select { + case <-callback: + return true + default: + return false + } +} + +func launcherExitCommand(exitCode int) *exec.Cmd { + code := strconv.Itoa(exitCode) + if runtime.GOOS == "windows" { + return exec.Command("cmd.exe", "/c", "exit", code) + } + return exec.Command("sh", "-c", "exit "+code) +} + +func newUnexpectedExitTestLogger(t *testing.T) *logger.Logger { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("new logger: %v", err) + } + return log +} + func TestCloseParentPipe_ClosesAndNils(t *testing.T) { r, w, err := os.Pipe() if err != nil { diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go index b485d96703..588af116fc 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go @@ -534,6 +534,7 @@ func (a *Adapter) sendUpdate(event AgentEvent) { if lifetimeCtx == nil { lifetimeCtx = context.Background() } + event.NormalizedPayload = event.NormalizedPayload.Snapshot() closedCh := a.closedCh if closedCh != nil { a.updateSendWg.Add(1) @@ -558,6 +559,7 @@ func (a *Adapter) sendUpdateLocked(event AgentEvent) bool { if a.closed { return false } + event.NormalizedPayload = event.NormalizedPayload.Snapshot() select { case a.updatesCh <- event: return true diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go index e908a32c68..193b0377e2 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go @@ -546,36 +546,7 @@ func (a *Adapter) clearCodexSubagentCorrelationsLocked(sessionID string) { } func cloneSubagentPayload(payload *streams.NormalizedPayload) *streams.NormalizedPayload { - if payload == nil || payload.SubagentTask() == nil { - return payload - } - src := payload.SubagentTask() - clone := streams.NewSubagentTask(src.Description, src.Prompt, src.SubagentType) - dst := clone.SubagentTask() - dst.Status = src.Status - dst.AgentID = src.AgentID - dst.Model = src.Model - dst.ChildSessionID = src.ChildSessionID - dst.DurationMs = src.DurationMs - dst.TotalTokens = src.TotalTokens - if src.ToolUseCount != nil { - count := *src.ToolUseCount - dst.ToolUseCount = &count - } - dst.ResultText = src.ResultText - dst.IsAsync = src.IsAsync - dst.OutputFile = src.OutputFile - dst.CanReadOutputFile = src.CanReadOutputFile - dst.SetIsAuggie(src.IsAuggie()) - if background := payload.BackgroundWork(); background != nil { - clone.SetBackgroundWorkIdentity( - background.Kind, - background.WorkID, - background.Detached, - background.Ended, - ) - } - return clone + return payload.Snapshot() } func mergeCodexSubagentPayload(current, candidate *streams.NormalizedPayload) { diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go index ae0563e6fe..e62ae0325b 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go @@ -2,6 +2,7 @@ package acp import ( "encoding/json" + "sync" "testing" "time" @@ -64,6 +65,106 @@ func TestSendUpdateBackpressureIsCanceledOnClose(t *testing.T) { } } +func TestSendUpdateSnapshotsNormalizedPayloadBeforeQueueing(t *testing.T) { + a := newTestAdapter() + t.Cleanup(func() { _ = a.Close() }) + + nested := map[string]any{"value": "before"} + payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested}) + a.sendUpdate(AgentEvent{ + Type: streams.EventTypeToolCall, + NormalizedPayload: payload, + }) + + event := <-a.updatesCh + nested["value"] = "after" + + data, err := json.Marshal(event.NormalizedPayload) + if err != nil { + t.Fatalf("marshal queued payload: %v", err) + } + want := `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}` + if string(data) != want { + t.Fatalf("queued payload changed after source mutation: got %s, want %s", data, want) + } +} + +func TestSendUpdateLockedSnapshotsNormalizedPayloadBeforeQueueing(t *testing.T) { + a := newTestAdapter() + t.Cleanup(func() { _ = a.Close() }) + + nested := map[string]any{"value": "before"} + payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested}) + a.mu.Lock() + if !a.sendUpdateLocked(AgentEvent{ + Type: streams.EventTypeToolCall, + NormalizedPayload: payload, + }) { + a.mu.Unlock() + t.Fatal("sendUpdateLocked did not enqueue event") + } + a.mu.Unlock() + + event := <-a.updatesCh + nested["value"] = "after" + data, err := json.Marshal(event.NormalizedPayload) + if err != nil { + t.Fatalf("marshal queued payload: %v", err) + } + if !json.Valid(data) || string(data) == "" { + t.Fatalf("queued payload is not valid JSON: %s", data) + } + if string(data) != `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}` { + t.Fatalf("queued payload changed after source mutation: %s", data) + } +} + +func TestQueuedNormalizedPayloadCanMarshalDuringSourceMutation(t *testing.T) { + a := newTestAdapter() + t.Cleanup(func() { _ = a.Close() }) + + nested := map[string]any{"value": "before"} + payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested}) + a.sendUpdate(AgentEvent{ + Type: streams.EventTypeToolCall, + NormalizedPayload: payload, + }) + event := <-a.updatesCh + + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + for index := 0; index < 10000; index++ { + nested["value"] = index + } + }() + go func() { + defer wg.Done() + <-start + for index := 0; index < 10000; index++ { + if _, err := json.Marshal(event.NormalizedPayload); err != nil { + t.Errorf("marshal queued payload: %v", err) + } + } + }() + close(start) + wg.Wait() + + data, err := json.Marshal(event.NormalizedPayload) + if err != nil { + t.Fatalf("marshal final queued payload: %v", err) + } + if !json.Valid(data) || string(data) == "" { + t.Fatalf("queued payload is not valid JSON: %s", data) + } + if string(data) != `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}` { + t.Fatalf("queued payload changed during source mutation: %s", data) + } +} + func TestConvertACPConfigOptions_PreservesDescriptions(t *testing.T) { description := "Controls how much reasoning the model performs." valueDescription := "Uses more reasoning for complex tasks." diff --git a/apps/backend/internal/agentctl/types/streams/tool_payload.go b/apps/backend/internal/agentctl/types/streams/tool_payload.go index 6d83d915f3..e8bd9c4de9 100644 --- a/apps/backend/internal/agentctl/types/streams/tool_payload.go +++ b/apps/backend/internal/agentctl/types/streams/tool_payload.go @@ -1,6 +1,9 @@ package streams -import "encoding/json" +import ( + "encoding/json" + "reflect" +) // ToolKind categorizes the normalized tool operation type ToolKind string @@ -83,6 +86,209 @@ func (p *NormalizedPayload) ShowPlan() *ShowPlanPayload { return p.showP func (p *NormalizedPayload) ManageTodos() *ManageTodosPayload { return p.manageTodos } func (p *NormalizedPayload) Misc() *MiscPayload { return p.misc } +// Snapshot returns a detached copy of the normalized payload. A snapshot owns +// every pointer, slice, and JSON-compatible value reachable from the payload so +// it can safely cross an asynchronous event boundary. +func (p *NormalizedPayload) Snapshot() *NormalizedPayload { + if p == nil { + return nil + } + return &NormalizedPayload{ + kind: p.kind, + readFile: cloneReadFilePayload(p.readFile), + modifyFile: cloneModifyFilePayload(p.modifyFile), + shellExec: cloneShellExecPayload(p.shellExec), + codeSearch: cloneCodeSearchPayload(p.codeSearch), + httpRequest: cloneHttpRequestPayload(p.httpRequest), + generic: cloneGenericPayload(p.generic), + createTask: cloneCreateTaskPayload(p.createTask), + subagentTask: cloneSubagentTaskPayload(p.subagentTask), + showPlan: cloneShowPlanPayload(p.showPlan), + manageTodos: cloneManageTodosPayload(p.manageTodos), + misc: cloneMiscPayload(p.misc), + backgroundWork: cloneBackgroundWorkPayload(p.backgroundWork), + monitor: cloneMonitorPayload(p.monitor), + } +} + +func cloneReadFilePayload(src *ReadFilePayload) *ReadFilePayload { + if src == nil { + return nil + } + clone := *src + if src.Output != nil { + output := *src.Output + clone.Output = &output + } + return &clone +} + +func cloneModifyFilePayload(src *ModifyFilePayload) *ModifyFilePayload { + if src == nil { + return nil + } + clone := *src + clone.Mutations = append([]FileMutation(nil), src.Mutations...) + return &clone +} + +func cloneShellExecPayload(src *ShellExecPayload) *ShellExecPayload { + if src == nil { + return nil + } + clone := *src + if src.Output != nil { + output := *src.Output + clone.Output = &output + } + return &clone +} + +func cloneCodeSearchPayload(src *CodeSearchPayload) *CodeSearchPayload { + if src == nil { + return nil + } + clone := *src + if src.Output != nil { + output := *src.Output + output.Files = append([]string(nil), src.Output.Files...) + clone.Output = &output + } + return &clone +} + +func cloneHttpRequestPayload(src *HttpRequestPayload) *HttpRequestPayload { + if src == nil { + return nil + } + clone := *src + return &clone +} + +func cloneGenericPayload(src *GenericPayload) *GenericPayload { + if src == nil { + return nil + } + clone := *src + clone.Input = cloneJSONValue(src.Input) + clone.Output = cloneJSONValue(src.Output) + return &clone +} + +func cloneCreateTaskPayload(src *CreateTaskPayload) *CreateTaskPayload { + if src == nil { + return nil + } + clone := *src + return &clone +} + +func cloneSubagentTaskPayload(src *SubagentTaskPayload) *SubagentTaskPayload { + if src == nil { + return nil + } + clone := *src + if src.ToolUseCount != nil { + count := *src.ToolUseCount + clone.ToolUseCount = &count + } + return &clone +} + +func cloneShowPlanPayload(src *ShowPlanPayload) *ShowPlanPayload { + if src == nil { + return nil + } + clone := *src + clone.Steps = append([]string(nil), src.Steps...) + return &clone +} + +func cloneManageTodosPayload(src *ManageTodosPayload) *ManageTodosPayload { + if src == nil { + return nil + } + clone := *src + clone.Items = append([]TodoItem(nil), src.Items...) + return &clone +} + +func cloneMiscPayload(src *MiscPayload) *MiscPayload { + if src == nil { + return nil + } + clone := *src + clone.Details = cloneJSONValue(src.Details) + return &clone +} + +func cloneBackgroundWorkPayload(src *BackgroundWorkPayload) *BackgroundWorkPayload { + if src == nil { + return nil + } + clone := *src + return &clone +} + +func cloneMonitorPayload(src *MonitorPayload) *MonitorPayload { + if src == nil { + return nil + } + clone := *src + return &clone +} + +func cloneJSONValue(value any) any { + if value == nil { + return nil + } + return cloneJSONReflectValue(reflect.ValueOf(value)).Interface() +} + +func cloneJSONReflectValue(value reflect.Value) reflect.Value { + if !value.IsValid() { + return value + } + switch value.Kind() { + case reflect.Interface: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + clone := cloneJSONReflectValue(value.Elem()) + result := reflect.New(value.Type()).Elem() + result.Set(clone) + return result + case reflect.Map: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + result := reflect.MakeMapWithSize(value.Type(), value.Len()) + iter := value.MapRange() + for iter.Next() { + result.SetMapIndex(iter.Key(), cloneJSONReflectValue(iter.Value())) + } + return result + case reflect.Slice: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + result := reflect.MakeSlice(value.Type(), value.Len(), value.Len()) + for index := 0; index < value.Len(); index++ { + result.Index(index).Set(cloneJSONReflectValue(value.Index(index))) + } + return result + case reflect.Pointer: + if value.IsNil() { + return reflect.Zero(value.Type()) + } + result := reflect.New(value.Type().Elem()) + result.Elem().Set(cloneJSONReflectValue(value.Elem())) + return result + default: + return value + } +} + // MarshalJSON implements custom JSON marshaling for NormalizedPayload. func (p *NormalizedPayload) MarshalJSON() ([]byte, error) { type jsonPayload struct { diff --git a/apps/backend/internal/agentctl/types/streams/tool_payload_test.go b/apps/backend/internal/agentctl/types/streams/tool_payload_test.go index 6f9a7ff205..c120a77ce6 100644 --- a/apps/backend/internal/agentctl/types/streams/tool_payload_test.go +++ b/apps/backend/internal/agentctl/types/streams/tool_payload_test.go @@ -85,3 +85,186 @@ func TestNormalizedPayloadMarshalRoundTrip(t *testing.T) { }) } } + +func TestNormalizedPayloadSnapshotDetachesMutablePayloads(t *testing.T) { + tests := []struct { + name string + payload *NormalizedPayload + check func(*NormalizedPayload) + }{ + { + name: "read file output", + payload: func() *NormalizedPayload { + payload := NewReadFile("before.go", 1, 2) + payload.ReadFile().Output = &ReadFileOutput{Content: "before"} + return payload + }(), + }, + { + name: "modify file mutations", + payload: NewModifyFile("before.go", []FileMutation{{Diff: "before"}}), + }, + { + name: "shell output", + payload: func() *NormalizedPayload { + payload := NewShellExec("echo before", "/tmp", "before", 1, false) + payload.ShellExec().Output = &ShellExecOutput{Stdout: "before", RawTerminalOutput: "before"} + return payload + }(), + }, + { + name: "code search output", + payload: func() *NormalizedPayload { + payload := NewCodeSearch("before", "before", "/tmp", "*.go") + payload.CodeSearch().Output = &CodeSearchOutput{Files: []string{"before.go"}} + return payload + }(), + }, + { + name: "http request", + payload: &NormalizedPayload{ + kind: ToolKindHttpRequest, + httpRequest: &HttpRequestPayload{URL: "https://before", Response: "before"}, + }, + }, + { + name: "generic nested JSON", + payload: func() *NormalizedPayload { + rawMessage := json.RawMessage(`{"nested":["before"]}`) + return &NormalizedPayload{ + kind: ToolKindGeneric, + generic: &GenericPayload{ + Name: "before", + Input: map[string]any{ + "nested": map[string]any{"values": []any{rawMessage}}, + }, + Output: []any{map[string]any{"value": "before"}}, + }, + } + }(), + }, + { + name: "create task", + payload: &NormalizedPayload{kind: ToolKindCreateTask, createTask: &CreateTaskPayload{Title: "before"}}, + }, + { + name: "subagent result", + payload: func() *NormalizedPayload { + count := 1 + payload := NewSubagentTask("before", "before", "before") + payload.SubagentTask().ToolUseCount = &count + payload.SubagentTask().SetIsAuggie(true) + return payload + }(), + check: func(snapshot *NormalizedPayload) { + if !snapshot.SubagentTask().IsAuggie() { + t.Error("subagent adapter marker was not copied") + } + }, + }, + { + name: "plan steps", + payload: &NormalizedPayload{ + kind: ToolKindShowPlan, + showPlan: &ShowPlanPayload{Summary: "before", Steps: []string{"before"}}, + }, + }, + { + name: "todo items", + payload: &NormalizedPayload{ + kind: ToolKindManageTodos, + manageTodos: &ManageTodosPayload{Operation: "before", Items: []TodoItem{{Description: "before"}}}, + }, + }, + { + name: "misc nested JSON", + payload: func() *NormalizedPayload { + rawMessage := json.RawMessage(`{"nested":["before"]}`) + return &NormalizedPayload{ + kind: ToolKindMisc, + misc: &MiscPayload{Label: "before", Details: map[string]any{ + "nested": []any{rawMessage}, + }}, + } + }(), + }, + { + name: "adapter provenance", + payload: func() *NormalizedPayload { + payload := &NormalizedPayload{ + kind: ToolKindMisc, + misc: &MiscPayload{Label: "before", Details: map[string]any{"value": "before"}}, + } + payload.SetMonitorIdentity("before-task", false) + return payload + }(), + check: func(snapshot *NormalizedPayload) { + if snapshot.Monitor().TaskID != "before-task" || snapshot.BackgroundWork().WorkID != "before-task" { + t.Error("adapter provenance was not copied") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before, err := json.Marshal(tt.payload.Snapshot()) + if err != nil { + t.Fatalf("marshal snapshot before mutation: %v", err) + } + snapshot := tt.payload.Snapshot() + mutateSnapshotSource(tt.payload) + after, err := json.Marshal(snapshot) + if err != nil { + t.Fatalf("marshal snapshot after mutation: %v", err) + } + if string(after) != string(before) { + t.Fatalf("snapshot changed after source mutation: got %s, want %s", after, before) + } + if tt.check != nil { + tt.check(snapshot) + } + }) + } +} + +func mutateSnapshotSource(payload *NormalizedPayload) { + switch payload.Kind() { + case ToolKindReadFile: + payload.ReadFile().Output.Content = "after" + case ToolKindModifyFile: + payload.ModifyFile().Mutations[0].Diff = "after" + case ToolKindShellExec: + payload.ShellExec().Output.Stdout = "after" + payload.ShellExec().Output.RawTerminalOutput = "after" + case ToolKindCodeSearch: + payload.CodeSearch().Output.Files[0] = "after.go" + case ToolKindHttpRequest: + payload.HttpRequest().Response = "after" + case ToolKindGeneric: + payload.Generic().Input.(map[string]any)["nested"].(map[string]any)["values"].([]any)[0].(json.RawMessage)[0] = 'x' + payload.Generic().Output.([]any)[0].(map[string]any)["value"] = "after" + if payload.Monitor() != nil { + payload.Monitor().TaskID = "after-task" + payload.BackgroundWork().WorkID = "after-task" + } + case ToolKindCreateTask: + payload.CreateTask().Title = "after" + case ToolKindSubagentTask: + payload.SubagentTask().Description = "after" + *payload.SubagentTask().ToolUseCount = 2 + case ToolKindShowPlan: + payload.ShowPlan().Steps[0] = "after" + case ToolKindManageTodos: + payload.ManageTodos().Items[0].Description = "after" + case ToolKindMisc: + details := payload.Misc().Details.(map[string]any) + if nested, ok := details["nested"]; ok { + nested.([]any)[0].(json.RawMessage)[0] = 'x' + } + if payload.Monitor() != nil { + payload.Monitor().TaskID = "after-task" + payload.BackgroundWork().WorkID = "after-task" + } + } +} diff --git a/apps/backend/internal/backendapp/agent_runtime_boot_test.go b/apps/backend/internal/backendapp/agent_runtime_boot_test.go new file mode 100644 index 0000000000..07a65902cb --- /dev/null +++ b/apps/backend/internal/backendapp/agent_runtime_boot_test.go @@ -0,0 +1,57 @@ +package backendapp + +import ( + "context" + "net/http/httptest" + "testing" + + client "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/webapp" +) + +func TestBootInitialStateHydratesAgentRuntimeAvailability(t *testing.T) { + availability := client.NewAvailability(nil, testLogger(t)) + availability.MarkAvailable() + + state := bootInitialState( + context.Background(), + nil, + routeParams{agentRuntimeAvailability: availability}, + webapp.ClassifyRoute("/"), + ) + + runtimeState, ok := state["agentRuntime"].(client.AvailabilitySnapshot) + if !ok { + t.Fatalf("agentRuntime state = %#v, want AvailabilitySnapshot", state["agentRuntime"]) + } + if runtimeState.Status != client.AvailabilityStatusAvailable || runtimeState.Reason != "" { + t.Fatalf("agentRuntime state = %+v", runtimeState) + } +} + +func TestBootInitialStateDoesNotExposeRuntimeToUnauthenticatedVisitors(t *testing.T) { + availability := client.NewAvailability(nil, testLogger(t)) + availability.MarkAvailable() + availability.MarkUnavailable() + request := httptest.NewRequest("GET", "/", nil) + + state := bootInitialState( + context.Background(), + request, + routeParams{ + authSvc: newSSOTestAuthService(t), + agentRuntimeAvailability: availability, + }, + webapp.ClassifyRoute("/"), + ) + + if _, ok := state["agentRuntime"]; ok { + t.Fatalf("unauthenticated boot state exposed agentRuntime: %#v", state) + } + if _, ok := state["features"]; !ok { + t.Fatal("unauthenticated boot state lost features") + } + if _, ok := state["auth"]; !ok { + t.Fatal("unauthenticated boot state lost auth") + } +} diff --git a/apps/backend/internal/backendapp/agentctl.go b/apps/backend/internal/backendapp/agentctl.go index 2ae83c5065..8f4a8d44fd 100644 --- a/apps/backend/internal/backendapp/agentctl.go +++ b/apps/backend/internal/backendapp/agentctl.go @@ -22,14 +22,21 @@ type agentctlLauncherResult struct { // for agent execution on the host machine. // If the configured port is unavailable, the launcher may fall back to an OS-assigned // port. In that case, cfg.Agent.StandalonePort is updated to reflect the actual port. -func provideAgentctlLauncher(ctx context.Context, cfg *config.Config, log *logger.Logger) (*agentctlLauncherResult, error) { +func provideAgentctlLauncher( + ctx context.Context, + cfg *config.Config, + log *logger.Logger, + availability *agentctlclient.Availability, +) (*agentctlLauncherResult, error) { l, cleanup, err := launcher.Provide(ctx, launcher.Config{ - Host: cfg.Agent.StandaloneHost, - Port: cfg.Agent.StandalonePort, + Host: cfg.Agent.StandaloneHost, + Port: cfg.Agent.StandalonePort, + OnUnexpectedExit: availability.MarkUnavailable, }, log) if err != nil { return nil, err } + availability.MarkAvailable() // Update config with the actual port (may differ if fallback was used) if actualPort := l.Port(); actualPort != cfg.Agent.StandalonePort { log.Info("agentctl port changed from configured value", diff --git a/apps/backend/internal/backendapp/boot_state.go b/apps/backend/internal/backendapp/boot_state.go index 7fa361ebe6..81e326f13a 100644 --- a/apps/backend/internal/backendapp/boot_state.go +++ b/apps/backend/internal/backendapp/boot_state.go @@ -75,6 +75,11 @@ func bootInitialState( return state } } + if p.agentRuntimeAvailability != nil { + if snapshot, ok := p.agentRuntimeAvailability.Snapshot(); ok { + state["agentRuntime"] = snapshot + } + } if route.Route == webapp.RouteSettings { builder.addWorkspaceState(ctx, state, nil) diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index e75659d024..c218ba4358 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -543,6 +543,7 @@ type routeParams struct { secretStore secrets.SecretStore mcpConfigSvc *mcpconfig.Service authSvc *auth.Service + agentRuntimeAvailability *client.Availability addCleanup func(func() error) repoCloner *repoclone.Cloner version string diff --git a/apps/backend/internal/backendapp/httpserver_test.go b/apps/backend/internal/backendapp/httpserver_test.go index a92142c4d8..1bbb802ce3 100644 --- a/apps/backend/internal/backendapp/httpserver_test.go +++ b/apps/backend/internal/backendapp/httpserver_test.go @@ -22,7 +22,7 @@ func TestBuildHTTPServerAbortsWhenInterlockTokenGenerationFails(t *testing.T) { newInterimSettingsInterlockToken = func() (string, error) { return "", wantErr } t.Cleanup(func() { newInterimSettingsInterlockToken = original }) - server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if !errors.Is(err, wantErr) { t.Fatalf("buildHTTPServer error = %v, want %v", err, wantErr) } diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index c6a4df3080..a4e2f4a680 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -363,7 +363,8 @@ func startServices( //nolint:cyclop // ============================================ // AGENTCTL LAUNCHER (for standalone mode) // ============================================ - agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log) + agentRuntimeAvailability := agentctlclient.NewAvailability(eventBus, log) + agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log, agentRuntimeAvailability) if err != nil { log.Error("Failed to start agentctl subprocess", zap.Error(err)) return false @@ -386,7 +387,8 @@ func startServices( //nolint:cyclop agentctlBinaryPath = agentctlResult.binaryPath } - return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups) + return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, agentRuntimeAvailability, + dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups) } // startAgentInfrastructure initializes the agent lifecycle manager, worktree, orchestrator, @@ -399,6 +401,7 @@ func startAgentInfrastructure( log *logger.Logger, addCleanup func(func() error), eventBus bus.EventBus, + agentRuntimeAvailability *agentctlclient.Availability, dbPool *db.Pool, repos *Repositories, services *Services, @@ -643,7 +646,7 @@ func startAgentInfrastructure( log.Info("Automation scheduler and evaluator started") } - return startGatewayAndServe(ctx, cfg, log, eventBus, dbPool, repos, services, + return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services, agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, addCleanup, runCleanups) } @@ -656,6 +659,7 @@ func startGatewayAndServe( cfg *config.Config, log *logger.Logger, eventBus bus.EventBus, + agentRuntimeAvailability *agentctlclient.Availability, dbPool *db.Pool, repos *Repositories, services *Services, @@ -842,13 +846,20 @@ func startGatewayAndServe( systemSvc.StartBackground(ctx) addCleanup(func() error { systemSvc.StopBackground(); return nil }) gateways.RegisterSystemNotifications(ctx, eventBus, gateway.Hub, log) + gateways.RegisterAgentRuntimeNotifications(ctx, eventBus, gateway.Hub, func() (any, bool) { + if agentRuntimeAvailability == nil { + return nil, false + } + snapshot, ok := agentRuntimeAvailability.Snapshot() + return snapshot, ok + }, log) // ============================================ // HTTP SERVER // ============================================ server, err := buildHTTPServer(cfg, log, gateway, repos, services, agentSettingsController, lifecycleMgr, eventBus, orchestratorSvc, notificationCtrl, msgCreator, agentRegistry, hostUtilityMgr, - addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer, dbPool) + addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer, dbPool, agentRuntimeAvailability) if err != nil { log.Error("Failed to build HTTP server", zap.Error(err)) return false @@ -1750,6 +1761,7 @@ func buildHTTPServer( systemSvc *systemsvc.Service, workspaceRestorer taskhandlers.WorkspaceQuarantineRestorer, dbPool *db.Pool, + agentRuntimeAvailability *agentctlclient.Availability, ) (*http.Server, error) { gin.SetMode(gin.ReleaseMode) router := gin.New() @@ -1827,6 +1839,7 @@ func buildHTTPServer( secretStore: userSecretStore, mcpConfigSvc: mcpconfig.NewService(repos.AgentSettings), authSvc: services.Auth, + agentRuntimeAvailability: agentRuntimeAvailability, addCleanup: addCleanup, repoCloner: repoCloner, version: Version, diff --git a/apps/backend/internal/backendapp/mentions.go b/apps/backend/internal/backendapp/mentions.go index 305c02829f..7d9d57eb5c 100644 --- a/apps/backend/internal/backendapp/mentions.go +++ b/apps/backend/internal/backendapp/mentions.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/entityrefs" "github.com/kandev/kandev/internal/gitlab" "github.com/kandev/kandev/internal/mentions" @@ -24,6 +25,7 @@ type MentionComponents struct { Registry *mentions.Registry Search mentions.Searcher Submission entityrefs.SubmissionValidator + Log *logger.Logger } type mentionWorkspaceResolver interface { @@ -35,6 +37,7 @@ type mentionConversationResolver interface { } func newMentionComponents( + log *logger.Logger, workspaces mentionWorkspaceResolver, conversations mentionConversationResolver, providers ...mentions.MentionProvider, @@ -47,12 +50,13 @@ func newMentionComponents( } search := &workspaceValidatingMentionSearcher{ workspaces: workspaces, - searcher: mentions.NewService(registry), + searcher: mentions.NewService(registry, mentions.WithLogger(log)), } return &MentionComponents{ Registry: registry, Search: search, Submission: entityrefs.NewSubmissionService(conversations, registry), + Log: log, }, nil } @@ -70,14 +74,14 @@ func (s *workspaceValidatingMentionSearcher) Search( return nil, mentions.ErrInvalidRequest } if s == nil || s.workspaces == nil || s.searcher == nil { - return nil, errors.New("mention search is unavailable") + return nil, mentions.NewSearchFailure(mentions.FailureStageSearch, mentions.FailureClassInternal, errors.New("mention search is unavailable")) } workspace, err := s.workspaces.GetWorkspace(ctx, request.WorkspaceID) if err != nil { if errors.Is(err, repository.ErrWorkspaceNotFound) { return nil, mentions.ErrWorkspaceNotFound } - return nil, fmt.Errorf("validate mention workspace: %w", err) + return nil, mentions.NewSearchFailure(mentions.FailureStageWorkspaceValidation, mentions.FailureClassWorkspaceLookup, err) } if workspace == nil || workspace.ID != request.WorkspaceID { return nil, mentions.ErrWorkspaceNotFound @@ -298,5 +302,5 @@ func registerMentionRoutes(router gin.IRoutes, components *MentionComponents) { if router == nil || components == nil || components.Search == nil { return } - mentions.NewHandler(components.Search).RegisterRoutes(router) + mentions.NewHandler(components.Search, components.Log).RegisterRoutes(router) } diff --git a/apps/backend/internal/backendapp/mentions_test.go b/apps/backend/internal/backendapp/mentions_test.go index fff8ce0bb4..b69ced31e3 100644 --- a/apps/backend/internal/backendapp/mentions_test.go +++ b/apps/backend/internal/backendapp/mentions_test.go @@ -4,16 +4,21 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/mentions" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/task/repository" apiv1 "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type fakeMentionScopeResolver struct { @@ -112,7 +117,7 @@ func TestMentionHTTPCompositionValidatesWorkspaceBeforeMixedProviderFanout(t *te resolver := &fakeMentionScopeResolver{workspaces: map[string]*models.Workspace{ "workspace-1": {ID: "workspace-1", Name: "One"}, }} - components, err := newMentionComponents(resolver, resolver, failure, timeout, success) + components, err := newMentionComponents(testLogger(t), resolver, resolver, failure, timeout, success) if err != nil { t.Fatalf("compose mentions: %v", err) } @@ -190,7 +195,7 @@ func TestMentionHTTPCompositionPropagatesRequestCancellation(t *testing.T) { resolver := &fakeMentionScopeResolver{workspaces: map[string]*models.Workspace{ "workspace-1": {ID: "workspace-1", Name: "One"}, }} - components, err := newMentionComponents(resolver, resolver, provider) + components, err := newMentionComponents(testLogger(t), resolver, resolver, provider) if err != nil { t.Fatalf("compose mentions: %v", err) } @@ -226,14 +231,55 @@ func TestMentionHTTPCompositionPropagatesRequestCancellation(t *testing.T) { t.Fatal("provider did not observe request cancellation") } - if response.Code != http.StatusInternalServerError { - t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + if response.Code != 499 { + t.Fatalf("status = %d, want 499; body = %s", response.Code, response.Body.String()) } if !provider.canceled { t.Fatal("provider did not observe request cancellation") } } +func TestMentionHTTPCompositionLogsWorkspaceFailureOnceWithSafeFields(t *testing.T) { + gin.SetMode(gin.TestMode) + core, observed := observer.New(zap.DebugLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("create observer logger: %v", err) + } + resolver := &fakeMentionScopeResolver{err: errors.New("secret database address")} + components, err := newMentionComponents(log, resolver, resolver) + if err != nil { + t.Fatalf("compose mentions: %v", err) + } + router := gin.New() + registerMentionRoutes(router, components) + + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest( + http.MethodGet, + "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query", + nil, + )) + + if response.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", response.Code, http.StatusInternalServerError) + } + entries := observed.All() + if len(entries) != 1 || entries[0].Level != zap.ErrorLevel { + t.Fatalf("workspace failure diagnostics = %#v", entries) + } + fields := entries[0].ContextMap() + if fields["workspace_id"] != "workspace-1" || + fields["failure_stage"] != mentions.FailureStageWorkspaceValidation || + fields["failure_class"] != mentions.FailureClassWorkspaceLookup { + t.Fatalf("workspace failure fields = %#v", fields) + } + if strings.Contains(fmt.Sprint(fields), "secret-query") || + strings.Contains(fmt.Sprint(fields), "secret database address") { + t.Fatalf("workspace failure diagnostics leaked sensitive detail: %#v", fields) + } +} + func TestBuiltinMentionProvidersRemainDescriptorDrivenWhenIntegrationsAreNil(t *testing.T) { providers := builtinMentionProviders(&Services{}, nil) want := []struct { diff --git a/apps/backend/internal/backendapp/services.go b/apps/backend/internal/backendapp/services.go index f32cb7c8e8..a19ddbfac2 100644 --- a/apps/backend/internal/backendapp/services.go +++ b/apps/backend/internal/backendapp/services.go @@ -212,6 +212,7 @@ func provideServices(cfg *config.Config, log *logger.Logger, repos *Repositories Notification: nil, } mentionComponents, err := newMentionComponents( + log, taskSvc, taskSvc, builtinMentionProviders(services, repos.Task)..., diff --git a/apps/backend/internal/events/types.go b/apps/backend/internal/events/types.go index 6ce6ec775a..09be4404bd 100644 --- a/apps/backend/internal/events/types.go +++ b/apps/backend/internal/events/types.go @@ -170,7 +170,8 @@ const PluginUserStateUpdated = "plugin.user-state.updated" // create/restore, disk walk). Published by internal/system/jobs.Tracker on // every state transition and broadcast to all WebSocket clients. const ( - SystemJobUpdate = "system.job.update" + SystemJobUpdate = "system.job.update" + AgentRuntimeAvailabilityChanged = "system.agent_runtime.availability_changed" ) // Event types for environments diff --git a/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go b/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go new file mode 100644 index 0000000000..517325fb37 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go @@ -0,0 +1,93 @@ +package websocket + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestAgentRuntimeNotificationsBroadcastAndReplaySanitizedState(t *testing.T) { + hub := newTestHub(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go hub.Run(ctx) + + eventBus := bus.NewMemoryEventBus(testLoggerForUserNotifications(t)) + t.Cleanup(eventBus.Close) + payload := map[string]any{"status": "unavailable", "reason": "agentctl_exited"} + broadcaster := RegisterAgentRuntimeNotifications(ctx, eventBus, hub, func() (any, bool) { + return payload, true + }, testLoggerForUserNotifications(t)) + t.Cleanup(broadcaster.Close) + + liveClient := newTestClient("live") + registerTestClient(hub, liveClient) + if err := eventBus.Publish(ctx, events.AgentRuntimeAvailabilityChanged, + bus.NewEvent(events.AgentRuntimeAvailabilityChanged, "test", payload)); err != nil { + t.Fatalf("publish availability event: %v", err) + } + liveMessage := readNotification(t, liveClient) + if liveMessage.Action != ws.ActionSystemAgentRuntimeStatusChanged { + t.Fatalf("live action = %q, want %q", liveMessage.Action, ws.ActionSystemAgentRuntimeStatusChanged) + } + + replayClient := newTestClient("replay") + registerTestClient(hub, replayClient) + hub.SubscribeToUser(replayClient, "user-1") + replayMessage := readNotification(t, replayClient) + if replayMessage.Action != ws.ActionSystemAgentRuntimeStatusChanged { + t.Fatalf("replay action = %q, want %q", replayMessage.Action, ws.ActionSystemAgentRuntimeStatusChanged) + } + var replayPayload map[string]any + if err := json.Unmarshal(replayMessage.Payload, &replayPayload); err != nil { + t.Fatalf("decode replay payload: %v", err) + } + if replayPayload["status"] != "unavailable" || replayPayload["reason"] != "agentctl_exited" { + t.Fatalf("replay payload = %#v", replayPayload) + } +} + +func TestSystemEventBroadcasterCloseIsIdempotent(t *testing.T) { + log := testLoggerForUserNotifications(t) + eventBus := bus.NewMemoryEventBus(log) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + hub := newTestHub(t) + broadcaster := RegisterSystemNotifications(ctx, eventBus, hub, log) + + var wg sync.WaitGroup + for index := 0; index < 16; index++ { + wg.Add(1) + go func() { + defer wg.Done() + broadcaster.Close() + }() + } + wg.Wait() + + if len(broadcaster.subscriptions) != 0 { + t.Fatalf("subscriptions after concurrent Close = %d, want 0", len(broadcaster.subscriptions)) + } + eventBus.Close() +} + +func readNotification(t *testing.T, client *Client) ws.Message { + t.Helper() + select { + case frame := <-client.send: + var message ws.Message + if err := json.Unmarshal(frame, &message); err != nil { + t.Fatalf("decode websocket frame: %v", err) + } + return message + case <-time.After(time.Second): + t.Fatal("timed out waiting for websocket notification") + return ws.Message{} + } +} diff --git a/apps/backend/internal/gateway/websocket/system_notifications.go b/apps/backend/internal/gateway/websocket/system_notifications.go index 91df4f4e50..e853a227bc 100644 --- a/apps/backend/internal/gateway/websocket/system_notifications.go +++ b/apps/backend/internal/gateway/websocket/system_notifications.go @@ -2,6 +2,7 @@ package websocket import ( "context" + "sync" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/events" @@ -18,6 +19,7 @@ type SystemEventBroadcaster struct { hub *Hub subscriptions []bus.Subscription logger *logger.Logger + closeOnce sync.Once } // RegisterSystemNotifications wires the system event broadcaster onto the @@ -41,14 +43,56 @@ func RegisterSystemNotifications(ctx context.Context, eventBus bus.EventBus, hub return b } +// RegisterAgentRuntimeNotifications forwards the install-wide agent runtime +// state and replays the latest snapshot when a client subscribes to its user. +// The snapshot provider keeps the gateway independent from the launcher +// package and ensures replay uses the same sanitized state as boot hydration. +func RegisterAgentRuntimeNotifications( + ctx context.Context, + eventBus bus.EventBus, + hub *Hub, + snapshotProvider func() (any, bool), + log *logger.Logger, +) *SystemEventBroadcaster { + b := &SystemEventBroadcaster{ + hub: hub, + logger: log.WithFields(zap.String("component", "ws-agent-runtime-broadcaster")), + } + if eventBus != nil { + b.subscribe(eventBus, events.AgentRuntimeAvailabilityChanged, ws.ActionSystemAgentRuntimeStatusChanged) + } + if snapshotProvider != nil { + hub.AddUserSubscriptionListener(func(userID string) { + payload, ok := snapshotProvider() + if !ok { + return + } + msg, err := ws.NewNotification(ws.ActionSystemAgentRuntimeStatusChanged, payload) + if err != nil { + b.logger.Error("failed to build agent runtime replay notification", zap.Error(err)) + return + } + hub.BroadcastToUser(userID, msg) + }) + } + + go func() { + <-ctx.Done() + b.Close() + }() + return b +} + // Close unsubscribes from all event-bus subscriptions. func (b *SystemEventBroadcaster) Close() { - for _, sub := range b.subscriptions { - if sub != nil && sub.IsValid() { - _ = sub.Unsubscribe() + b.closeOnce.Do(func() { + for _, sub := range b.subscriptions { + if sub != nil && sub.IsValid() { + _ = sub.Unsubscribe() + } } - } - b.subscriptions = nil + b.subscriptions = nil + }) } func (b *SystemEventBroadcaster) subscribe(eventBus bus.EventBus, subject, action string) { @@ -58,6 +102,7 @@ func (b *SystemEventBroadcaster) subscribe(eventBus bus.EventBus, subject, actio b.logger.Error("failed to build websocket notification", zap.String("action", action), zap.Error(err)) return nil } + //ws:global install-wide system and runtime notifications. b.hub.Broadcast(msg) return nil }) diff --git a/apps/backend/internal/mentions/handler.go b/apps/backend/internal/mentions/handler.go index 6134e6335f..4c3c02d4ed 100644 --- a/apps/backend/internal/mentions/handler.go +++ b/apps/backend/internal/mentions/handler.go @@ -9,7 +9,9 @@ import ( "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/common/logger" apiv1 "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" ) const errorResponseField = "error" @@ -22,10 +24,11 @@ type Searcher interface { // Handler exposes mention search over HTTP. type Handler struct { searcher Searcher + log *logger.Logger } -func NewHandler(searcher Searcher) *Handler { - return &Handler{searcher: searcher} +func NewHandler(searcher Searcher, log *logger.Logger) *Handler { + return &Handler{searcher: searcher, log: log} } func (h *Handler) RegisterRoutes(router gin.IRoutes) { @@ -35,7 +38,7 @@ func (h *Handler) RegisterRoutes(router gin.IRoutes) { func (h *Handler) search(c *gin.Context) { limit, err := parseLimit(c.Query("limit")) if err != nil { - writeSearchError(c, ErrInvalidRequest) + h.writeSearchError(c, ErrInvalidRequest) return } response, err := h.searcher.Search(c.Request.Context(), SearchRequest{ @@ -44,7 +47,7 @@ func (h *Handler) search(c *gin.Context) { Limit: limit, }) if err != nil { - writeSearchError(c, err) + h.writeSearchError(c, err) return } c.JSON(http.StatusOK, response) @@ -68,13 +71,29 @@ func parseLimit(rawLimit string) (int, error) { return limit, nil } -func writeSearchError(c *gin.Context, err error) { +func (h *Handler) writeSearchError(c *gin.Context, err error) { switch { case errors.Is(err, ErrInvalidRequest): c.JSON(http.StatusBadRequest, gin.H{errorResponseField: "invalid mention search request"}) case errors.Is(err, ErrWorkspaceNotFound): c.JSON(http.StatusNotFound, gin.H{errorResponseField: "mention search workspace not found"}) + case errors.Is(err, context.Canceled): + c.JSON(499, gin.H{errorResponseField: "mention search request canceled"}) default: + h.logUnexpectedFailure(c, err) c.JSON(http.StatusInternalServerError, gin.H{errorResponseField: "mention search failed"}) } } + +func (h *Handler) logUnexpectedFailure(c *gin.Context, err error) { + if h.log == nil { + return + } + stage, class := SearchFailureMetadata(err) + h.log.Error( + "mention search failed", + zap.String("workspace_id", c.Param("id")), + zap.String("failure_stage", stage), + zap.String("failure_class", class), + ) +} diff --git a/apps/backend/internal/mentions/handler_test.go b/apps/backend/internal/mentions/handler_test.go index b167afa283..422a609529 100644 --- a/apps/backend/internal/mentions/handler_test.go +++ b/apps/backend/internal/mentions/handler_test.go @@ -11,8 +11,11 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/common/logger" apiv1 "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type fakeSearcher struct { @@ -25,7 +28,7 @@ func TestHandlerRegisterRoutesUsesExistingWorkspaceWildcardName(t *testing.T) { router.GET("/api/v1/workspaces/:id", func(c *gin.Context) { c.Status(http.StatusNoContent) }) handler := NewHandler(fakeSearcher{search: func(_ context.Context, request SearchRequest) (*apiv1.MentionSearchResponse, error) { return &apiv1.MentionSearchResponse{Query: request.Query}, nil - }}) + }}, nil) var recovered any func() { @@ -56,7 +59,7 @@ func TestHandlerSearch_ReturnsNormalizedResponse(t *testing.T) { Results: []apiv1.EntityReference{}, }}, }, nil - }}).RegisterRoutes(router) + }}, nil).RegisterRoutes(router) request := httptest.NewRequest(http.MethodGet, "/api/v1/workspaces/workspace-1/mentions/search?q=auth&limit=99", nil) @@ -81,7 +84,7 @@ func TestHandlerSearch_RejectsInvalidLimitWithoutSearching(t *testing.T) { NewHandler(fakeSearcher{search: func(_ context.Context, _ SearchRequest) (*apiv1.MentionSearchResponse, error) { t.Fatal("search called for invalid limit") return nil, nil - }}).RegisterRoutes(router) + }}, nil).RegisterRoutes(router) request := httptest.NewRequest(http.MethodGet, "/api/v1/workspaces/workspace-1/mentions/search?q=auth&limit=not-a-number", nil) @@ -110,7 +113,7 @@ func TestHandlerSearch_MapsSafeErrorsWithoutLeaking(t *testing.T) { router := gin.New() NewHandler(fakeSearcher{search: func(_ context.Context, _ SearchRequest) (*apiv1.MentionSearchResponse, error) { return nil, test.err - }}).RegisterRoutes(router) + }}, nil).RegisterRoutes(router) request := httptest.NewRequest(http.MethodGet, "/api/v1/workspaces/workspace-1/mentions/search?q=auth", nil) @@ -126,3 +129,69 @@ func TestHandlerSearch_MapsSafeErrorsWithoutLeaking(t *testing.T) { }) } } + +func TestHandlerSearch_MapsClientCancellationTo499WithoutLogging(t *testing.T) { + log, observed := newMentionObserverLogger(t) + gin.SetMode(gin.TestMode) + router := gin.New() + NewHandler(fakeSearcher{search: func(context.Context, SearchRequest) (*apiv1.MentionSearchResponse, error) { + return nil, context.Canceled + }}, log).RegisterRoutes(router) + + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest( + http.MethodGet, + "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query", + nil, + )) + + if response.Code != 499 { + t.Fatalf("status = %d, want 499; body = %s", response.Code, response.Body.String()) + } + if observed.Len() != 0 { + t.Fatalf("cancellation emitted %d log records, want none", observed.Len()) + } +} + +func TestHandlerSearchLogsUnexpectedFailureOnceWithSafeFields(t *testing.T) { + log, observed := newMentionObserverLogger(t) + gin.SetMode(gin.TestMode) + router := gin.New() + NewHandler(fakeSearcher{search: func(context.Context, SearchRequest) (*apiv1.MentionSearchResponse, error) { + return nil, errors.New("secret database address") + }}, log).RegisterRoutes(router) + + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest( + http.MethodGet, + "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query", + nil, + )) + + if response.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", response.Code, http.StatusInternalServerError) + } + entries := observed.All() + if len(entries) != 1 || entries[0].Level != zap.ErrorLevel { + t.Fatalf("unexpected error records = %#v", entries) + } + fields := entries[0].ContextMap() + if fields["workspace_id"] != "workspace-1" || fields["failure_stage"] != "search" || + fields["failure_class"] != "internal" { + t.Fatalf("safe failure fields = %#v", fields) + } + serialized := entries[0].Message + response.Body.String() + fmt.Sprint(entries[0].ContextMap()) + if strings.Contains(serialized, "secret-query") || strings.Contains(serialized, "secret database address") { + t.Fatalf("sensitive mention-search detail leaked: %s", serialized) + } +} + +func newMentionObserverLogger(t *testing.T) (*logger.Logger, *observer.ObservedLogs) { + t.Helper() + core, observed := observer.New(zap.DebugLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("create observer logger: %v", err) + } + return log, observed +} diff --git a/apps/backend/internal/mentions/service.go b/apps/backend/internal/mentions/service.go index 69007f506f..5daf9731d5 100644 --- a/apps/backend/internal/mentions/service.go +++ b/apps/backend/internal/mentions/service.go @@ -13,8 +13,10 @@ import ( "unicode" "unicode/utf8" + "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/entityrefs" apiv1 "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" ) type registeredProvider struct { @@ -155,6 +157,7 @@ func (r *Registry) snapshot() []registeredProvider { // Service validates and aggregates every registered provider. type Service struct { registry *Registry + log *logger.Logger providerTimeout time.Duration maxConcurrency int providerSlots chan struct{} @@ -186,6 +189,12 @@ func WithMaxConcurrency(limit int) Option { } } +func WithLogger(log *logger.Logger) Option { + return func(service *Service) { + service.log = log + } +} + func NewService(registry *Registry, options ...Option) *Service { service := &Service{ registry: registry, @@ -274,7 +283,7 @@ func (s *Service) searchProvider(ctx context.Context, registered registeredProvi defer cancelProvider() result := make(chan apiv1.MentionGroup, 1) go func() { - group := searchProviderWithinContext(providerCtx, registered, request) + group := searchProviderWithinContext(providerCtx, registered, request, s.log) // The provider slot belongs to the underlying call. A provider that ignores // cancellation stays quarantined without consuming shared search capacity. <-providerSlot @@ -308,11 +317,22 @@ func searchProviderWithinContext( ctx context.Context, registered registeredProvider, request SearchRequest, + log *logger.Logger, ) apiv1.MentionGroup { group := groupFromDescriptor(registered.descriptor) candidates, err := registered.provider.Search(ctx, request) if err != nil { - group.Status = classifyProviderError(err) + status := classifyProviderError(err) + if log != nil && !errors.Is(err, context.Canceled) { + log.Warn( + "mention provider search failed", + zap.String("source", registered.descriptor.Source), + zap.String("provider", registered.descriptor.Provider), + zap.String("kind", registered.descriptor.Kind), + zap.String("status", string(status)), + ) + } + group.Status = status return group } if ctx.Err() != nil { diff --git a/apps/backend/internal/mentions/service_test.go b/apps/backend/internal/mentions/service_test.go index 14afb2a9fd..bec76fcbaa 100644 --- a/apps/backend/internal/mentions/service_test.go +++ b/apps/backend/internal/mentions/service_test.go @@ -9,7 +9,10 @@ import ( "testing" "time" + "github.com/kandev/kandev/internal/common/logger" apiv1 "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type fakeProvider struct { @@ -96,6 +99,50 @@ func TestServiceSearch_RegistryOwnsNamespacedProviderIdentity(t *testing.T) { } } +func TestServiceSearchLogsProviderFailureWithSafeIdentity(t *testing.T) { + core, observed := observer.New(zap.DebugLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("create observer logger: %v", err) + } + registry := NewRegistry() + if err := registry.Register(fakeProvider{ + descriptor: ProviderDescriptor{ + Source: "acme_issues", Provider: "acme", Kind: "issue", + DisplayName: "Acme issues", KindLabel: "Issue", + }, + search: func(context.Context, SearchRequest) ([]Candidate, error) { + return nil, NewProviderError(StatusUnauthorized, errors.New("secret upstream detail")) + }, + }); err != nil { + t.Fatalf("register provider: %v", err) + } + + response, err := NewService(registry, WithLogger(log)).Search(context.Background(), SearchRequest{ + WorkspaceID: "workspace-1", + Query: "secret-query", + }) + if err != nil { + t.Fatalf("search: %v", err) + } + if response.Groups[0].Status != StatusUnauthorized { + t.Fatalf("provider status = %q, want %q", response.Groups[0].Status, StatusUnauthorized) + } + entries := observed.All() + if len(entries) != 1 || entries[0].Level != zap.WarnLevel { + t.Fatalf("provider diagnostics = %#v", entries) + } + fields := entries[0].ContextMap() + if fields["source"] != "acme_issues" || fields["provider"] != "acme" || + fields["kind"] != "issue" || fields["status"] != string(StatusUnauthorized) { + t.Fatalf("provider diagnostic fields = %#v", fields) + } + if strings.Contains(fmt.Sprint(fields), "secret-query") || + strings.Contains(fmt.Sprint(fields), "secret upstream detail") { + t.Fatalf("provider diagnostic leaked sensitive detail: %#v", fields) + } +} + func TestRegistryRegister_RejectsUnsafeAndDuplicateSources(t *testing.T) { provider := func(source string) fakeProvider { return fakeProvider{ diff --git a/apps/backend/internal/mentions/types.go b/apps/backend/internal/mentions/types.go index 2b237e4047..d79d990ddd 100644 --- a/apps/backend/internal/mentions/types.go +++ b/apps/backend/internal/mentions/types.go @@ -31,6 +31,55 @@ var ( type Status = apiv1.MentionStatus +const ( + FailureStageSearch = "search" + FailureStageWorkspaceValidation = "workspace_validation" + FailureClassInternal = "internal" + FailureClassWorkspaceLookup = "workspace_lookup" +) + +// SearchFailure carries safe classification metadata across the composition +// boundary without requiring handlers to inspect an implementation error. +type SearchFailure struct { + stage string + class string + cause error +} + +func (e *SearchFailure) Error() string { + if e == nil || e.cause == nil { + return "mention search failed" + } + return e.cause.Error() +} + +func (e *SearchFailure) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +// NewSearchFailure attaches stable, non-sensitive diagnostic classification. +func NewSearchFailure(stage, class string, cause error) error { + if stage == "" { + stage = FailureStageSearch + } + if class == "" { + class = FailureClassInternal + } + return &SearchFailure{stage: stage, class: class, cause: cause} +} + +// SearchFailureMetadata returns safe fields for structured HTTP diagnostics. +func SearchFailureMetadata(err error) (stage, class string) { + var failure *SearchFailure + if errors.As(err, &failure) && failure != nil { + return failure.stage, failure.class + } + return FailureStageSearch, FailureClassInternal +} + const ( StatusOK = apiv1.MentionStatusOK StatusNotConfigured = apiv1.MentionStatusNotConfigured diff --git a/apps/backend/internal/office/costs/modelsdev/client.go b/apps/backend/internal/office/costs/modelsdev/client.go index 94d6ec1636..8528b48301 100644 --- a/apps/backend/internal/office/costs/modelsdev/client.go +++ b/apps/backend/internal/office/costs/modelsdev/client.go @@ -15,6 +15,7 @@ import ( "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/office/shared" + "golang.org/x/sync/singleflight" "go.uber.org/zap" ) @@ -43,7 +44,8 @@ type Client struct { httpClient *http.Client logger *logger.Logger - once sync.Once + once sync.Once + refreshGroup singleflight.Group mu sync.RWMutex index map[string]shared.ModelPricing @@ -162,7 +164,7 @@ func (c *Client) warmFromDisk(ctx context.Context) { } // File missing on first boot — schedule a refresh; lookup // returns miss this turn. - go c.refreshSafe(ctx) + c.startBackgroundRefresh(ctx) return } buf, err := os.ReadFile(c.cachePath) @@ -176,7 +178,7 @@ func (c *Client) warmFromDisk(ctx context.Context) { c.loadedAt = stat.ModTime() c.mu.Unlock() if time.Since(stat.ModTime()) >= c.ttl { - go c.refreshSafe(ctx) + c.startBackgroundRefresh(context.WithoutCancel(ctx)) } } @@ -220,35 +222,56 @@ func (c *Client) parseModelInfoFromBuffer(key string) (ModelInfo, bool) { return info, true } -// maybeRefresh fires a background refresh when the loaded buffer is -// stale. No-op when refresh is in progress (sync.Once-style guard -// implicitly enforced by atomicity of the staleness check + ttl). +// maybeRefresh fires a background refresh when the loaded buffer is stale. +// Refresh's per-client singleflight guard coalesces all callers, including +// concurrent pricing and metadata lookups. func (c *Client) maybeRefresh(ctx context.Context) { c.mu.RLock() stale := c.loadedAt.IsZero() || time.Since(c.loadedAt) >= c.ttl c.mu.RUnlock() if stale { - go c.refreshSafe(ctx) + c.startBackgroundRefresh(context.WithoutCancel(ctx)) } } -// refreshSafe wraps Refresh in a panic guard + warning log; safe to -// run from a goroutine. -func (c *Client) refreshSafe(ctx context.Context) { - defer func() { - if r := recover(); r != nil { - c.logger.Warn("models.dev refresh panicked", zap.Any("recover", r)) +// startBackgroundRefresh registers the refresh synchronously so every stale +// lookup joins the same in-flight singleflight call before returning. The +// result is still observed asynchronously, so stale lookups remain nonblocking. +// Callers choose whether the refresh should inherit or detach from their context. +func (c *Client) startBackgroundRefresh(ctx context.Context) { + result := c.refreshResult(ctx) + go func() { + if err := (<-result).Err; err != nil { + c.logger.Warn("models.dev refresh failed", zap.Error(err)) } }() - if err := c.Refresh(ctx); err != nil { - c.logger.Warn("models.dev refresh failed", zap.Error(err)) - } } // Refresh pulls the latest dataset from models.dev and atomically // swaps the cache file. Network or write errors leave the existing // file (and in-memory index) untouched. func (c *Client) Refresh(ctx context.Context) error { + result := c.refreshResult(ctx) + return (<-result).Err +} + +func (c *Client) refreshResult(ctx context.Context) <-chan singleflight.Result { + return c.refreshGroup.DoChan("modelsdev-refresh", func() (interface{}, error) { + return c.refreshPhysicalSafely(ctx) + }) +} + +func (c *Client) refreshPhysicalSafely(ctx context.Context) (value interface{}, err error) { + defer func() { + if recovered := recover(); recovered != nil { + c.logger.Warn("models.dev refresh panicked", zap.Any("recover", recovered)) + err = fmt.Errorf("models.dev refresh panicked") + } + }() + return nil, c.refreshPhysical(ctx) +} + +func (c *Client) refreshPhysical(ctx context.Context) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, nil) if err != nil { return fmt.Errorf("build request: %w", err) @@ -271,24 +294,22 @@ func (c *Client) Refresh(ctx context.Context) error { } c.mu.Lock() - c.cacheBuf = buf - c.loadedAt = time.Now() - // Re-populate every entry already in the in-memory index so a - // stale price doesn't survive the refresh. + newIndex := make(map[string]shared.ModelPricing, len(c.index)) for k := range c.index { if pricing, ok := lookupInDataset(buf, k); ok { - c.index[k] = pricing - } else { - delete(c.index, k) + newIndex[k] = pricing } } + newInfo := make(map[string]ModelInfo, len(c.info)) for k := range c.info { if info, ok := lookupModelInfoInDataset(buf, k); ok { - c.info[k] = info - } else { - delete(c.info, k) + newInfo[k] = info } } + c.cacheBuf = buf + c.index = newIndex + c.info = newInfo + c.loadedAt = time.Now().UTC() c.mu.Unlock() return nil } @@ -300,11 +321,27 @@ func (c *Client) writeCacheAtomic(buf []byte) error { if err := os.MkdirAll(filepath.Dir(c.cachePath), 0o755); err != nil { return fmt.Errorf("mkdir cache dir: %w", err) } - tmp := c.cachePath + ".tmp" - if err := os.WriteFile(tmp, buf, 0o644); err != nil { - return fmt.Errorf("write cache tmp: %w", err) + tmp, err := os.CreateTemp(filepath.Dir(c.cachePath), "."+filepath.Base(c.cachePath)+".tmp-*") + if err != nil { + return fmt.Errorf("create cache tmp: %w", err) } - if err := os.Rename(tmp, c.cachePath); err != nil { + tmpName := tmp.Name() + defer func() { + _ = tmp.Close() + _ = os.Remove(tmpName) + }() + if written, err := tmp.Write(buf); err != nil { + return fmt.Errorf("write cache tmp: %w", err) + } else if written != len(buf) { + return fmt.Errorf("write cache tmp: %w", io.ErrShortWrite) + } + if err := tmp.Sync(); err != nil { + return fmt.Errorf("sync cache tmp: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close cache tmp: %w", err) + } + if err := os.Rename(tmpName, c.cachePath); err != nil { return fmt.Errorf("rename cache: %w", err) } return nil diff --git a/apps/backend/internal/office/costs/modelsdev/client_test.go b/apps/backend/internal/office/costs/modelsdev/client_test.go index 2e4610693c..b3f1c53666 100644 --- a/apps/backend/internal/office/costs/modelsdev/client_test.go +++ b/apps/backend/internal/office/costs/modelsdev/client_test.go @@ -6,6 +6,8 @@ import ( "net/http/httptest" "os" "path/filepath" + "sync" + "sync/atomic" "testing" "time" @@ -236,3 +238,289 @@ func TestClient_FirstBootMissesGracefully(t *testing.T) { t.Error("expected miss on cold-boot lookup") } } + +type requestGate struct { + server *httptest.Server + started chan struct{} + release chan struct{} + requests atomic.Int32 +} + +func newRequestGate(t *testing.T, body string) *requestGate { + t.Helper() + gate := &requestGate{ + started: make(chan struct{}), + release: make(chan struct{}), + } + gate.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := gate.requests.Add(1) + if count == 1 { + close(gate.started) + } + select { + case <-gate.release: + case <-r.Context().Done(): + return + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(gate.server.Close) + return gate +} + +func (g *requestGate) waitForFirstRequest(t *testing.T) { + t.Helper() + select { + case <-g.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for the first models.dev request") + } +} + +func TestClient_ConcurrentRefreshCallsShareOneFetch(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "models-dev.json") + gate := newRequestGate(t, sampleDataset) + c := modelsdev.New(modelsdev.Config{ + CachePath: cachePath, + URL: gate.server.URL, + TTL: time.Hour, + HTTPClient: gate.server.Client(), + }, logger.Default()) + + start := make(chan struct{}) + var wg sync.WaitGroup + for index := 0; index < 8; index++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if err := c.Refresh(context.Background()); err != nil { + t.Errorf("Refresh: %v", err) + } + }() + } + close(start) + gate.waitForFirstRequest(t) + close(gate.release) + wg.Wait() + + if got := gate.requests.Load(); got != 1 { + t.Fatalf("models.dev request count = %d, want 1", got) + } +} + +func TestClient_ConcurrentLookupsShareOneBackgroundFetch(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "models-dev.json") + if err := os.WriteFile(cachePath, []byte(sampleDataset), 0o644); err != nil { + t.Fatalf("seed cache: %v", err) + } + old := time.Now().Add(-time.Hour) + if err := os.Chtimes(cachePath, old, old); err != nil { + t.Fatalf("age cache: %v", err) + } + gate := newRequestGate(t, sampleDataset) + c := modelsdev.New(modelsdev.Config{ + CachePath: cachePath, + URL: gate.server.URL, + TTL: time.Millisecond, + HTTPClient: gate.server.Client(), + }, logger.Default()) + + start := make(chan struct{}) + var wg sync.WaitGroup + for index := 0; index < 12; index++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + <-start + if index%2 == 0 { + _, _ = c.LookupForModel(context.Background(), "gpt-5.3-codex-spark") + return + } + _, _ = c.LookupModelInfo(context.Background(), "gpt-5.3-codex-spark") + }(index) + } + close(start) + wg.Wait() + gate.waitForFirstRequest(t) + if got := gate.requests.Load(); got != 1 { + t.Fatalf("background models.dev request count before release = %d, want 1", got) + } + refreshDone := make(chan error, 1) + go func() { refreshDone <- c.Refresh(context.Background()) }() + close(gate.release) + select { + case err := <-refreshDone: + if err != nil { + t.Fatalf("joined background Refresh: %v", err) + } + case <-time.After(time.Second): + t.Fatal("joined background Refresh did not complete") + } + if got := gate.requests.Load(); got != 1 { + t.Fatalf("background models.dev request count = %d, want 1", got) + } +} + +func TestClient_CanceledStaleLookupDoesNotCancelJoinedRefresh(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "models-dev.json") + if err := os.WriteFile(cachePath, []byte(sampleDataset), 0o644); err != nil { + t.Fatalf("seed cache: %v", err) + } + old := time.Now().Add(-time.Hour) + if err := os.Chtimes(cachePath, old, old); err != nil { + t.Fatalf("age cache: %v", err) + } + gate := newRequestGate(t, sampleDataset) + c := modelsdev.New(modelsdev.Config{ + CachePath: cachePath, + URL: gate.server.URL, + TTL: time.Millisecond, + HTTPClient: gate.server.Client(), + }, logger.Default()) + + lookupCtx, cancelLookup := context.WithCancel(context.Background()) + _, _ = c.LookupForModel(lookupCtx, "gpt-5.3-codex-spark") + gate.waitForFirstRequest(t) + cancelLookup() + + refreshDone := make(chan error, 1) + go func() { refreshDone <- c.Refresh(context.Background()) }() + select { + case err := <-refreshDone: + t.Fatalf("joined refresh returned before the shared request completed: %v", err) + case <-time.After(25 * time.Millisecond): + } + + close(gate.release) + select { + case err := <-refreshDone: + if err != nil { + t.Fatalf("joined Refresh: %v", err) + } + case <-time.After(time.Second): + t.Fatal("joined Refresh did not complete") + } + if got := gate.requests.Load(); got != 1 { + t.Fatalf("models.dev request count after canceled lookup = %d, want 1", got) + } +} + +func TestClient_FailedRefreshPreservesCacheAndAllowsRetry(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "models-dev.json") + var fail atomic.Bool + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + if fail.Load() { + http.Error(w, "upstream unavailable", http.StatusBadGateway) + return + } + _, _ = w.Write([]byte(sampleDataset)) + })) + t.Cleanup(server.Close) + c := modelsdev.New(modelsdev.Config{ + CachePath: cachePath, + URL: server.URL, + TTL: time.Hour, + HTTPClient: server.Client(), + }, logger.Default()) + + if err := c.Refresh(context.Background()); err != nil { + t.Fatalf("initial Refresh: %v", err) + } + before, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("read initial cache: %v", err) + } + if _, ok := c.LookupForModel(context.Background(), "claude-opus-4-7"); !ok { + t.Fatal("initial cache lookup failed") + } + + fail.Store(true) + if err := c.Refresh(context.Background()); err == nil { + t.Fatal("failed Refresh unexpectedly succeeded") + } + after, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("read cache after failed refresh: %v", err) + } + if string(after) != string(before) { + t.Fatal("failed refresh replaced the valid cache") + } + if _, ok := c.LookupForModel(context.Background(), "claude-opus-4-7"); !ok { + t.Fatal("failed refresh discarded valid in-memory data") + } + + fail.Store(false) + if err := c.Refresh(context.Background()); err != nil { + t.Fatalf("retry Refresh: %v", err) + } + if got := requests.Load(); got != 3 { + t.Fatalf("request count after failed retry = %d, want 3", got) + } + assertNoCacheTemps(t, cachePath) +} + +func TestClient_CanceledRefreshAllowsRetryAndCleansTemporaryFile(t *testing.T) { + dir := t.TempDir() + cachePath := filepath.Join(dir, "models-dev.json") + firstStarted := make(chan struct{}) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + close(firstStarted) + <-r.Context().Done() + return + } + _, _ = w.Write([]byte(sampleDataset)) + })) + t.Cleanup(server.Close) + c := modelsdev.New(modelsdev.Config{ + CachePath: cachePath, + URL: server.URL, + TTL: time.Hour, + HTTPClient: server.Client(), + }, logger.Default()) + + ctx, cancel := context.WithCancel(context.Background()) + refreshDone := make(chan error, 1) + go func() { refreshDone <- c.Refresh(ctx) }() + select { + case <-firstStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for cancelable refresh") + } + cancel() + select { + case err := <-refreshDone: + if err == nil { + t.Fatal("canceled Refresh unexpectedly succeeded") + } + case <-time.After(time.Second): + t.Fatal("canceled Refresh did not return") + } + + if err := c.Refresh(context.Background()); err != nil { + t.Fatalf("retry Refresh: %v", err) + } + if got := requests.Load(); got != 2 { + t.Fatalf("request count after canceled retry = %d, want 2", got) + } + assertNoCacheTemps(t, cachePath) +} + +func assertNoCacheTemps(t *testing.T, cachePath string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(filepath.Dir(cachePath), "."+filepath.Base(cachePath)+".tmp-*")) + if err != nil { + t.Fatalf("glob cache temps: %v", err) + } + if len(matches) != 0 { + t.Fatalf("cache temporary files remain: %v", matches) + } +} diff --git a/apps/backend/internal/orchestrator/reconcile_liveness.go b/apps/backend/internal/orchestrator/reconcile_liveness.go index aca749b350..c9295a0cec 100644 --- a/apps/backend/internal/orchestrator/reconcile_liveness.go +++ b/apps/backend/internal/orchestrator/reconcile_liveness.go @@ -3,9 +3,13 @@ package orchestrator import ( "context" "errors" + "fmt" "go.uber.org/zap" + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/orchestrator/executor" "github.com/kandev/kandev/internal/task/models" ) @@ -20,6 +24,114 @@ type rowLivenessProber interface { RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness } +type startupCleanupDisposition string + +const ( + startupCleanupDispositionStopped startupCleanupDisposition = "stopped" + startupCleanupDispositionAlreadyAbsent startupCleanupDisposition = "already_absent" + startupCleanupDispositionPreserved startupCleanupDisposition = "preserved" +) + +type startupCleanupDecision struct { + disposition startupCleanupDisposition + liveness string + stopErrorClass string + proceed bool + expected bool +} + +// classifyStartupCleanup is the single decision matrix for startup stop +// results. Only a typed runtime absence plus confirmed-dead local liveness +// permits cleanup to continue; every uncertain result remains preserved. +func classifyStartupCleanup(err error, liveness models.ProcessLiveness) startupCleanupDecision { + decision := startupCleanupDecision{ + disposition: startupCleanupDispositionPreserved, + liveness: processLivenessClass(liveness), + stopErrorClass: "none", + } + if err == nil { + decision.disposition = startupCleanupDispositionStopped + decision.proceed = true + return decision + } + if stopReportsRuntimeAbsent(err) { + decision.stopErrorClass = "runtime_not_found" + if liveness == models.ProcessLivenessDead { + decision.disposition = startupCleanupDispositionAlreadyAbsent + decision.proceed = true + return decision + } + decision.expected = true + return decision + } + decision.stopErrorClass = "unexpected" + return decision +} + +func processLivenessClass(liveness models.ProcessLiveness) string { + switch liveness { + case models.ProcessLivenessAlive: + return "alive" + case models.ProcessLivenessDead: + return "dead" + default: + return "unknown" + } +} + +type startupCleanupReport struct { + expectedPreservedCount int + expectedOutcomes map[string]int +} + +func newStartupCleanupReport() *startupCleanupReport { + return &startupCleanupReport{expectedOutcomes: make(map[string]int)} +} + +func (r *startupCleanupReport) recordExpected(running *models.ExecutorRunning, decision startupCleanupDecision) { + if r == nil { + return + } + key := fmt.Sprintf("disposition=%s,liveness=%s,stop_error=%s,local_pid=%t", + decision.disposition, decision.liveness, decision.stopErrorClass, + running != nil && running.LocalPID > 0) + r.expectedPreservedCount++ + r.expectedOutcomes[key]++ +} + +func (r *startupCleanupReport) flush(log *logger.Logger) { + if r == nil || r.expectedPreservedCount == 0 { + return + } + log.Warn("startup runtime cleanup summary", + zap.Int("expected_preserved_count", r.expectedPreservedCount), + zap.Any("expected_preserved_outcomes", r.expectedOutcomes)) +} + +func startupCleanupFields(running *models.ExecutorRunning, decision startupCleanupDecision) []zap.Field { + fields := []zap.Field{ + zap.String("liveness", decision.liveness), + zap.String("stop_error_class", decision.stopErrorClass), + zap.String("disposition", string(decision.disposition)), + zap.Bool("local_pid_present", running != nil && running.LocalPID > 0), + } + if running == nil { + return fields + } + return append(fields, + zap.String("session_id", running.SessionID), + zap.String("task_id", running.TaskID), + zap.String("execution_id", running.AgentExecutionID), + ) +} + +// stopReportsRuntimeAbsent accepts only typed runtime/executor absence +// sentinels. Generic lookup errors remain retryable failures. +func stopReportsRuntimeAbsent(err error) bool { + return errors.Is(err, runtimeapi.ErrNotFound) || + errors.Is(err, executor.ErrExecutionNotFound) +} + // rowLiveness classifies a row's backing-process liveness, returning Unknown when // no prober is wired (unit tests, degraded startup) so a caller never mistakes // "can't probe" for "dead". The probe is runtime-aware: a local process check diff --git a/apps/backend/internal/orchestrator/reconcile_liveness_test.go b/apps/backend/internal/orchestrator/reconcile_liveness_test.go new file mode 100644 index 0000000000..cbcf3a5590 --- /dev/null +++ b/apps/backend/internal/orchestrator/reconcile_liveness_test.go @@ -0,0 +1,145 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + runtimeapi "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +func TestClassifyStartupCleanupDecision(t *testing.T) { + tests := []struct { + name string + err error + liveness models.ProcessLiveness + wantDisposition startupCleanupDisposition + wantProceed bool + wantExpected bool + }{ + { + name: "successful stop", + liveness: models.ProcessLivenessAlive, + wantDisposition: startupCleanupDispositionStopped, + wantProceed: true, + }, + { + name: "not found dead local", + err: runtimeapi.ErrNotFound, + liveness: models.ProcessLivenessDead, + wantDisposition: startupCleanupDispositionAlreadyAbsent, + wantProceed: true, + }, + { + name: "not found alive local", + err: runtimeapi.ErrNotFound, + liveness: models.ProcessLivenessAlive, + wantDisposition: startupCleanupDispositionPreserved, + wantExpected: true, + }, + { + name: "not found unknown", + err: runtimeapi.ErrNotFound, + liveness: models.ProcessLivenessUnknown, + wantDisposition: startupCleanupDispositionPreserved, + wantExpected: true, + }, + { + name: "unexpected stop failure", + err: errors.New("database unavailable"), + liveness: models.ProcessLivenessDead, + wantDisposition: startupCleanupDispositionPreserved, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + decision := classifyStartupCleanup(tt.err, tt.liveness) + if decision.disposition != tt.wantDisposition { + t.Fatalf("disposition = %q, want %q", decision.disposition, tt.wantDisposition) + } + if decision.proceed != tt.wantProceed { + t.Fatalf("proceed = %v, want %v", decision.proceed, tt.wantProceed) + } + if decision.expected != tt.wantExpected { + t.Fatalf("expected = %v, want %v", decision.expected, tt.wantExpected) + } + }) + } +} + +func TestStartupCleanupReportAggregatesExpectedPreservationWithoutSecrets(t *testing.T) { + core, logs := observer.New(zap.WarnLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("NewFromZap: %v", err) + } + report := newStartupCleanupReport() + row := &models.ExecutorRunning{ + SessionID: "session-safe", + TaskID: "task-safe", + AgentExecutionID: "exec-safe", + ResumeToken: "resume-token-must-not-log", + } + decision := classifyStartupCleanup(runtimeapi.ErrNotFound, models.ProcessLivenessUnknown) + report.recordExpected(row, decision) + report.recordExpected(row, decision) + report.flush(log) + + if logs.Len() != 1 { + t.Fatalf("summary log count = %d, want 1", logs.Len()) + } + entry := logs.All()[0] + if entry.Message != "startup runtime cleanup summary" { + t.Fatalf("summary message = %q", entry.Message) + } + got := fmt.Sprint(entry.ContextMap()) + if !containsAll(got, "expected_preserved_count", "runtime_not_found", "unknown") { + t.Fatalf("summary fields = %s", got) + } + if containsAll(got, "resume-token-must-not-log") { + t.Fatal("summary leaked a resume token") + } +} + +func TestStopRuntimeForStartupCleanupPreservesUnknownRowsWithoutHandle(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + now := time.Now().UTC() + seedTaskAndSession(t, repo, "taskUnknown", "sessionUnknown", models.TaskSessionStateCancelled) + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "sessionUnknown", SessionID: "sessionUnknown", TaskID: "taskUnknown", + Runtime: agentruntime.RuntimeSSH, Status: models.ExecutorRunningStatusRunning, + PID: 9191, CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatalf("upsert unknown row: %v", err) + } + + service := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), &mockAgentManager{ + rowLivenessFn: func(*models.ExecutorRunning) models.ProcessLiveness { + return models.ProcessLivenessUnknown + }, + }) + service.reconcileSessionsOnStartup(ctx) + + if _, err := repo.GetExecutorRunningBySessionID(ctx, "sessionUnknown"); err != nil { + t.Fatalf("unknown row without a stop handle must be preserved: %v", err) + } +} + +func containsAll(value string, needles ...string) bool { + for _, needle := range needles { + if !strings.Contains(value, needle) { + return false + } + } + return true +} diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index 886d0956e0..d44be00c13 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" - runtimeapi "github.com/kandev/kandev/internal/agent/runtime" "github.com/kandev/kandev/internal/agent/runtime/routingerr" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" @@ -1739,6 +1738,7 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) { s.logger.Info("reconciling sessions on startup (lazy recovery)", zap.Int("count", len(runningExecutors))) + report := newStartupCleanupReport() var remoteRecords []executor.RemoteStatusPollRequest for _, running := range runningExecutors { if models.IsRemoteExecutorType(models.ExecutorType(running.Runtime)) { @@ -1750,8 +1750,9 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) { Metadata: running.Metadata, }) } - s.reconcileOneSessionOnStartup(ctx, running) + s.reconcileOneSessionOnStartup(ctx, running, report) } + report.flush(s.logger) // Pre-poll remote executor status so task lists show accurate state if len(remoteRecords) > 0 && s.agentManager != nil { @@ -1760,7 +1761,7 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) { } // reconcileOneSessionOnStartup adjusts DB state for a single session without launching agents. -func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning) { +func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) { sessionID := running.SessionID if sessionID == "" { return @@ -1769,7 +1770,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod session, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil { if isTaskSessionNotFound(err) { - s.handleMissingSessionOnStartup(ctx, running) + s.handleMissingSessionOnStartup(ctx, running, report) return } s.logger.Warn("failed to load session for reconciliation; preserving executor record", @@ -1781,7 +1782,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod previousState := session.State // Handle terminal and never-started states (reuse existing cleanup logic) - if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState); skip { + if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState, report); skip { return } if previousState == models.TaskSessionStateCreated { @@ -1819,6 +1820,15 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod return } + s.reconcileActiveSessionOnStartup(ctx, running, sessionID, previousState) +} + +func (s *Service) reconcileActiveSessionOnStartup( + ctx context.Context, + running *models.ExecutorRunning, + sessionID string, + previousState models.TaskSessionState, +) { // Active states: STARTING, RUNNING, WAITING_FOR_INPUT // Set session to WAITING_FOR_INPUT (idle, ready for lazy resume when user opens it) if previousState != models.TaskSessionStateWaitingForInput { @@ -1892,13 +1902,19 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod zap.Bool("has_worktree", running.WorktreePath != "")) } -func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning) { +func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) { sessionID := running.SessionID executionID := strings.TrimSpace(running.AgentExecutionID) if executionID == "" || s.agentManager == nil { - s.logger.Warn("executor record has no session and no stoppable runtime handle; preserving record", - zap.String("session_id", sessionID), - zap.String("task_id", running.TaskID)) + decision := startupCleanupDecision{ + disposition: startupCleanupDispositionPreserved, + liveness: processLivenessClass(s.rowLiveness(running)), + stopErrorClass: "missing_stop_handle", + expected: true, + } + report.recordExpected(running, decision) + s.logger.Debug("executor record has no stoppable runtime handle; preserving record", + startupCleanupFields(running, decision)...) return } if err := s.agentManager.StopAgentWithReason(ctx, executionID, "startup missing session cleanup", true); err != nil { @@ -1908,18 +1924,20 @@ func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *mo // invariant, so the orphan row does not survive restarts forever. Any // other error (alive/unknown/remote row, or a non-not-found failure) is // preserved and left for a later attempt. - if !stopReportsRuntimeAbsent(err) || s.rowLiveness(running) != models.ProcessLivenessDead { - s.logger.Warn("failed to stop missing-session runtime; preserving executor record", - zap.String("session_id", sessionID), - zap.String("task_id", running.TaskID), - zap.String("execution_id", executionID), - zap.Error(err)) + decision := classifyStartupCleanup(err, s.rowLiveness(running)) + if !decision.proceed { + if decision.expected { + report.recordExpected(running, decision) + s.logger.Debug("failed to stop missing-session runtime; preserving executor record", + startupCleanupFields(running, decision)...) + } else { + s.logger.Warn("failed to stop missing-session runtime; preserving executor record", + startupCleanupFields(running, decision)...) + } return } s.logger.Info("missing-session runtime already gone (confirmed dead); pruning executor record", - zap.String("session_id", sessionID), - zap.String("task_id", running.TaskID), - zap.String("execution_id", executionID)) + startupCleanupFields(running, decision)...) s.pruneOrRepairExecutorRow(ctx, running, models.TaskSessionStateCancelled) return } @@ -1931,17 +1949,6 @@ func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *mo } } -// stopReportsRuntimeAbsent reports whether a stop error means the runtime is -// already gone (a typed not-found sentinel), as opposed to a transient failure. -// Only a typed sentinel counts — a generic store/lookup error must never be -// reinterpreted as an absent runtime. The runtime seam (lifecycleAdapter) -// normalizes the backend lifecycle not-found sentinel to runtimeapi.ErrNotFound, -// so reconciliation depends only on the public runtime and executor sentinels. -func stopReportsRuntimeAbsent(err error) bool { - return errors.Is(err, runtimeapi.ErrNotFound) || - errors.Is(err, executor.ErrExecutionNotFound) -} - func (s *Service) abandonOpenTurnsOnStartup(ctx context.Context, sessionID, reason string) { if s.turnService == nil { return @@ -1961,7 +1968,7 @@ func isTaskSessionNotFound(err error) bool { // handleTerminalSessionOnStartup processes sessions in terminal states during startup. // Returns true if the session should be skipped (no further processing needed). -func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState) bool { +func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState, report *startupCleanupReport) bool { sessionID := session.ID switch previousState { case models.TaskSessionStateCompleted, models.TaskSessionStateCancelled: @@ -1970,7 +1977,7 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m zap.String("task_id", session.TaskID), zap.String("state", string(previousState))) s.abandonOpenTurnsOnStartup(ctx, sessionID, "terminal session cleanup") - if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup") { + if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup", report) { return true } // Resume-safety invariant: prune the row only if it is not still resumable @@ -1980,14 +1987,14 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m s.pruneOrRepairExecutorRow(ctx, running, previousState) return true case models.TaskSessionStateFailed: - s.handleFailedSessionOnStartup(ctx, session, running) + s.handleFailedSessionOnStartup(ctx, session, running, report) return true } return false } // handleFailedSessionOnStartup handles a failed session during startup recovery. -func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning) { +func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, report *startupCleanupReport) { sessionID := session.ID s.abandonOpenTurnsOnStartup(ctx, sessionID, "failed session cleanup") // If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS) @@ -2015,7 +2022,7 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod s.logger.Info("stopping failed session runtime before cleaning up executor record", zap.String("session_id", sessionID), zap.String("task_id", session.TaskID)) - if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup") { + if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup", report) { return } // Prune only subject to the resume-safety invariant (a lingering @@ -2031,30 +2038,50 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod // leaking stale rows: a not-found stop for a confirmed-dead LOCAL runtime means // the runtime is already gone and cleanup proceeds; any other error // (alive/unknown/remote row, or a non-not-found failure) preserves the row for a -// later attempt. A row with no stoppable handle is already stoppable-complete. -func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string) bool { +// later attempt. A row with no stoppable handle proceeds only when its local +// process liveness is confirmed dead; alive and Unknown rows remain durable. +func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string, report *startupCleanupReport) bool { executionID := strings.TrimSpace(running.AgentExecutionID) if executionID == "" || s.agentManager == nil { + liveness := s.rowLiveness(running) + decision := startupCleanupDecision{ + disposition: startupCleanupDispositionPreserved, + liveness: processLivenessClass(liveness), + stopErrorClass: "missing_stop_handle", + expected: true, + } + if liveness == models.ProcessLivenessDead { + decision.disposition = startupCleanupDispositionAlreadyAbsent + decision.proceed = true + decision.expected = false + } + if !decision.proceed { + report.recordExpected(running, decision) + s.logger.Debug("session runtime has no stoppable handle; preserving executor record", + append(startupCleanupFields(running, decision), zap.String("reason", reason))...) + return false + } + s.logger.Info("session runtime is confirmed dead without a stop handle; proceeding to prune/repair", + append(startupCleanupFields(running, decision), zap.String("reason", reason))...) return true } err := s.agentManager.StopAgentWithReason(ctx, executionID, reason, true) if err == nil { return true } - if !stopReportsRuntimeAbsent(err) || s.rowLiveness(running) != models.ProcessLivenessDead { - s.logger.Warn("failed to stop session runtime during startup cleanup; preserving executor record", - zap.String("session_id", running.SessionID), - zap.String("task_id", running.TaskID), - zap.String("execution_id", executionID), - zap.String("reason", reason), - zap.Error(err)) + decision := classifyStartupCleanup(err, s.rowLiveness(running)) + if !decision.proceed { + fields := append(startupCleanupFields(running, decision), zap.String("reason", reason)) + if decision.expected { + report.recordExpected(running, decision) + s.logger.Debug("failed to stop session runtime during startup cleanup; preserving executor record", fields...) + } else { + s.logger.Warn("failed to stop session runtime during startup cleanup; preserving executor record", fields...) + } return false } s.logger.Info("session runtime already gone (confirmed dead) during startup cleanup; proceeding to prune/repair", - zap.String("session_id", running.SessionID), - zap.String("task_id", running.TaskID), - zap.String("execution_id", executionID), - zap.String("reason", reason)) + append(startupCleanupFields(running, decision), zap.String("reason", reason))...) return true } diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index dda9ae2323..8837c1634e 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -335,7 +335,8 @@ const ( // System maintenance jobs (VACUUM, factory reset, snapshot create/restore, // disk walk). Broadcast to all connected clients so the System pages can // render progress. - ActionSystemJobUpdate = "system.job.update" + ActionSystemJobUpdate = "system.job.update" + ActionSystemAgentRuntimeStatusChanged = "system.agent_runtime.status_changed" // VS Code server actions ActionVscodeStart = "vscode.start" // Start code-server for a session diff --git a/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx b/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx new file mode 100644 index 0000000000..21773a6001 --- /dev/null +++ b/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx @@ -0,0 +1,144 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { StateProvider, useAppStoreApi } from "@/components/state-provider"; +import type { RestartCapabilityState } from "@/hooks/domains/system/use-restart-capability"; +import type { KandevRestartController } from "@/hooks/domains/system/use-kandev-restart"; +import { AgentRuntimeUnavailableAlert } from "./agent-runtime-unavailable-alert"; + +const RESTART_BUTTON_LABEL = "Restart Kandev"; + +const restartCapability = vi.hoisted(() => ({ + value: { + status: "resolved", + capability: { supported: true, mode: "supervisor", adapter: "supervisor" }, + } as RestartCapabilityState, +})); +const restartController = vi.hoisted(() => ({ + value: { + phase: "idle", + errorMessage: null, + isRestarting: false, + start: vi.fn(), + dismiss: vi.fn(), + } as KandevRestartController, +})); + +vi.mock("@/hooks/domains/system/use-restart-capability", () => ({ + useRestartCapability: () => restartCapability.value, +})); + +vi.mock("@/hooks/domains/system/use-kandev-restart", () => ({ + useKandevRestart: () => restartController.value, +})); + +vi.mock("@/components/settings/system/restart-progress-dialog", () => ({ + RestartProgressDialog: ({ phase }: { phase: string }) => + phase === "idle" ? null :
, +})); + +function RuntimeStateControls() { + const store = useAppStoreApi(); + return ( + <> + + ) : ( +

+ {capabilityLoading + ? t("system:agentRuntimeCheckingRestart") + : t("system:agentRuntimeUnavailableManual")} +

+ )} + + +
+ + + ); +} diff --git a/apps/web/components/app-status-bar/app-status-surface-provider.test.tsx b/apps/web/components/app-status-bar/app-status-surface-provider.test.tsx index 67689bd895..243d1771ef 100644 --- a/apps/web/components/app-status-bar/app-status-surface-provider.test.tsx +++ b/apps/web/components/app-status-bar/app-status-surface-provider.test.tsx @@ -8,6 +8,16 @@ const responsiveState = vi.hoisted(() => ({ breakpoint: "desktop" as "mobile" | "tablet" | "desktop", })); const featureState = vi.hoisted(() => ({ appStatusBar: true })); +const restartState = vi.hoisted(() => ({ + capability: { status: "resolved", capability: { supported: false, mode: "manual" } }, + restart: { + phase: "idle", + errorMessage: null, + isRestarting: false, + start: vi.fn(), + dismiss: vi.fn(), + }, +})); const STATUS_BAR_TEST_ID = "app-status-bar"; const STATUS_DRAWER_TEST_ID = "app-status-drawer"; const STATUS_DRAWER_TRIGGER_TEST_ID = "app-status-drawer-trigger"; @@ -29,6 +39,14 @@ vi.mock("@/hooks/domains/features/use-feature", () => ({ useFeature: (name: string) => (name === "appStatusBar" ? featureState.appStatusBar : true), })); +vi.mock("@/hooks/domains/system/use-restart-capability", () => ({ + useRestartCapability: () => restartState.capability, +})); + +vi.mock("@/hooks/domains/system/use-kandev-restart", () => ({ + useKandevRestart: () => restartState.restart, +})); + vi.mock("./app-status-bar", () => ({ AppStatusBar: () =>
, })); @@ -119,6 +137,21 @@ describe("AppStatusSurfaceProvider", () => { expect(screen.queryByTestId(STATUS_DRAWER_TRIGGER_TEST_ID)).toBeNull(); }); + it("keeps the runtime alert visible when the app-status-bar feature is disabled", () => { + featureState.appStatusBar = false; + renderSurface({ + agentRuntime: { + status: "unavailable", + reason: "agentctl_exited", + occurred_at: "2026-08-08T14:22:52Z", + }, + }); + + expect(screen.getByRole("alert")).toBeTruthy(); + expect(screen.queryByTestId(STATUS_BAR_TEST_ID)).toBeNull(); + expect(screen.queryByTestId(STATUS_DRAWER_TEST_ID)).toBeNull(); + }); + it("exposes a connection-only Status drawer for an active phone warning when disabled", () => { responsiveState.breakpoint = "mobile"; featureState.appStatusBar = false; diff --git a/apps/web/components/app-status-bar/app-status-surface-provider.tsx b/apps/web/components/app-status-bar/app-status-surface-provider.tsx index e2624e175f..50c99321e5 100644 --- a/apps/web/components/app-status-bar/app-status-surface-provider.tsx +++ b/apps/web/components/app-status-bar/app-status-surface-provider.tsx @@ -21,6 +21,7 @@ import { AppStatusBar } from "./app-status-bar"; import { AppStatusDrawer } from "./app-status-drawer"; import { connectionIssueDetails } from "./connection-status-item"; import type { ConnectionIssueSeverity } from "@/lib/types/connection"; +import { AgentRuntimeUnavailableAlert } from "./agent-runtime-unavailable-alert"; type AppStatusDrawerContextValue = { enabled: boolean; @@ -148,6 +149,7 @@ export function AppStatusSurfaceProvider({ children }: { children: ReactNode }) return (
+ {children} {(useDrawerSurface ? drawerEnabled : appStatusBarEnabled) && (useDrawerSurface ? ( diff --git a/apps/web/e2e/helpers/agent-runtime-availability.ts b/apps/web/e2e/helpers/agent-runtime-availability.ts new file mode 100644 index 0000000000..62eacbfe82 --- /dev/null +++ b/apps/web/e2e/helpers/agent-runtime-availability.ts @@ -0,0 +1,52 @@ +import type { Page } from "@playwright/test"; +import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime"; + +export type AgentRuntimeAvailabilitySnapshot = AgentRuntimeAvailability; + +type E2EStoreWindow = Window & { + __KANDEV_E2E_STORE__?: { + getState: () => { + setAgentRuntime: (snapshot: AgentRuntimeAvailabilitySnapshot | null) => void; + }; + }; +}; + +export async function setAgentRuntimeAvailability( + page: Page, + snapshot: AgentRuntimeAvailabilitySnapshot | null, +): Promise { + await page.evaluate((nextSnapshot) => { + const store = (window as E2EStoreWindow).__KANDEV_E2E_STORE__; + if (!store) throw new Error("E2E store bridge missing"); + store.getState().setAgentRuntime(nextSnapshot); + }, snapshot); +} + +export async function stubAgentRuntimeRestart(page: Page): Promise<() => number> { + let restartRequests = 0; + + await page.route("**/api/v1/system/restart-capability", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ supported: true, mode: "supervisor", adapter: "supervisor" }), + }), + ); + await page.route("**/api/v1/system/info", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ boot_id: "e2e-stable-boot" }), + }), + ); + await page.route("**/api/v1/system/restart", (route) => { + restartRequests += 1; + return route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ accepted: true, message: "accepted" }), + }); + }); + + return () => restartRequests; +} diff --git a/apps/web/e2e/tests/layout/agent-runtime-unavailable.spec.ts b/apps/web/e2e/tests/layout/agent-runtime-unavailable.spec.ts new file mode 100644 index 0000000000..6335897f70 --- /dev/null +++ b/apps/web/e2e/tests/layout/agent-runtime-unavailable.spec.ts @@ -0,0 +1,76 @@ +import type { Page } from "@playwright/test"; +import { expect, test } from "../../fixtures/test-base"; +import { + setAgentRuntimeAvailability, + stubAgentRuntimeRestart, +} from "../../helpers/agent-runtime-availability"; +import { KanbanPage } from "../../pages/kanban-page"; + +type E2EStoreWindow = Window & { + __KANDEV_E2E_STORE__?: { + getState: () => { + features: Record; + setFeatures: (features: Record) => void; + }; + }; +}; + +async function setAppStatusBarEnabled(page: Page, enabled: boolean): Promise { + await page.evaluate((nextEnabled) => { + const store = (window as E2EStoreWindow).__KANDEV_E2E_STORE__; + if (!store) throw new Error("E2E store bridge missing"); + const state = store.getState(); + state.setFeatures({ ...state.features, appStatusBar: nextEnabled }); + }, enabled); +} + +test.describe("Agent runtime availability", () => { + test("retains the current board, supports restart, and clears after recovery", async ({ + testPage, + apiClient, + seedData, + prCapture, + }) => { + const taskTitle = "Runtime availability retained board task"; + const task = await apiClient.createTask(seedData.workspaceId, taskTitle, { + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + }); + const restartRequestCount = await stubAgentRuntimeRestart(testPage); + const kanban = new KanbanPage(testPage); + + await kanban.goto(); + const taskCard = kanban.taskCard(task.id); + await expect(taskCard).toBeVisible(); + + await setAgentRuntimeAvailability(testPage, { + status: "unavailable", + reason: "agentctl_exited", + occurred_at: "2026-08-08T14:22:52Z", + }); + + const alert = testPage.getByTestId("agent-runtime-alert"); + await expect(alert).toBeVisible(); + await expect(alert).toContainText("Local agent runtime stopped"); + await expect(alert).toContainText("saved data remains safe"); + await expect(taskCard).toBeVisible(); + await prCapture.screenshot("agent-runtime-unavailable-desktop", { + caption: "Persistent desktop agent runtime recovery alert", + }); + + await setAppStatusBarEnabled(testPage, false); + await expect(testPage.getByTestId("app-status-bar")).toHaveCount(0); + await expect(alert).toBeVisible(); + + await alert.getByRole("button", { name: "Restart Kandev" }).click(); + await expect(testPage.getByTestId("restart-progress-dialog")).toHaveAttribute( + "data-phase", + "restarting", + ); + expect(restartRequestCount()).toBe(1); + + await setAgentRuntimeAvailability(testPage, { status: "available" }); + await expect(alert).toHaveCount(0); + await expect(taskCard).toBeVisible(); + }); +}); diff --git a/apps/web/e2e/tests/layout/mobile-agent-runtime-unavailable.spec.ts b/apps/web/e2e/tests/layout/mobile-agent-runtime-unavailable.spec.ts new file mode 100644 index 0000000000..1bfd05c263 --- /dev/null +++ b/apps/web/e2e/tests/layout/mobile-agent-runtime-unavailable.spec.ts @@ -0,0 +1,71 @@ +import { expect, test } from "../../fixtures/test-base"; +import { + setAgentRuntimeAvailability, + stubAgentRuntimeRestart, +} from "../../helpers/agent-runtime-availability"; +import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions"; +import { MobileKanbanPage } from "../../pages/mobile-kanban-page"; + +test.describe("Mobile agent runtime availability", () => { + test("keeps the task route safe around the bottom navigation", async ({ + testPage, + apiClient, + seedData, + }) => { + const taskTitle = "Mobile runtime availability task"; + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + taskTitle, + seedData.agentProfileId, + { + description: 'e2e:message("mobile runtime availability")', + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + executor_profile_id: seedData.worktreeExecutorProfileId, + }, + ); + if (!task.session_id) throw new Error("mobile runtime availability task has no session"); + + await stubAgentRuntimeRestart(testPage); + const mobile = new MobileKanbanPage(testPage); + await mobile.goto(); + await expect(mobile.taskCard(task.id)).toBeVisible(); + await mobile.taskCard(task.id).tap(); + await expect(testPage).toHaveURL(new RegExp(`/t/${task.id}$`)); + await expect(testPage.getByTestId("mobile-task-layout")).toBeVisible(); + + await setAgentRuntimeAvailability(testPage, { + status: "unavailable", + reason: "agentctl_exited", + occurred_at: "2026-08-08T14:22:52Z", + }); + + const alert = testPage.getByTestId("agent-runtime-alert"); + await expect(alert).toBeVisible(); + await expect(alert).toContainText("Local agent runtime stopped"); + await expect(testPage.getByTestId("mobile-task-layout")).toBeVisible(); + + const action = alert.getByRole("button", { name: "Restart Kandev" }); + const actionBox = await action.boundingBox(); + expect(actionBox, "runtime recovery action has no rendered hitbox").not.toBeNull(); + expect(actionBox?.height ?? 0).toBeGreaterThanOrEqual(44); + + const bottomNav = testPage.getByTestId("mobile-task-layout").locator("nav"); + await expect(bottomNav).toBeVisible(); + const alertBox = await alert.boundingBox(); + const bottomNavBox = await bottomNav.boundingBox(); + expect(alertBox).not.toBeNull(); + expect(bottomNavBox).not.toBeNull(); + if (alertBox && bottomNavBox) { + expect(alertBox.y + alertBox.height).toBeLessThanOrEqual(bottomNavBox.y + 1); + } + + await assertNoDocumentHorizontalOverflow(testPage, "mobile agent runtime alert"); + + await setAgentRuntimeAvailability(testPage, { status: "available" }); + await expect(alert).toHaveCount(0); + await expect(bottomNav).toBeVisible(); + await assertNoDocumentHorizontalOverflow(testPage, "mobile agent runtime recovery"); + }); +}); diff --git a/apps/web/lib/state/default-state.ts b/apps/web/lib/state/default-state.ts index cb0ca6354f..d05f0767d7 100644 --- a/apps/web/lib/state/default-state.ts +++ b/apps/web/lib/state/default-state.ts @@ -18,6 +18,7 @@ import { defaultReviewState, } from "./slices"; import { applyStoredQuickChatNames } from "@/lib/state/slices/ui/quick-chat-sync"; +import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime"; import type { HydrationState } from "./store"; import { migrateSidebarViewDraft, migrateView } from "./slices/ui/ui-slice"; @@ -104,6 +105,7 @@ export const defaultState = { automations: defaultAutomationsState.automations, automationRuns: defaultAutomationsState.automationRuns, system: defaultSystemState.system, + agentRuntime: null as AgentRuntimeAvailability | null, previewPanel: defaultUIState.previewPanel, rightPanel: defaultUIState.rightPanel, diffs: defaultUIState.diffs, diff --git a/apps/web/lib/state/hydration/hydrator.ts b/apps/web/lib/state/hydration/hydrator.ts index 6d50e99c6d..c281a33ee4 100644 --- a/apps/web/lib/state/hydration/hydrator.ts +++ b/apps/web/lib/state/hydration/hydrator.ts @@ -379,6 +379,7 @@ export function hydrateState( // subset they fetched, so use the same leaf-level deepMerge as the other // multi-field slices above rather than overwriting the whole object. if (state.system) deepMerge(draft.system, state.system); + if (state.agentRuntime !== undefined) draft.agentRuntime = state.agentRuntime; } /** Hydrate GitHub slices, preserving loading states. */ diff --git a/apps/web/lib/state/store-overrides.ts b/apps/web/lib/state/store-overrides.ts index 682e691083..c1966bc90e 100644 --- a/apps/web/lib/state/store-overrides.ts +++ b/apps/web/lib/state/store-overrides.ts @@ -78,6 +78,7 @@ export function buildStateOverrides(m: DefaultState) { automations: m.automations, automationRuns: m.automationRuns, system: m.system, + agentRuntime: m.agentRuntime, previewPanel: m.previewPanel, rightPanel: m.rightPanel, diffs: m.diffs, diff --git a/apps/web/lib/state/store.test.ts b/apps/web/lib/state/store.test.ts index 6cb597f313..1d2cba99df 100644 --- a/apps/web/lib/state/store.test.ts +++ b/apps/web/lib/state/store.test.ts @@ -2,6 +2,22 @@ import { describe, expect, it } from "vitest"; import { createAppStore, type AppState } from "./store"; describe("createAppStore", () => { + it("hydrates and replaces the full agent runtime availability snapshot", () => { + const unavailable = { + status: "unavailable" as const, + reason: "agentctl_exited" as const, + occurred_at: "2026-08-08T14:22:52Z", + }; + const available = { status: "available" as const }; + const store = createAppStore({ agentRuntime: unavailable }); + + expect(store.getState().agentRuntime).toEqual(unavailable); + + store.getState().setAgentRuntime(available); + + expect(store.getState().agentRuntime).toEqual(available); + }); + it("retains sidebar boot settings after slice initialization", () => { const store = createAppStore({ userSettings: { diff --git a/apps/web/lib/state/store.ts b/apps/web/lib/state/store.ts index b5c665e67d..efd6610c3f 100644 --- a/apps/web/lib/state/store.ts +++ b/apps/web/lib/state/store.ts @@ -11,6 +11,7 @@ import type { TaskWalkthrough, } from "@/lib/types/http"; import type { SystemHealthResponse } from "@/lib/types/health"; +import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime"; import type { UISliceActions as UIA } from "./slices/ui/types"; import type * as UISliceTypes from "./slices/ui/types"; import type { AgentUpdateJob, InstallJob } from "./slices/settings/types"; @@ -212,6 +213,7 @@ export type AppState = KanbanSlice & { // System slice (actions merged via SystemSliceActions intersection on AppState) system: (typeof defaultSystemState)["system"]; + agentRuntime: AgentRuntimeAvailability | null; // Plugins slice (actions merged via PluginsSliceActions intersection on AppState) plugins: (typeof defaultPluginsState)["plugins"]; @@ -342,6 +344,7 @@ export type AppState = KanbanSlice & { setSystemHealth: (response: SystemHealthResponse) => void; setSystemHealthLoading: (loading: boolean) => void; invalidateSystemHealth: () => void; + setAgentRuntime: (snapshot: AgentRuntimeAvailability | null) => void; openQuickChat: UIA["openQuickChat"]; addQuickChatSession: UIA["addQuickChatSession"]; reuseOrCreateQuickTerminal: UIA["reuseOrCreateQuickTerminal"]; @@ -582,6 +585,10 @@ export function createAppStore(initialState?: HydrationState) { ...createAuthSlice(set as any), // eslint-disable-next-line @typescript-eslint/no-explicit-any ...createSystemSlice(set as any, get as any, api as any), + setAgentRuntime: (snapshot) => + set((draft) => { + draft.agentRuntime = snapshot; + }), // eslint-disable-next-line @typescript-eslint/no-explicit-any ...createUISlice(set as any, get as any, api as any), // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/apps/web/lib/types/agent-runtime.ts b/apps/web/lib/types/agent-runtime.ts new file mode 100644 index 0000000000..f04883cdc6 --- /dev/null +++ b/apps/web/lib/types/agent-runtime.ts @@ -0,0 +1,11 @@ +export type AgentRuntimeAvailabilityStatus = "available" | "unavailable"; + +export type AgentRuntimeAvailabilityReason = "agentctl_exited"; + +export type AgentRuntimeAvailability = + | { status: Extract } + | { + status: Extract; + reason: AgentRuntimeAvailabilityReason; + occurred_at: string; + }; diff --git a/apps/web/lib/types/backend.ts b/apps/web/lib/types/backend.ts index 2ab46077b2..dbcbdfd586 100644 --- a/apps/web/lib/types/backend.ts +++ b/apps/web/lib/types/backend.ts @@ -2,6 +2,8 @@ import type { QueuedMessage } from "@/lib/state/slices/session/types"; import type { TaskPlanEventPayload, TaskPlanRevisionEventPayload } from "./task-plan-events"; import type { CaptureRequest } from "@/lib/logger/capture"; +export const SYSTEM_AGENT_RUNTIME_STATUS_CHANGED = "system.agent_runtime.status_changed" as const; + export type BackendMessageType = keyof BackendMessageMap; export type { BackendMessage } from "./backend-message"; @@ -33,6 +35,7 @@ import type { TaskMR } from "@/lib/types/gitlab"; import type { TaskStatusSummary } from "@/lib/types/task-status-summary"; import type { TaskMRAutomationOptions } from "@/lib/types/gitlab"; import type { SystemMetricsSnapshot } from "./system"; +import type { AgentRuntimeAvailability } from "./agent-runtime"; import type { FileChangeNotificationPayload } from "./workspace-files"; import type { AgentCapabilitiesPayload, @@ -533,6 +536,10 @@ export type BackendMessageMap = OfficeBackendMessageMap & "session.git.event": BackendMessage<"session.git.event", GitEventPayload>; "system.job.update": BackendMessage<"system.job.update", import("./system").SystemJob>; "system.metrics.updated": BackendMessage<"system.metrics.updated", SystemMetricsSnapshot>; + [SYSTEM_AGENT_RUNTIME_STATUS_CHANGED]: BackendMessage< + typeof SYSTEM_AGENT_RUNTIME_STATUS_CHANGED, + AgentRuntimeAvailability + >; "system.logs.capture_requested": BackendMessage< "system.logs.capture_requested", CaptureRequest diff --git a/apps/web/lib/ws/handlers/system-events.test.ts b/apps/web/lib/ws/handlers/system-events.test.ts new file mode 100644 index 0000000000..24140992e7 --- /dev/null +++ b/apps/web/lib/ws/handlers/system-events.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { createAppStore } from "@/lib/state/store"; +import { SYSTEM_AGENT_RUNTIME_STATUS_CHANGED } from "@/lib/types/backend"; +import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime"; +import { registerSystemEventsHandlers } from "./system-events"; + +const unavailable: AgentRuntimeAvailability = { + status: "unavailable", + reason: "agentctl_exited", + occurred_at: "2026-08-08T14:22:52Z", +}; + +const available: AgentRuntimeAvailability = { status: "available" }; + +describe("registerSystemEventsHandlers", () => { + it("replaces the complete runtime snapshot without touching domain state", () => { + const store = createAppStore({ + agentRuntime: unavailable, + tasks: { + activeTaskId: "task-1", + activeSessionId: null, + pinnedSessionId: null, + lastSessionByTaskId: {}, + }, + }); + const handlers = registerSystemEventsHandlers(store); + + handlers[SYSTEM_AGENT_RUNTIME_STATUS_CHANGED]?.({ + type: "notification", + action: SYSTEM_AGENT_RUNTIME_STATUS_CHANGED, + payload: available, + }); + + expect(store.getState().agentRuntime).toEqual(available); + expect(store.getState().tasks.activeTaskId).toBe("task-1"); + }); + + it("does not infer recovery from a connection status update", () => { + const store = createAppStore({ agentRuntime: unavailable }); + + store.getState().setConnectionStatus("connected"); + store.getState().setConnectionIssueSeverity("none"); + + expect(store.getState().agentRuntime).toEqual(unavailable); + }); +}); diff --git a/apps/web/lib/ws/handlers/system-events.ts b/apps/web/lib/ws/handlers/system-events.ts index 831ef43a4c..ea3a18b036 100644 --- a/apps/web/lib/ws/handlers/system-events.ts +++ b/apps/web/lib/ws/handlers/system-events.ts @@ -1,5 +1,6 @@ import type { StoreApi } from "zustand"; import type { AppState } from "@/lib/state/store"; +import { SYSTEM_AGENT_RUNTIME_STATUS_CHANGED } from "@/lib/types/backend"; import type { WsHandlers } from "@/lib/ws/handlers/types"; import { handleBrowserLogCapture } from "@/lib/logger/capture"; @@ -14,6 +15,9 @@ export function registerSystemEventsHandlers(store: StoreApi): WsHandl "system.metrics.updated": (message) => { store.getState().setSystemMetricsSnapshot(message.payload); }, + [SYSTEM_AGENT_RUNTIME_STATUS_CHANGED]: (message) => { + store.getState().setAgentRuntime(message.payload); + }, "system.logs.capture_requested": (message) => { const auth = store.getState().auth; const identityScope = auth.authenticated diff --git a/apps/web/src/locales/en/system.json b/apps/web/src/locales/en/system.json index 86a05c2ba3..7aa7c69541 100644 --- a/apps/web/src/locales/en/system.json +++ b/apps/web/src/locales/en/system.json @@ -203,6 +203,12 @@ "navUsers": "Users", "reloadPage": "Reload page", "restartAction": "Restart", + "agentRuntimeUnavailableTitle": "Local agent runtime stopped", + "agentRuntimeUnavailableBody": "Agent actions and workspace tools may not update while the local runtime is stopped. Your saved data remains safe.", + "agentRuntimeUnavailableRestartRequired": "Restart Kandev fully to recover.", + "agentRuntimeUnavailableRestart": "Restart Kandev", + "agentRuntimeUnavailableManual": "Restart Kandev from your terminal or service manager.", + "agentRuntimeCheckingRestart": "Checking restart support…", "restartManualHint": "Restart it from your terminal or service manager when convenient.", "restartPendingChanges": "Saved toggle changes will apply the next time Kandev starts.", "restartPhaseDoneBody": "The backend is running again. Reload the page to reconnect with the latest feature toggle state.", diff --git a/apps/web/src/locales/pseudo/system.json b/apps/web/src/locales/pseudo/system.json index 3a43f98f5d..d94069b8d9 100644 --- a/apps/web/src/locales/pseudo/system.json +++ b/apps/web/src/locales/pseudo/system.json @@ -203,6 +203,12 @@ "navUsers": "Ũśēŕś", "reloadPage": "Ŕēĺōàď ƥàĝē", "restartAction": "Ŕēśţàŕţ", + "agentRuntimeUnavailableTitle": "Ĺōćàĺ àĝēńţ ŕũńţĩḿē śţōƥƥēď", + "agentRuntimeUnavailableBody": "Àĝēńţ àćţĩōńś àńď ŵōŕķśƥàćē ţōōĺś ḿàŷ ńōţ ũƥďàţē ŵĥĩĺē ţĥē ĺōćàĺ ŕũńţĩḿē ĩś śţōƥƥēď. Ŷōũŕ śàvēď ďàţà ŕēḿàĩńś śàƒē.", + "agentRuntimeUnavailableRestartRequired": "Ŕēśţàŕţ Ķàńďēv ƒũĺĺŷ ţō ŕēćōvēŕ.", + "agentRuntimeUnavailableRestart": "Ŕēśţàŕţ Ķàńďēv", + "agentRuntimeUnavailableManual": "Ŕēśţàŕţ Ķàńďēv ƒŕōḿ ŷōũŕ ţēŕḿĩńàĺ ōŕ śēŕvĩćē ḿàńàĝēŕ.", + "agentRuntimeCheckingRestart": "Ćĥēćķĩńĝ ŕēśţàŕţ śũƥƥōŕţ…", "restartManualHint": "Ŕēśţàŕţ ĩţ ƒŕōḿ ŷōũŕ ţēŕḿĩńàĺ ōŕ śēŕvĩćē ḿàńàĝēŕ ŵĥēń ćōńvēńĩēńţ.", "restartPendingChanges": "Śàvēď ţōĝĝĺē ćĥàńĝēś ŵĩĺĺ àƥƥĺŷ ţĥē ńēxţ ţĩḿē Ķàńďēv śţàŕţś.", "restartPhaseDoneBody": "Ţĥē ƀàćķēńď ĩś ŕũńńĩńĝ àĝàĩń. Ŕēĺōàď ţĥē ƥàĝē ţō ŕēćōńńēćţ ŵĩţĥ ţĥē ĺàţēśţ ƒēàţũŕē ţōĝĝĺē śţàţē.", diff --git a/apps/web/src/locales/pt-pt/system.json b/apps/web/src/locales/pt-pt/system.json index 7dfa13af1d..574349d19c 100644 --- a/apps/web/src/locales/pt-pt/system.json +++ b/apps/web/src/locales/pt-pt/system.json @@ -203,6 +203,12 @@ "navUsers": "Utilizadores", "reloadPage": "Recarregar a página", "restartAction": "Reiniciar", + "agentRuntimeUnavailableTitle": "O runtime local do agente parou", + "agentRuntimeUnavailableBody": "As ações do agente e as ferramentas da área de trabalho podem não ser atualizadas enquanto o runtime local estiver parado. Os seus dados guardados continuam seguros.", + "agentRuntimeUnavailableRestartRequired": "Reinicie completamente o Kandev para recuperar.", + "agentRuntimeUnavailableRestart": "Reiniciar o Kandev", + "agentRuntimeUnavailableManual": "Reinicie o Kandev a partir do seu terminal ou gestor de serviços.", + "agentRuntimeCheckingRestart": "A verificar o suporte de reinício…", "restartManualHint": "Reinicie-o a partir do seu terminal ou gestor de serviços quando for conveniente.", "restartPendingChanges": "As alterações guardadas aos interruptores serão aplicadas no próximo arranque do Kandev.", "restartPhaseDoneBody": "O backend está novamente em execução. Recarregue a página para voltar a ligar com o estado mais recente dos interruptores de funcionalidades.", diff --git a/apps/web/src/locales/zh-cn/system.json b/apps/web/src/locales/zh-cn/system.json index 3a20b24227..874a802a97 100644 --- a/apps/web/src/locales/zh-cn/system.json +++ b/apps/web/src/locales/zh-cn/system.json @@ -203,6 +203,12 @@ "navUsers": "用户", "reloadPage": "重新加载页面", "restartAction": "重启", + "agentRuntimeUnavailableTitle": "本地代理运行时已停止", + "agentRuntimeUnavailableBody": "本地运行时停止后,代理操作和工作区工具可能不会更新。你保存的数据仍然安全。", + "agentRuntimeUnavailableRestartRequired": "请完全重启 Kandev 以恢复运行。", + "agentRuntimeUnavailableRestart": "重启 Kandev", + "agentRuntimeUnavailableManual": "请从终端或服务管理器重启 Kandev。", + "agentRuntimeCheckingRestart": "正在检查重启支持…", "restartManualHint": "方便时从终端或服务管理器重启。", "restartPendingChanges": "已保存的开关更改将在下次启动 Kandev 时生效。", "restartPhaseDoneBody": "后端已重新运行。请重新加载页面以使用最新功能开关状态重新连接。", diff --git a/docs/decisions/2026-08-08-agentctl-crash-containment.md b/docs/decisions/2026-08-08-agentctl-crash-containment.md new file mode 100644 index 0000000000..59467bdad6 --- /dev/null +++ b/docs/decisions/2026-08-08-agentctl-crash-containment.md @@ -0,0 +1,110 @@ +# ADR-2026-08-08-agentctl-crash-containment: Publish Immutable Agent Events and Explicit Runtime Failure State + +**Status:** proposed +**Date:** 2026-08-08 +**Area:** backend, frontend, protocol, operations + +## Context + +The standalone `agentctl` process crashed with Go's fatal `concurrent map +iteration and map write` while `NormalizedPayload.MarshalJSON` serialized a tool +update. ACP adapter state is mutated under `Adapter.mu`, but emitted events can +retain pointers to that state after the lock is released. The existing +`cloneSubagentPayload` helper snapshots subagent payloads only; generic +`Input`, `Output`, and miscellaneous `Details` maps can still be updated while +the HTTP stream writer marshals an earlier event. + +The backend survives an `agentctl` child exit. Its HTTP server and browser +WebSocket therefore remain reachable even though the local agent runtime has +stopped. Browser connectivity alone cannot identify the outage, and the current +UI gives no durable feedback when task and workspace activity becomes stale. + +Restarting only the child is not a local launcher concern. The per-launch auth +token, PID, control clients, lifecycle manager, host utility manager, and active +execution metadata are captured by multiple owners during startup. Replacing +the process without atomically rebinding those owners would create a nominally +healthy but internally split runtime. + +## Decision + +Kandev contains this failure at two boundaries: + +1. `NormalizedPayload` gains a complete immutable snapshot operation. Every ACP + event snapshots its payload while `Adapter.mu` is held and before the event + crosses the updates-channel boundary. The snapshot copies all typed pointer + and slice fields and recursively copies JSON-compatible values held in + `GenericPayload.Input`, `GenericPayload.Output`, and `MiscPayload.Details`. + The one-off subagent clone is replaced by this common ownership rule. +2. The launcher reports unexpected child exit through a callback after it has + distinguished the event from intentional shutdown. A concurrency-safe, + install-wide availability owner records an `unavailable` snapshot and + publishes a stable `agentctl_exited` reason. Raw process details remain in + logs. +3. The current availability snapshot is distributed through the boot/app-state + payload, a global system WebSocket notification, and replay after user + subscription. The event is install-wide and intentionally uses the gateway's + global broadcast path. +4. The application shell renders a persistent, non-dismissible runtime alert + independent of the optional App status bar. It preserves last-known data and + reuses the existing supervisor capability and restart flow when available; + unsupported launch modes receive manual guidance. + +Availability is in memory and monotonic within one backend boot. Recovery is a +full supervised Kandev restart, which creates a fresh backend and authenticated +child together. The general backend health endpoint remains unchanged. + +The observable behavior is specified in +[`docs/specs/platform/agent-runtime-availability.md`](../specs/platform/agent-runtime-availability.md). + +## Consequences + +- Stream consumers receive values they own, so later ACP updates cannot race + their serialization or retroactively change an earlier event. +- An unexpected child exit becomes visible even when browser-to-backend + connectivity remains healthy, and late clients converge on the same state. +- The first repair avoids a partial child restart that would leave stale + credentials and process identity in downstream services. +- The unavailable alert can coexist with the WebSocket connectivity warning + because they report separate backend-child and browser-backend boundaries. +- Snapshotting adds bounded copy work for tool events. Payload size is already + bounded for stream presentation, and correctness takes precedence at this + asynchronous ownership boundary. + +## Alternatives Considered + +### Recover around JSON marshaling + +Rejected. Go's concurrent-map runtime failure is fatal and cannot be reliably +recovered. The mutable alias must not reach the serializer. + +### Copy only generic maps in `MarshalJSON` + +Rejected. Marshaling occurs after adapter ownership has been released, so a +copy attempted there can race the same mutation. It also leaves earlier queued +events mutable and misses typed slices and nested values. + +### Keep the subagent-only clone and patch the observed field + +Rejected. The bug is an ownership violation shared by every mutable normalized +payload, not a one-field exception. Another payload kind could recreate the +same crash or event-history corruption. + +### Restart `agentctl` in process + +Deferred. It requires one coordinated owner for child generation, auth token, +PID, clients, lifecycle state, host utility state, and active execution +reconciliation. A future design may supersede this decision once that rebinding +contract exists. + +### Exit the backend when the child exits + +Rejected for this repair. An arbitrary backend exit is not guaranteed to invoke +the explicit supervisor restart protocol and can terminate a normal CLI launch. +Keeping the backend available preserves saved data, diagnostics, and the +supported user-initiated restart endpoint. + +### Rely on the existing connectivity warning, toast, or App status bar + +Rejected. The browser WebSocket can remain connected, toasts are transient, and +the status bar is optional. A core-runtime outage needs persistent shell-level +feedback regardless of route state or feature flags. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index e35ce445fb..2c88d73492 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -130,4 +130,5 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-08-05-homebrew-remote-helper-audit | [Preserve Remote Helpers in Homebrew Installs](2026-08-05-homebrew-remote-helper-audit.md) | accepted | infra, workflow | 2026-08-05 | | 2026-08-07-model-aware-provider-capability-resolution | [Model-aware provider capability resolution](2026-08-07-model-aware-provider-capability-resolution.md) | accepted | backend, frontend, protocol | 2026-08-07 | | 2026-08-07-claude-allowlist-label-bridge | [Use the Claude Allowlist as a Trusted Preview Gate](2026-08-07-claude-allowlist-label-bridge.md) | accepted | infra, workflow, security | 2026-08-07 | +| 2026-08-08-agentctl-crash-containment | [Publish Immutable Agent Events and Explicit Runtime Failure State](2026-08-08-agentctl-crash-containment.md) | proposed | backend, frontend, protocol, operations | 2026-08-08 | | 2026-08-08-workspace-scoped-task-create-workflow-memory | [Remember Task-Create Workflows Per Workspace](2026-08-08-workspace-scoped-task-create-workflow-memory.md) | accepted | backend, frontend | 2026-08-08 | diff --git a/docs/plans/backend-failure-containment/plan.md b/docs/plans/backend-failure-containment/plan.md new file mode 100644 index 0000000000..635e01d30b --- /dev/null +++ b/docs/plans/backend-failure-containment/plan.md @@ -0,0 +1,255 @@ +--- +spec: docs/specs/platform/agent-runtime-availability.md +related_specs: + - docs/specs/office/costs.md + - docs/specs/tasks/runtime-cleanup.md + - docs/specs/ui/entity-reference-composer.md +created: 2026-08-08 +status: completed +--- + +# Implementation Plan: Backend Failure Containment + +## Overview + +Contain four failures found in the August 8 incident review: + +1. Prevent the fatal `agentctl` concurrent-map crash and publish an explicit + install-wide runtime outage to the UI. +2. Coalesce `models.dev` refreshes and preserve atomic cache replacement. +3. Make startup runtime cleanup decisions explicit, idempotent, and bounded in + logs without weakening fail-closed row preservation. +4. Treat canceled mention searches as normal client cancellation and guarantee + structured diagnostics for every real HTTP 500. + +The work uses the existing subsystem boundaries. It does not add automatic +`agentctl` child restart, delete runtime rows with uncertain liveness, or turn +provider-level mention failures into aggregate request failures. + +## Confirmed causes and evidence limits + +### Agentctl crash and silent UI stall + +`convertToolCallResultUpdate` mutates normalized tool payloads under +`Adapter.mu`, but `cloneSubagentPayload` returns the original pointer for every +non-subagent payload. `sendUpdate` queues that pointer and releases the lock. +The HTTP stream writer later marshals it while another update can mutate a +nested map. The fatal stack ends in `NormalizedPayload.MarshalJSON` with +`concurrent map iteration and map write`. + +The launcher records an unexpected child exit but publishes no health state. +The backend and browser WebSocket remain connected, so the UI has no signal +that agent and workspace activity is unavailable. + +### Models.dev cache race + +`warmFromDisk` and `maybeRefresh` can both launch background refreshes, and each +stale lookup can launch another. Direct `Refresh` calls are also uncoordinated. +Every refresh writes the same `.tmp`; one caller can rename or remove +that path before another caller renames it, matching the observed `ENOENT`. + +### Startup cleanup warnings + +The runtime, lifecycle, and executor layers already preserve typed not-found +identity, and startup cleanup already removes typed-not-found rows only when +local liveness is confirmed dead. The pre-start database shows the warned +legacy standalone rows have `local_pid=0`; liveness therefore classifies them as +Unknown and correctly preserves them. The defect is ambiguous per-row warning +flooding and incomplete decision evidence, not a safe basis for deleting those +rows. + +### Mention-search HTTP 500 + +The handler maps every error other than invalid input and missing workspace to +500 without logging the underlying error. Provider failures normally remain in +HTTP 200 groups. The frontend cancels superseded debounced requests, while the +backend cancellation composition test currently expects that cancellation to +become 500. This deterministic path must become 499. Because the historical 500 +had no error record, its exact path cannot be proven after the fact; the fix +also makes every future unexpected 500 observable. + +## Backend design + +### Immutable ACP event boundary + +- Add `NormalizedPayload.Snapshot` in + `apps/backend/internal/agentctl/types/streams/tool_payload.go`. Copy pointers, + slices, and recursively copy JSON-compatible map/slice values. +- Snapshot `AgentEvent.Payload` while `Adapter.mu` is held in `sendUpdate` and + `sendUpdateLocked`, before the event enters `updatesCh`. +- Replace `cloneSubagentPayload` with the common snapshot contract. +- Add deterministic alias tests and concurrent update/marshal race coverage. + +### Agent runtime availability + +- Add a concurrency-safe tracker under + `apps/backend/internal/agent/runtime/agentctl/availability.go`. It owns the + public `available|unavailable` snapshot, stable reason, occurrence time, and + event publication. +- Extend launcher configuration with an unexpected-exit callback. Invoke it + only after exit monitoring confirms shutdown was not intentional. +- Mark the runtime available only after child health and authentication succeed. +- Add an internal system event and WebSocket action for status changes. Replay + the latest full snapshot after each authenticated `user.subscribe`. +- Add the snapshot to authenticated boot/app-state payloads. Keep raw exit text + and unauthenticated runtime details private. + +### Models.dev refresh coordination + +- Add one `singleflight.Group` to each models.dev `Client`. +- Route background and explicit refreshes through `Refresh`; move one physical + fetch/write/swap operation into a private helper. +- Keep stale lookups nonblocking. Concurrent explicit callers wait for and share + one result. +- Use a unique same-directory temporary file, remove it on every failure, and + atomically rename it over the cache only after a complete write. +- Preserve current disk and memory data after fetch, cancellation, or write + failure. Update indexes and `loadedAt` only after the file commit and complete + in-memory replacement succeed. + +### Startup cleanup classification and diagnostics + +- Centralize the startup cleanup decision in + `apps/backend/internal/orchestrator/reconcile_liveness.go`. Record typed runtime + absence, row liveness, local-PID presence, stop-error class, and disposition. +- Reuse the classifier for missing, terminal, and failed sessions. +- Keep typed-not-found plus confirmed-dead local rows idempotently removable. + Preserve alive, Unknown, remote, and generic-failure rows. +- Aggregate expected fail-closed outcomes into one structured startup warning + summary with counts. Keep detailed expected rows at Debug level if useful. +- Keep individual warnings for generic stop failures and other unexpected + outcomes. Never log resume tokens or credentials. +- Add adapter-boundary tests that confirm typed sentinel normalization and + repeated-reconciliation tests that distinguish safely removed rows from + intentionally preserved rows. + +### Mention-search cancellation and diagnostics + +- Inject a logger into the mention handler and service composition. +- Map `context.Canceled` from a client request to 499, allowing the existing HTTP + middleware to treat it as routine cancellation. +- Before any unexpected 500, emit exactly one structured Error record with the + workspace ID and stable failure stage/class. +- Record provider failures with safe source/provider/kind/status context while + preserving their HTTP 200 group representation. +- Do not log the search query, prompts, credentials, tokens, raw provider error + bodies, or provider payloads. +- Preserve 400 for invalid requests and 404 for a missing workspace. + +## Frontend design + +### Authoritative runtime state + +- Add the shared wire type and a top-level `agentRuntime` Zustand snapshot. +- Hydrate it from boot state and replace it on + `system.agent_runtime.status_changed` notifications. +- Do not infer runtime availability from WebSocket connectivity or clear task, + workspace, session, message, or route state during an outage. + +### Persistent recovery alert + +- Render `AgentRuntimeUnavailableAlert` as the first in-flow child of the App + status surface, outside the App status bar feature gate. +- Show one localized, non-dismissible `role="alert"` for `unavailable` without + raw process diagnostics. +- Reuse the existing Kandev restart capability and progress flow when a + supervisor supports restart. Otherwise, show manual restart guidance. +- Keep the alert mounted through capability and restart errors; remove it only + after a new available snapshot. + +### Responsive contract + +- Desktop/tablet: span the route-content column without covering sidebar or + bottom status surfaces. +- Phone: stack text and action, keep a 44 px target, preserve route content, and + add no drawer, toast, overlay, or second bottom bar. + +## Test strategy + +- ACP snapshot tests mutate nested maps/slices after enqueue and verify queued + JSON is unchanged; concurrent update/marshal tests run under `-race`. +- Runtime availability tests cover intentional and unexpected exits, sanitized + state, boot hydration, subscription replay, reconnect, and retained domain + data. +- Models.dev tests use channel-controlled servers instead of sleeps. They cover + concurrent homogeneous and mixed lookups, direct refresh calls, failed and + canceled retry, stale-data retention, and temporary-file cleanup. +- Startup cleanup tests cover missing and terminal/failed sessions with typed + not-found plus confirmed-dead local rows, alive and Unknown local rows, remote + rows, generic failures, sentinel normalization, and repeated reconciliation. +- Mention tests cover invalid requests, workspace lookup failure, missing + workspace, provider failure, cancellation, and unexpected search failure. + They assert status codes and safe structured log context. +- Desktop and phone E2E tests inject a backend-shaped unavailable runtime + snapshot. They do not kill the managed E2E `agentctl` process. + +## Implementation waves and task order + +### Wave 1 — independent backend repairs + +- [x] [Task 01: Snapshot mutable ACP event payloads](task-01-snapshot-acp-event-payloads.md) +- [x] [Task 02: Publish agent runtime availability](task-02-publish-agent-runtime-availability.md) +- [x] [Task 05: Coalesce models.dev refreshes](task-05-coalesce-modelsdev-refreshes.md) +- [x] [Task 06: Classify startup runtime cleanup](task-06-classify-startup-runtime-cleanup.md) + +These tasks are parallel candidates because they own separate packages and +contracts. + +### Wave 2 — consumers and shared route composition + +- [x] [Task 03: Surface runtime failure and recovery](task-03-surface-runtime-failure.md) — depends on Task 02. +- [x] [Task 07: Make mention-search failures observable](task-07-observe-mention-search-failures.md) — scheduled after Task 02 because both may edit backend route composition helpers; it has no behavioral dependency on runtime availability. + +### Wave 3 — browser coverage + +- [x] [Task 04: Cover runtime failure across viewports](task-04-runtime-failure-e2e.md) — depends on Tasks 02 and 03. + +Execution remains sequential in the primary conversation unless the user +explicitly authorizes planned implementation sessions. + +## Validation commands + +- `cd apps/backend && go test -race ./internal/agentctl/types/streams ./internal/agentctl/server/adapter/transport/acp` +- `cd apps/backend && go test -race ./internal/agent/runtime/agentctl/... ./internal/backendapp ./internal/gateway/websocket` +- `cd apps/backend && go test -race ./internal/office/costs/modelsdev` +- `cd apps/backend && go test -race -run 'TestReconcileSessionsOnStartup|TestStopReportsRuntimeAbsent|TestRowProcessLiveness' ./internal/orchestrator ./internal/agent/runtime/lifecycle ./internal/backendapp` +- `cd apps/backend && go test -race ./internal/mentions ./internal/backendapp` +- `cd apps/backend && golangci-lint run ./internal/agentctl/... ./internal/agent/runtime/agentctl/... ./internal/orchestrator ./internal/office/costs/modelsdev ./internal/mentions ./internal/backendapp ./internal/gateway/websocket --timeout=5m` +- `cd apps && pnpm --filter @kandev/web test -- --run lib/state lib/ws/handlers/system-events.test.ts components/app-status-bar` +- `cd apps/web && pnpm run typecheck` +- `cd apps/web && pnpm run i18n:check && pnpm run i18n:ratchet` +- `cd apps/web && pnpm e2e:run --host tests/layout/agent-runtime-unavailable.spec.ts` +- `cd apps/web && pnpm e2e:run --project mobile-chrome tests/layout/mobile-agent-runtime-unavailable.spec.ts` + +Run `pnpm install --frozen-lockfile` from `apps/` first if this worktree has no +`node_modules`. + +## Integrated verification results + +All targeted backend race suites passed: ACP (803 tests), agent runtime and +backend composition (562 tests), models.dev (51 tests), startup cleanup (42 +tests), and mentions/backend composition (430 tests). The complete narrow +`golangci-lint` command reported no issues. + +The frontend slice passed with 758 tests across 88 files, `pnpm run typecheck` +passed, and `i18n:check && i18n:ratchet` passed. The repository’s existing 61 +real-locale parity warnings remain advisory and are unrelated to this change. +Desktop and mobile runtime-alert E2E scenarios each passed once; they inject +store state and never stop the managed E2E `agentctl`. + +## Risks and boundaries + +- Snapshotting must preserve supported JSON scalar types, `json.RawMessage`, and + typed payload fields without retaining aliases. +- Runtime event publication and user-subscription replay can race; every event + therefore carries a complete snapshot. +- Singleflight is per client. Unique temporary files also avoid collisions + across multiple client objects or processes sharing one cache path. +- Startup cleanup must never infer death from typed not-found alone. Historical + rows with no PID remain intentionally preserved until a separate safe + migration or operator recovery contract exists. +- The exact cause of the historical mention 500 is unknowable because no error + was logged. The cancellation path is confirmed by code/tests; structured + logging closes the remaining diagnostic gap. +- Automatic child restart, health-endpoint changes, historical runtime-row + deletion, and broad UI action disabling remain outside this package. diff --git a/docs/plans/backend-failure-containment/task-01-snapshot-acp-event-payloads.md b/docs/plans/backend-failure-containment/task-01-snapshot-acp-event-payloads.md new file mode 100644 index 0000000000..96226fb609 --- /dev/null +++ b/docs/plans/backend-failure-containment/task-01-snapshot-acp-event-payloads.md @@ -0,0 +1,91 @@ +--- +id: "01-snapshot-acp-event-payloads" +title: "Snapshot mutable ACP event payloads" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/platform/agent-runtime-availability.md" +--- + +# Task 01: Snapshot mutable ACP event payloads + +## Intent + +Make every queued ACP event own an immutable normalized payload so later tool +updates cannot race serialization or retroactively alter earlier events. + +## Acceptance + +- `NormalizedPayload.Snapshot` detaches every pointer, typed slice, nested map, + nested slice, and `json.RawMessage` that can be observed through JSON. +- Both `sendUpdate` and `sendUpdateLocked` snapshot while `Adapter.mu` is held + and before queue ownership transfers. +- Generic, miscellaneous, subagent, monitor, and typed tool payloads follow one + snapshot rule; the subagent-only clone is removed or delegates to it. +- Mutating an adapter-owned payload after enqueue does not alter queued JSON. +- A concurrent update/marshal regression passes under `go test -race`. + +## TDD sequence + +1. Add a table test that snapshots each mutable payload category, mutates the + original deeply, and confirm the snapshot JSON remains unchanged; observe RED + because no complete snapshot exists. +2. Add an adapter regression that enqueues a generic payload, performs later + nested updates, and marshals the captured event concurrently; confirm RED or + a race report on the current aliasing path. +3. Implement the smallest complete snapshot helper and call it at both event + emission boundaries under the adapter lock. +4. Replace the special-purpose subagent clone, rerun under `-race`, and refactor + duplicated copy logic without changing the wire shape. + +## Files likely touched + +- `apps/backend/internal/agentctl/types/streams/tool_payload.go` +- `apps/backend/internal/agentctl/types/streams/tool_payload_test.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/subagent_test.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go` + +## Dependencies + +None. + +## Parallelism + +`parallel-safe` with Task 02. This task owns normalized stream and ACP adapter +files; Task 02 owns launcher, backend composition, events, and gateway files. + +## Verification + +- `cd apps/backend && go test -race ./internal/agentctl/types/streams ./internal/agentctl/server/adapter/transport/acp` + +## Inputs + +- Confirmed crash stack through `NormalizedPayload.MarshalJSON`. +- `convertToolCallResultUpdate`, `sendUpdate`, and `sendUpdateLocked` ownership + paths. +- Existing subagent snapshot tests as the nearest regression exemplar. + +## Output contract + +Record the failing test/race evidence, the final focused race-test result, and +any normalized payload types deliberately excluded because they are immutable. + +## Results + +- RED: `TestSendUpdateSnapshotsNormalizedPayloadBeforeQueueing` failed because + the queued generic payload still aliased the adapter-owned nested map. +- Added `NormalizedPayload.Snapshot`, which copies all typed payload pointers + and slices, recursively clones JSON-compatible maps/slices and + `json.RawMessage`, and preserves adapter-only provenance markers. +- Both `sendUpdate` and `sendUpdateLocked` snapshot while the adapter lock is + held. `cloneSubagentPayload` now delegates to the common snapshot contract. +- Focused verification: `go test -run + 'TestSendUpdate(Snapshots|LockedSnapshots)|TestQueuedNormalizedPayloadCanMarshalDuringSourceMutation|TestNormalizedPayloadSnapshotDetachesMutablePayloads' + ./internal/agentctl/types/streams ./internal/agentctl/server/adapter/transport/acp` + — passed (16 tests). +- Required verification: `go test -race + ./internal/agentctl/types/streams + ./internal/agentctl/server/adapter/transport/acp` — passed (803 tests). diff --git a/docs/plans/backend-failure-containment/task-02-publish-agent-runtime-availability.md b/docs/plans/backend-failure-containment/task-02-publish-agent-runtime-availability.md new file mode 100644 index 0000000000..1e4b0bf1ca --- /dev/null +++ b/docs/plans/backend-failure-containment/task-02-publish-agent-runtime-availability.md @@ -0,0 +1,106 @@ +--- +id: "02-publish-agent-runtime-availability" +title: "Publish agent runtime availability" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/platform/agent-runtime-availability.md" +--- + +# Task 02: Publish agent runtime availability + +## Intent + +Turn an unexpected standalone `agentctl` exit into an authoritative, +sanitized, replayable runtime snapshot without changing process ownership or +backend health semantics. + +## Acceptance + +- The launcher invokes one unexpected-exit callback for zero or non-zero child + exits that were not caused by `Stop`; intentional shutdown invokes none. +- The availability tracker is concurrency-safe, does not expose startup as + available, becomes available only after health and authentication succeed, + and transitions monotonically to unavailable with reason `agentctl_exited` + within one boot. +- Connected clients receive `system.agent_runtime.status_changed`, and clients + that subscribe later receive the latest full snapshot. +- Every authenticated boot/app-state response includes `agentRuntime`; the + unauthenticated minimal boot payload does not disclose it. +- Process error text, PID, and exit code remain logs-only. +- No `/health`, child restart, token, PID, lifecycle-client, or execution + rebinding behavior changes. + +## TDD sequence + +1. Add launcher tests for unexpected non-zero exit, unexpected clean exit, and + intentional stop; confirm callback cases fail. +2. Add tracker tests for internal startup, post-handshake availability, one-way + unavailable transition, concurrent snapshot reads, stable reason, and event + publication; confirm RED before the tracker exists. +3. Add boot-state and gateway tests for live broadcast and late subscription + replay, including auth-disabled/authenticated versus unauthenticated boot + payloads. +4. Implement the callback, tracker, composition, event/action constants, boot + hydration, and replay wiring. Rerun focused tests and refactor only after + GREEN. + +## Files likely touched + +- `apps/backend/internal/agent/runtime/agentctl/availability.go` +- `apps/backend/internal/agent/runtime/agentctl/availability_test.go` +- `apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go` +- `apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go` +- `apps/backend/internal/backendapp/agentctl.go` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/backend/internal/backendapp/boot_state.go` +- `apps/backend/internal/backendapp/helpers_test.go` +- `apps/backend/internal/events/types.go` +- `apps/backend/pkg/websocket/actions.go` +- `apps/backend/internal/gateway/websocket/system_notifications.go` +- `apps/backend/internal/gateway/websocket/system_notifications_test.go` +- `apps/backend/cmd/kandev/main.go` + +## Dependencies + +None. + +## Parallelism + +`parallel-safe` with Task 01. This task does not touch ACP normalized stream or +adapter files. + +## Verification + +- `cd apps/backend && go test ./internal/agent/runtime/agentctl/... ./internal/backendapp ./internal/gateway/websocket` + +## Inputs + +- Existing launcher `monitorExit` intentional-shutdown classification. +- Existing `system/jobs.Tracker`, `SystemEventBroadcaster`, boot-state builder, + and `Hub.AddUserSubscriptionListener` replay patterns. +- ADR 0019 supervisor-owned restart boundary. + +## Output contract + +Record focused test results and the final public JSON/action shape. Note any +startup-composition file that differs from the likely list because wiring moved. + +## Results + +Implemented the launcher callback, concurrency-safe monotonic tracker, event +and WebSocket action, authenticated boot hydration, and user-subscription +replay. The wire snapshot is `{status, reason?, occurred_at?}`; startup is +unpublished, healthy startup is `{status: "available"}`, and the only failure +transition is `{status: "unavailable", reason: "agentctl_exited"}` with a UTC +occurrence timestamp. Raw process errors, PID, and exit code remain logs-only. + +RED coverage was added for the missing launcher callback, tracker publication +and monotonicity, boot hydration/authenticated boundary, and live/replayed +WebSocket delivery. Focused verification passed: + +```text +go test ./internal/agent/runtime/agentctl/... ./internal/backendapp ./internal/gateway/websocket +562 passed in 4 packages +``` diff --git a/docs/plans/backend-failure-containment/task-03-surface-runtime-failure.md b/docs/plans/backend-failure-containment/task-03-surface-runtime-failure.md new file mode 100644 index 0000000000..482d68c5ce --- /dev/null +++ b/docs/plans/backend-failure-containment/task-03-surface-runtime-failure.md @@ -0,0 +1,114 @@ +--- +id: "03-surface-runtime-failure" +title: "Surface runtime failure and recovery" +status: done +wave: 2 +depends_on: ["02-publish-agent-runtime-availability"] +plan: "plan.md" +spec: "../../specs/platform/agent-runtime-availability.md" +--- + +# Task 03: Surface runtime failure and recovery + +## Intent + +Hydrate the backend-owned runtime snapshot and show a persistent, accessible +shell-level explanation with the supported Kandev restart path. + +## Acceptance + +- Zustand hydrates and replaces the full `agentRuntime` snapshot from boot and + `system.agent_runtime.status_changed` without modifying domain data. +- One non-dismissible `role="alert"` renders for unavailable state on every + authenticated route, including when the App status bar feature is disabled. +- Copy explains the local runtime impact, saved-data safety, and required full + restart without exposing raw diagnostics. +- Supported supervisors show a working **Restart Kandev** action and existing + restart progress flow; unsupported or failed capability lookup shows localized + manual guidance. +- Restart request errors leave the alert mounted and readable. +- A new available snapshot removes the alert; browser connection changes alone + do not. +- All new user-visible copy is localized in every shipped locale. + +## TDD sequence + +1. Add store/hydration and system-event handler tests for unavailable, available, + and unrelated reconnect updates; confirm RED before the state contract exists. +2. Add alert/provider tests with the App status bar disabled, retained route + children, supported capability, unsupported capability, restart error, and + recovery; confirm RED before rendering. +3. Implement the type, store state/action, handler, alert, provider placement, + capability selection, restart flow reuse, and translations. +4. Rerun focused tests, typecheck, and i18n checks before refactoring. + +## Files likely touched + +- `apps/web/lib/types/agent-runtime.ts` +- `apps/web/lib/state/default-state.ts` +- `apps/web/lib/state/store.ts` +- `apps/web/lib/state/store-overrides.ts` +- `apps/web/lib/ws/handlers/system-events.ts` +- `apps/web/lib/ws/handlers/system-events.test.ts` +- `apps/web/components/app-status-bar/agent-runtime-unavailable-alert.tsx` +- `apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx` +- `apps/web/components/app-status-bar/app-status-surface-provider.tsx` +- `apps/web/components/app-status-bar/app-status-surface-provider.test.tsx` +- `apps/web/src/locales/en/system.json` +- `apps/web/src/locales/pt-pt/system.json` +- `apps/web/src/locales/zh-cn/system.json` + +## Dependencies + +Task 02's final boot and WebSocket contract. + +## Parallelism + +`sequential` — this task consumes the backend contract and owns the shared shell +surface used by Task 04. + +## Mobile parity + +The nearest shell exemplar is the in-flow App status surface, while the nearest +error exemplar is the existing SPA/session recovery alert. Phone composition +stacks copy and action, keeps the action at least 44 px, remains inside the +route/status column, and adds no fixed or bottom surface. Unit tests assert +responsive classes and semantics; Task 04 verifies real viewport geometry. + +## Verification + +- `cd apps && pnpm --filter @kandev/web test -- --run lib/state lib/ws/handlers/system-events.test.ts components/app-status-bar` +- `cd apps/web && pnpm run typecheck` +- `cd apps/web && pnpm run i18n:check && pnpm run i18n:ratchet` + +## Inputs + +- Task 02 public snapshot and action. +- Existing `useRestartCapability`, `useKandevRestart`, and + `RestartProgressDialog` behavior. +- App status surface shell geometry and mobile UI language reference. + +## Output contract + +Record focused unit, typecheck, and i18n results plus the final localized copy +keys and any deliberate change to alert placement. + +## Results + +Implemented the backend-shaped `agentRuntime` snapshot in the web store, +hydrated it from boot state, and replaced it on the global runtime WebSocket +event without clearing domain state. Added the in-flow localized alert outside +the App status bar feature gate, reusing the existing capability and restart +progress hooks. Supported supervisors expose `Restart Kandev`; capability +lookup failures render manual guidance, and restart errors leave the alert +mounted until an available snapshot arrives. + +Focused validation passed: + +- `pnpm --filter @kandev/web test -- --run lib/state/store.test.ts lib/ws/handlers/system-events.test.ts components/app-status-bar/agent-runtime-unavailable-alert.test.tsx components/app-status-bar/app-status-surface-provider.test.tsx` — 21 tests passed. +- `pnpm run typecheck` — passed. +- `pnpm run i18n:check && pnpm run i18n:ratchet` — passed; pseudo catalog regenerated and synced. + +The alert uses localized English, Portuguese, Chinese, and pseudo-locale +entries, remains in normal shell flow on phone layouts, and keeps its restart +action at the required 44 px minimum. diff --git a/docs/plans/backend-failure-containment/task-04-runtime-failure-e2e.md b/docs/plans/backend-failure-containment/task-04-runtime-failure-e2e.md new file mode 100644 index 0000000000..9961d358de --- /dev/null +++ b/docs/plans/backend-failure-containment/task-04-runtime-failure-e2e.md @@ -0,0 +1,99 @@ +--- +id: "04-runtime-failure-e2e" +title: "Cover runtime failure across viewports" +status: done +wave: 3 +depends_on: ["02-publish-agent-runtime-availability", "03-surface-runtime-failure"] +plan: "plan.md" +spec: "../../specs/platform/agent-runtime-availability.md" +--- + +# Task 04: Cover runtime failure across viewports + +## Intent + +Prove that desktop and phone users see the outage without losing last-known +route data or breaking viewport geometry, while keeping the managed E2E runtime +alive for the rest of the suite. + +## Acceptance + +- Desktop shows exactly one runtime alert over a populated task route when the + backend-shaped unavailable snapshot is injected. +- Last-known workspace/task content remains visible and the optional App status + bar setting does not hide the alert. +- The supported restart control calls the existing capability/restart endpoints + and enters the restart progress flow. +- Phone shows readable stacked content, a 44 px action, no horizontal overflow, + and no overlap with route bottom navigation or safe-area content. +- The tests do not terminate the shared E2E `agentctl` or rely on timing a real + process crash. + +## TDD sequence + +1. Add desktop and `mobile-` Playwright scenarios that inject unavailable state + through `window.__KANDEV_E2E_STORE__`; observe RED until Task 03 is present. +2. Route restart capability/request responses for the desktop recovery action + and assert the progress phase without allowing the backend to exit. +3. Add phone bounding-box, hit-target, retained-content, and horizontal-overflow + assertions using existing mobile resilience helpers. +4. Run each focused project independently and remove only test flakiness or + locator ambiguity revealed by the real browser. + +## Files likely touched + +- `apps/web/e2e/tests/layout/agent-runtime-unavailable.spec.ts` +- `apps/web/e2e/tests/layout/mobile-agent-runtime-unavailable.spec.ts` +- Existing E2E helper/page-object files only if a shared store injection or + overflow assertion already has a canonical home. + +## Dependencies + +Tasks 02 and 03. + +## Parallelism + +`sequential` — browser assertions depend on the final backend-shaped contract +and alert composition. + +## Mobile parity + +The mobile test is mandatory and is named with the `mobile-` prefix so the +`mobile-chrome` project discovers it. It checks touch sizing, stacked layout, +route content retention, bottom-navigation separation, and document overflow; +desktop success alone is not sufficient. + +## Verification + +- `cd apps/web && pnpm e2e:run --host tests/layout/agent-runtime-unavailable.spec.ts` +- `cd apps/web && pnpm e2e:run --project mobile-chrome tests/layout/mobile-agent-runtime-unavailable.spec.ts` + +## Inputs + +- Task 03 alert and recovery surface. +- `ws-connectivity-warning.spec.ts` store-injection pattern. +- `mobile-spa-resilience.spec.ts` alert, hit-target, and overflow assertions. + +## Output contract + +Record both focused Playwright results and any artifacts. Confirm explicitly +that no real `agentctl` process was stopped by the scenarios. + +## Results + +Added a shared E2E store-injection/restart-stub helper plus independent desktop +and `mobile-` Playwright scenarios. The desktop scenario retains a populated +board, verifies the alert remains visible with the App status bar disabled, +asserts the supported restart request and `restarting` progress phase, and +clears only after an available snapshot. The phone scenario opens a real +seeded task session, verifies the stacked alert stays above the real bottom +navigation, checks the 44 px action target and document overflow, and verifies +recovery without losing the task route. + +Focused validation passed: + +- `pnpm e2e --project=chromium e2e/tests/layout/agent-runtime-unavailable.spec.ts` — 1 passed. +- `pnpm e2e --project=mobile-chrome e2e/tests/layout/mobile-agent-runtime-unavailable.spec.ts` — 1 passed. + +Both scenarios inject store state and stub only the restart HTTP responses; no +managed E2E `agentctl` process was stopped or crashed. diff --git a/docs/plans/backend-failure-containment/task-05-coalesce-modelsdev-refreshes.md b/docs/plans/backend-failure-containment/task-05-coalesce-modelsdev-refreshes.md new file mode 100644 index 0000000000..13c0778a31 --- /dev/null +++ b/docs/plans/backend-failure-containment/task-05-coalesce-modelsdev-refreshes.md @@ -0,0 +1,95 @@ +--- +id: "05-coalesce-modelsdev-refreshes" +title: "Coalesce models.dev refreshes" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/office/costs.md" +--- + +# Task 05: Coalesce models.dev refreshes + +## Intent + +Preserve stale-while-revalidate pricing while ensuring that one client performs +only one physical refresh and never collides on a shared temporary cache file. + +## Acceptance + +- Concurrent stale `LookupForModel` calls start one network request. +- Mixed concurrent `LookupForModel` and `LookupModelInfo` calls start one network + request. +- Concurrent direct `Refresh` calls wait for and receive one shared result. +- Background and direct refresh paths use the same per-client guard. +- Failed and canceled refreshes release the guard so a later refresh can retry. +- Existing valid disk and memory cache data remains readable after failure. +- Atomic replacement uses a unique same-directory temporary file and leaves no + temporary file after success, failure, or cancellation. +- `loadedAt` and the active in-memory indexes change only after the new cache + file is committed and the complete in-memory replacement succeeds. + +## TDD sequence + +1. Add channel-controlled HTTP server tests for concurrent homogeneous lookups, + mixed lookup methods, and direct `Refresh` calls. Assert one request and + observe RED on the current unguarded paths. +2. Add failed-then-successful and canceled-then-successful tests. Assert stale + data remains available and the second request runs. +3. Add filesystem assertions that no cache temporary files remain and that the + prior valid cache survives a failed write/fetch. +4. Add a per-client `singleflight.Group`, route every refresh through it, and + extract one physical refresh helper. +5. Replace the fixed `.tmp` path with a unique same-directory temporary file, + deferred cleanup, and atomic rename. Update freshness only after commit and + memory swap. +6. Run the package with the race detector, then refactor only after GREEN. + +## Files likely touched + +- `apps/backend/internal/office/costs/modelsdev/client.go` +- `apps/backend/internal/office/costs/modelsdev/client_test.go` + +## Dependencies + +None. `golang.org/x/sync/singleflight` is already a repository dependency. + +## Parallelism + +`parallel-safe` with Tasks 01, 02, and 06. This task owns only the models.dev +cost client package. + +## Verification + +- `cd apps/backend && go test -race ./internal/office/costs/modelsdev` +- `cd apps/backend && golangci-lint run ./internal/office/costs/modelsdev --timeout=5m` + +## Inputs + +- Observed cache rename `ENOENT` at backend log lines 37734 and 22060. +- Current `warmFromDisk`, `maybeRefresh`, `refreshSafe`, `Refresh`, and + `writeCacheAtomic` paths. +- Existing repository uses of `singleflight.Group`. + +## Output contract + +Record the initial duplicate request or race evidence, final request counts, +retry/cancellation results, temporary-file assertions, and focused race/lint +results. + +## Results + +Added channel-controlled RED coverage that reproduced duplicate concurrent +fetches (8 direct requests and duplicate background lookup refreshes), then +coalesced every refresh path through one per-client `singleflight.Group`. +Refresh failures and cancellation now release the guard for a later retry; +valid disk and in-memory data remain intact. Cache replacement writes a unique +same-directory temporary file, syncs and closes it before rename, and defers +cleanup on every path. Indexes and `loadedAt` are swapped only after commit. + +Focused race verification passed: + +```text +go test -race ./internal/office/costs/modelsdev +51 passed in 1 package +``` diff --git a/docs/plans/backend-failure-containment/task-06-classify-startup-runtime-cleanup.md b/docs/plans/backend-failure-containment/task-06-classify-startup-runtime-cleanup.md new file mode 100644 index 0000000000..a7ca1a3cf5 --- /dev/null +++ b/docs/plans/backend-failure-containment/task-06-classify-startup-runtime-cleanup.md @@ -0,0 +1,114 @@ +--- +id: "06-classify-startup-runtime-cleanup" +title: "Classify startup runtime cleanup" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/tasks/runtime-cleanup.md" +--- + +# Task 06: Classify startup runtime cleanup + +## Intent + +Make startup reconciliation decisions and diagnostics precise: safely remove +already-absent confirmed-dead local runtimes, preserve uncertain rows, and stop +expected legacy preservation from producing an ambiguous warning flood. + +## Acceptance + +- Missing-session and terminal/failed-session paths use one explicit cleanup + decision classifier. +- Typed runtime-not-found plus confirmed-dead local liveness removes or repairs + the row according to the resume-safety invariant. +- Alive local, Unknown local, and remote rows remain preserved. +- Generic stop errors remain preserved and produce individual structured + warnings. +- Runtime/lifecycle/executor adapters preserve typed not-found identity through + the real composition boundary. +- Local PID liveness after backend restart is tested for alive, dead, and + missing-handle cases. +- Expected fail-closed outcomes are aggregated into a bounded startup warning + summary with safe classifications and counts. +- A second reconciliation does not encounter a safely removable row removed by + the first pass; intentionally uncertain rows remain preserved and summarized. +- Diagnostics never contain resume tokens, credentials, or provider payloads. + +## TDD sequence + +1. Add focused tests for missing sessions and terminal/failed sessions with a + typed not-found result and confirmed-dead local PID. Assert cleanup and no + repeated warning on a second pass. +2. Add alive local, missing-PID/Unknown local, remote, and generic-failure cases. + Assert preservation and the correct diagnostic disposition. +3. Add adapter-boundary tests proving lifecycle/executor not-found sentinels + normalize to the runtime API sentinel, and liveness tests for post-restart PID + classification. +4. Add observer-backed tests for one aggregate expected-outcome summary and + individual unexpected-error warnings. +5. Extract the smallest shared decision classifier and summary collector. Reuse + them in missing and terminal/failed startup paths without changing the + deletion safety predicate. +6. Run focused packages under `-race`, then refactor only after GREEN. + +## Files likely touched + +- `apps/backend/internal/orchestrator/service.go` +- `apps/backend/internal/orchestrator/reconcile_liveness.go` +- `apps/backend/internal/orchestrator/reconcile_restart_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/liveness.go` +- `apps/backend/internal/agent/runtime/lifecycle/liveness_test.go` +- `apps/backend/internal/backendapp/adapters.go` +- `apps/backend/internal/backendapp/adapters_test.go` + +## Dependencies + +None. The safe confirmed-dead cleanup behavior already exists; this task adds +classification consistency, integration evidence, and bounded diagnostics. + +## Parallelism + +`parallel-safe` with Tasks 01, 02, and 05. It owns orchestrator reconciliation +and liveness tests, not launcher/runtime availability files. + +## Verification + +- `cd apps/backend && go test -race -run 'TestReconcileSessionsOnStartup|TestStopReportsRuntimeAbsent|TestRowProcessLiveness' ./internal/orchestrator ./internal/agent/runtime/lifecycle ./internal/backendapp` +- `cd apps/backend && golangci-lint run ./internal/orchestrator ./internal/agent/runtime/lifecycle ./internal/backendapp --timeout=5m` + +## Inputs + +- Startup warning clusters for missing and terminal/failed sessions. +- Pre-start database evidence that warned standalone rows had `local_pid=0`. +- Existing `stopReportsRuntimeAbsent`, `rowLiveness`, + `pruneOrRepairExecutorRow`, and runtime error-normalization paths. +- Existing startup reconciliation tests from the confirmed-dead cleanup repair. + +## Output contract + +Record the exact decision matrix, sentinel-normalization evidence, first/second +reconciliation outcomes, structured log fields and counts, and focused +race/lint results. + +## Results + +Centralized startup stop-result classification in `reconcile_liveness.go`: +typed runtime absence is cleanup-safe only with confirmed-dead local liveness; +alive, Unknown, remote, missing-handle, and generic-failure outcomes preserve +the durable row. Rows without an execution stop handle proceed only when a +local process is independently confirmed dead. Expected fail-closed +preservation is emitted as one bounded structured startup summary with +liveness, stop-error class, disposition, and local-PID-presence counts. Generic +stop failures retain individual structured warnings without raw errors, tokens, +credentials, or provider payloads. + +The existing lifecycle and backend adapter boundary tests confirm real +sentinel normalization, and the liveness tests cover alive, reaped/dead, SSH, +docker, empty-runtime, nil, and missing-local-handle rows. Focused race +verification passed: + +```text +go test -race -run 'TestReconcileSessionsOnStartup|TestStopReportsRuntimeAbsent|TestRowProcessLiveness|TestStopRuntimeForStartupCleanupPreservesUnknownRowsWithoutHandle' ./internal/orchestrator ./internal/agent/runtime/lifecycle ./internal/backendapp +42 passed +``` diff --git a/docs/plans/backend-failure-containment/task-07-observe-mention-search-failures.md b/docs/plans/backend-failure-containment/task-07-observe-mention-search-failures.md new file mode 100644 index 0000000000..2d5c1605c6 --- /dev/null +++ b/docs/plans/backend-failure-containment/task-07-observe-mention-search-failures.md @@ -0,0 +1,105 @@ +--- +id: "07-observe-mention-search-failures" +title: "Make mention-search failures observable" +status: done +wave: 2 +depends_on: ["02-publish-agent-runtime-availability"] +plan: "plan.md" +spec: "../../specs/ui/entity-reference-composer.md" +--- + +# Task 07: Make mention-search failures observable + +## Intent + +Keep partial provider search resilient, treat superseded client requests as +normal cancellation, and ensure every unexpected mention-search 500 has one +safe structured backend error record. + +## Acceptance + +- Invalid requests remain 400 and missing workspaces remain 404. +- `context.Canceled` from a client request returns 499 and stops provider work. +- Workspace repository failures and unexpected aggregate search failures return + 500 only after exactly one structured Error record. +- Error records include workspace ID and a stable failure stage/class where + available. +- Provider failures remain safe per-provider groups in an HTTP 200 response and + can emit source/provider/kind/status diagnostics. +- Logs do not contain the query, prompts, credentials, tokens, raw provider + payloads, or raw provider error bodies. +- Request cancellation is not recorded as an unexpected 500. + +## TDD sequence + +1. Update/add handler and backend composition tests for invalid input, missing + workspace, generic workspace lookup failure, cancellation, provider failure, + and an unexpected searcher error. +2. Attach a `zaptest/observer` and assert status, level, message, safe fields, + exactly one record for each 500, and absence of query/secret fixtures. +3. Confirm RED for cancellation (currently 500) and unlogged aggregate failures. +4. Inject the logger, classify `context.Canceled` as 499, and log unexpected + aggregate failures before writing 500. +5. Add safe provider status diagnostics without changing the service's partial + success response contract. +6. Run both mention and backend composition packages under `-race`, then refactor + only after GREEN. + +## Files likely touched + +- `apps/backend/internal/mentions/handler.go` +- `apps/backend/internal/mentions/handler_test.go` +- `apps/backend/internal/mentions/service.go` +- `apps/backend/internal/mentions/service_test.go` +- `apps/backend/internal/backendapp/mentions.go` +- `apps/backend/internal/backendapp/mentions_test.go` +- `apps/backend/internal/backendapp/helpers.go` + +## Dependencies + +Task 02 only for implementation sequencing: both tasks can edit backend route +composition helpers. There is no behavioral dependency on runtime availability. + +## Parallelism + +`sequential-after-task-02` to avoid overlapping edits in +`internal/backendapp/helpers.go`. It is otherwise independent from Tasks 03, +05, and 06. + +## Verification + +- `cd apps/backend && go test -race ./internal/mentions ./internal/backendapp` +- `cd apps/backend && golangci-lint run ./internal/mentions ./internal/backendapp --timeout=5m` + +## Inputs + +- Unexplained historical 500 for + `GET /api/v1/workspaces/:id/mentions/search`. +- Current `writeSearchError` generic fallback and workspace-validation wrapper. +- Existing frontend debounce/`AbortController` behavior and backend cancellation + composition test that currently expects 500. +- Existing middleware treatment of HTTP 499 as routine. + +## Output contract + +Record the error-path matrix, observed RED statuses/log absence, final safe log +schema, sensitive-field assertions, and focused race/lint results. State that +the historical request's exact cause remains unknown unless new evidence proves +it. + +## Results + +Implemented stable search failure classification and safe structured logging. +Invalid input remains 400, missing workspaces remain 404, client cancellation +maps to 499 without an unexpected-error record, and aggregate failures emit one +Error record with workspace ID plus failure stage/class before returning 500. +Provider failures retain their HTTP 200 grouped response and emit only safe +source/provider/kind/status diagnostics; query text, raw provider bodies, and +other sensitive values are excluded. + +Focused validation passed: + +- `gofmt` on the changed Go files. +- `go test -count=1 -run 'TestHandlerSearch|TestServiceSearchLogsProviderFailure|TestMentionHTTPComposition' ./internal/mentions ./internal/backendapp` — 12 tests passed. +- `go test -race ./internal/mentions ./internal/backendapp` — 430 tests passed. +- `golangci-lint run ./internal/mentions ./internal/backendapp --timeout=5m` — no issues. diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index 9787a8a928..2a4a049859 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -38,6 +38,7 @@ Product-wide capabilities that are not tied to a single feature area. | Spec | Status | |---|---| +| [agent-runtime-availability](platform/agent-runtime-availability.md) | draft | | [background-work-liveness](platform/background-work-liveness.md) | shipped | | [task-sleep-inhibition](platform/task-sleep-inhibition.md) | building | | [i18n](platform/i18n.md) | building | diff --git a/docs/specs/office/costs.md b/docs/specs/office/costs.md index 5f7578dc75..19f26d32b5 100644 --- a/docs/specs/office/costs.md +++ b/docs/specs/office/costs.md @@ -1,6 +1,7 @@ --- status: in-progress created: 2026-04-25 +updated: 2026-08-08 owner: cfl --- @@ -20,6 +21,9 @@ Office adds per-session cost estimation, aggregated views by agent/project/model - Cost events are aggregated on read into multiple views (by agent, project, task, model, time window). - Budget policies SHALL enforce spending limits per agent, project, or workspace with `notify_only`, `pause_agent`, or `block_new_tasks` actions. - Cost is resolved per cost event via a two-layer lookup (provider-reported then `models.dev` cache); no static fallback table exists. +- Stale `models.dev` pricing remains readable while one coalesced refresh runs per + client. Concurrent background lookups and explicit refresh calls do not start + duplicate network fetches. - For providers that emit token telemetry on the ACP wire (claude-acp, opencode-acp, gemini, codex-acp), cost events are populated from the `complete` event at end-of-turn. - For providers without usable wire telemetry (codex per-turn split, amp), a disk-runner subsystem reads on-disk session files via pinned `@ccusage/*` packages and feeds normalized cost events into the same pipeline. - The cost explorer UI surfaces aggregations and lets the user manage budget policies. @@ -74,6 +78,15 @@ Each agent instance has `budget_monthly_cents` set at creation (or inherited fro The `models.dev` dataset is fetched once per day to a workspace-local disk cache at `/cache/models-dev.json`. The file lives on disk full-fat; only queried models load into the in-memory map. Refresh runs in a background goroutine; refresh failures fall back to the existing on-disk file. Pricing is recorded per-million-tokens for input, cached read, cached write, and output separately (Anthropic charges different rates for cached read vs. cached write). +Each client permits only one physical refresh at a time. Stale lookups return +current data without waiting, while concurrent explicit `Refresh` callers wait +for and share the same result. A failed or canceled refresh releases the guard +so a later request can retry. Cache replacement uses a unique temporary file in +the cache directory and an atomic rename. Failure removes that temporary file +and leaves the last valid disk and memory data unchanged. The client marks the +cache fresh only after the new file is committed and the in-memory indexes are +replaced successfully. + ### Per-CLI usage shapes (ACP wire) - **claude-acp**: `result.usage` (camelCase, `cachedRead`/`cachedWriteTokens`), plus cumulative `usage_update.cost.amount` in USD; the adapter emits its nonnegative turn delta when present. claude-acp uses logical aliases (`sonnet`, `haiku`); provider-reported cost is the only accurate signal. @@ -157,7 +170,12 @@ There is no per-field permission model. Conformance tests should assert that cos ## Failure modes - **models.dev miss**: row recorded with `cost_subcents=0` and `estimated=true`; UI shows "pricing unavailable". -- **models.dev fetch fails**: background refresh falls back to existing on-disk file; no crash. +- **models.dev fetch fails or is canceled**: background refresh falls back to + existing on-disk and in-memory data; no crash. The in-flight guard is released + and a later refresh can retry. +- **models.dev refresh is requested concurrently**: one network request and one + atomic cache replacement run per client. All explicit callers observe the + shared result, and temporary files do not collide or remain after failure. - **Disk-runner `npx` unavailable**: log one warning per provider per process lifetime, mark run skipped, backend continues. Codex sessions retain wire-side `estimated=true` rows; amp sessions remain untracked. - **ccusage JSON schema drift**: schema validator returns decode error; office subscriber treats run as no-op (no rows touched). Codex falls back to wire-side estimated path; amp absent. Nightly fixture-smoke CI alerts maintainers. - **`@ccusage/@latest` yanked**: next runner invocation fails; coverage degrades the same as parse failure. @@ -173,6 +191,8 @@ There is no per-field permission model. Conformance tests should assert that cos - Per-agent `budget_monthly_cents` stored on the agent instance row. - Per-model pricing overrides stored in workspace settings. - The on-disk `models.dev` cache at `/cache/models-dev.json`. Recovery on next boot: the in-memory pricing map is empty until first query; queries fall back to the on-disk file when the background refresh has not yet completed. +- A failed refresh never replaces or invalidates the last valid cache. Refresh + coordination and temporary files are transient and do not survive restart. - Activity log entries `budget.alert` and `budget.exceeded` (workspace-scoped, included in the standard office backup as part of normal SQLite persistence - see `persistence.Provide` for the snapshot policy). **Does NOT survive restart:** @@ -191,6 +211,14 @@ No TTL or retention is applied to `office_cost_events`; rows accumulate for the - **GIVEN** a model not found in the models.dev dataset and no user override configured, **WHEN** a cost event is recorded, **THEN** token counts are stored but `cost_subcents` is zero. The cost explorer shows "pricing unavailable" for that model. +- **GIVEN** many stale pricing lookups and direct refresh calls arrive together, + **WHEN** refresh starts, **THEN** one network fetch runs for that client while + stale lookup data remains available and direct callers share its result. + +- **GIVEN** a refresh fails or its context is canceled, **WHEN** a later lookup + requests refresh, **THEN** the later refresh can run, the previous valid cache + remains readable, and no temporary cache file is left behind. + - **GIVEN** an agent instance with a monthly budget of $10 and 80% alert threshold, **WHEN** spending reaches $8, **THEN** an alert appears in the user's inbox and a `budget_alert` wakeup is queued for the CEO. - **GIVEN** an agent instance with a monthly budget of $10 and `action_on_exceed=pause_agent`, **WHEN** spending exceeds $10, **THEN** the agent is paused, pending wakeups are cancelled, and an activity log entry records the auto-pause. @@ -231,3 +259,7 @@ No TTL or retention is applied to `office_cost_events`; rows accumulate for the - **Node/npx in the backend container**: the backend image currently does not include Node. Options: `apt-get install nodejs npm` (~50MB), or bundle ccusage as a single-file `bun build --compile` per provider, eliminating runtime Node dependency. - **Future providers**: when a new agent CLI lands (hypothetical `@ccusage/auggie`, new `@ccusage/copilot`), the runner should accept provider plugins via a registry, not require new code per provider. - **Tokscale as alternative wrapper** (considered, deferred): single Rust binary, 20+ CLIs, but lacks per-session output today. Decision for v1: stay with `@ccusage/*` because per-session output ships. Revisit when (a) we need a provider tokscale supports natively that ccusage doesn't, or (b) the session-grouping contribution lands upstream. + +## Implementation plan + +[Backend failure containment](../../plans/backend-failure-containment/plan.md) diff --git a/docs/specs/platform/agent-runtime-availability.md b/docs/specs/platform/agent-runtime-availability.md new file mode 100644 index 0000000000..aed26bac92 --- /dev/null +++ b/docs/specs/platform/agent-runtime-availability.md @@ -0,0 +1,160 @@ +--- +status: draft +created: 2026-08-08 +owner: kandev +--- + +# Agent Runtime Availability + +## Why + +Kandev's standalone `agentctl` process is a core runtime dependency, but the +application backend can remain reachable after that child exits. In that state +the browser WebSocket still looks connected while agent actions, task updates, +and workspace tools stop progressing. Users need an authoritative explanation +and a supported recovery path without losing the last known task or workspace +data. + +## What + +- Kandev exposes one install-wide local agent-runtime availability snapshot: + `available` or `unavailable`. +- The runtime becomes `available` only after the standalone `agentctl` process + passes its health check and authentication handshake. +- An unexpected `agentctl` exit atomically changes the snapshot to + `unavailable`, records the stable reason `agentctl_exited` and occurrence + time, and publishes that snapshot to connected browsers. +- Intentional shutdown does not publish an outage. +- A newly loaded or reconnected browser receives the latest snapshot even when + it missed the original exit event. +- While unavailable, every authenticated application route shows one + persistent, non-dismissible alert explaining that the local agent runtime + stopped, agent actions and workspace tools may not update, saved data remains + intact, and Kandev must restart to recover. +- The alert is independent of the optional App status bar and remains visible + when route data is empty or stale. It is not a transient toast. +- When the local restart capability is supported, the alert offers **Restart + Kandev** and reuses the existing supervised restart progress flow. Otherwise + it gives localized terminal or service-manager guidance. A failed restart + request leaves the alert visible and exposes the failure. +- The browser retains last-known task, session, workspace, and message state + while the runtime is unavailable. The availability event never clears those + stores. +- The alert clears only when a fresh backend boot reports a healthy authenticated + `agentctl`; browser WebSocket reconnection alone does not clear it. + +## API surface + +The boot payload and `/api/v1/app-state` initial state include: + +```json +{ + "agentRuntime": { + "status": "unavailable", + "reason": "agentctl_exited", + "occurred_at": "2026-08-08T14:22:52Z" + } +} +``` + +`reason` and `occurred_at` are omitted while available. Unexpected exits publish +the same shape as the install-wide WebSocket action +`system.agent_runtime.status_changed`. The reason is a stable enum for UI +selection; raw process errors, stack traces, PIDs, and exit details remain in +diagnostic logs. + +The existing `/api/v1/system/restart-capability`, `/api/v1/system/restart`, and +`boot_id` contracts remain the recovery API. This feature does not change the +general backend health endpoint. + +## State machine + +| Current state | Event | Next state | User result | +| --- | --- | --- | --- | +| startup | Health and auth handshake succeed | `available` | No alert | +| `available` | Child exits unexpectedly | `unavailable` | Persistent alert and recovery guidance | +| `unavailable` | Browser reconnects or reloads | `unavailable` | Snapshot is replayed; alert remains | +| `unavailable` | Supervisor starts a fresh healthy backend and child | `available` | New boot removes the alert | +| any | Intentional backend shutdown stops child | no outage transition | No stale failure is emitted | + +The in-memory snapshot is monotonic within one backend boot: an unexpected exit +cannot return to `available` because this repair does not restart `agentctl` in +process. + +## Failure modes + +- If the runtime-status event cannot be delivered, boot hydration and replay on + user subscription provide the current snapshot on reload or reconnect. +- If restart capability lookup fails, the alert stays visible and presents + manual restart guidance instead of assuming the launch mode. +- If a supervised restart request fails, the existing backend remains in the + unavailable state and the UI exposes the request error without dismissing the + alert. +- If the browser WebSocket is also offline, the existing connectivity warning + remains independent. Both conditions may be visible because they describe + different failures. +- If `agentctl` fails before startup completes, normal backend startup failure + behavior applies; no browser is served an inaccurate available snapshot. + +## Persistence guarantees + +Availability is process-local operational state, not database history. It is +initialized from the current launch, retained for the lifetime of that backend, +and replayed to clients. Kandev does not persist crash history or restore an +unavailable marker after a successful full restart. Existing task, workspace, +message, and session persistence is unchanged. + +## Responsive and mobile contract + +- Desktop and tablet render the alert in flow at the top of the route-content + column, above route data and below no route-owned chrome. +- Phone uses the same in-flow alert rather than a toast, drawer, or second + persistent bottom bar. Copy and actions stack vertically, stay within the + viewport, and do not introduce document horizontal overflow. +- The restart action is at least 44 px on phone. The alert remains above the + route's single scroll owner and does not cover bottom navigation or safe-area + content. +- The alert uses `role="alert"`, a textual heading and explanation, and does not + rely on color alone. + +## Scenarios + +- **GIVEN** a healthy running Kandev, **WHEN** `agentctl` exits unexpectedly, + **THEN** every connected browser receives `unavailable` and shows one + persistent alert while preserving last-known application data. +- **GIVEN** the runtime became unavailable before a browser connected, **WHEN** + the browser loads or subscribes after WebSocket reconnect, **THEN** it receives + the current unavailable snapshot and shows the same alert. +- **GIVEN** the App status bar feature is disabled, **WHEN** the runtime becomes + unavailable, **THEN** the alert is still visible on every authenticated route. +- **GIVEN** supervised restart is supported, **WHEN** the user chooses **Restart + Kandev**, **THEN** the existing restart flow waits for a changed `boot_id` and + the fresh healthy boot clears the alert. +- **GIVEN** restart is unsupported, **WHEN** the alert renders, **THEN** it gives + localized terminal or service-manager guidance and does not offer a broken + restart action. +- **GIVEN** Kandev is intentionally shutting down, **WHEN** the launcher stops + `agentctl`, **THEN** no unavailable event is emitted. +- **GIVEN** a phone viewport with sparse or stale route data, **WHEN** the runtime + is unavailable, **THEN** the alert and any restart action fit without overlap, + horizontal scrolling, or an inaccessible touch target. + +## Out of scope + +- Restarting `agentctl` inside the existing backend process or rebinding its + launch token, PID, lifecycle manager, control clients, and active executions. +- Automatically restarting all of Kandev without an operator action. +- Changing `/health` readiness semantics, container/service restart policies, + or browser WebSocket retry behavior. +- Persisted crash history, crash counters, raw diagnostics in the UI, or a new + operational dashboard. +- Declaring remote executors unavailable or disabling every application action; + the alert describes the local standalone runtime dependency. + +## Decision record + +[Publish immutable agent events and explicit runtime failure state](../../decisions/2026-08-08-agentctl-crash-containment.md) + +## Implementation plan + +[Backend failure containment](../../plans/backend-failure-containment/plan.md) diff --git a/docs/specs/tasks/runtime-cleanup.md b/docs/specs/tasks/runtime-cleanup.md index 106c3e4b00..5b390005c7 100644 --- a/docs/specs/tasks/runtime-cleanup.md +++ b/docs/specs/tasks/runtime-cleanup.md @@ -1,6 +1,7 @@ --- status: draft created: 2026-06-22 +updated: 2026-08-08 owner: cfl --- @@ -49,6 +50,13 @@ worktrees, or executor rows behind and the machine slowly runs out of memory. window before reporting shutdown complete. - Backend startup reconciles stale runtime rows for archived/deleted/missing task state and attempts safe cleanup instead of treating the rows as live sessions. +- Startup reconciliation classifies each stop result and row liveness before it + chooses cleanup or preservation. Already-absent, confirmed-dead local runtimes + are removed idempotently without repeated warnings. Alive, unknown, remote, or + generically failed rows remain preserved. +- Expected fail-closed preservation is reported as a bounded structured startup + summary with counts and safe classification fields. Unexpected generic stop + failures retain individual warning records. - Cleanup is idempotent: repeating archive/delete cleanup, startup reconciliation, or explicit session stop does not fail because the process, task, session, or worktree was already removed. @@ -231,6 +239,13 @@ The durable cleanup job wraps that resource lifecycle: - If startup reconciliation finds rows for archived tasks, deleted tasks, missing sessions, or terminal sessions with no live runtime, it removes only rows that are positively confirmed safe to remove. +- If startup reconciliation receives a typed runtime-not-found result for a local + row with no persisted process liveness handle, it classifies liveness as + Unknown and preserves the row. It summarizes that expected fail-closed outcome + instead of emitting one ambiguous warning per legacy row. +- Startup diagnostics identify the runtime, session state, liveness class, + presence of a local PID, stop-error class, disposition, and aggregate count. + They never include resume tokens, credentials, or provider payloads. - If cleanup intent or its resource snapshot cannot be persisted, the lifecycle mutation fails before destructive cleanup begins; Kandev does not rely on an unrecorded background goroutine. @@ -246,6 +261,9 @@ The durable cleanup job wraps that resource lifecycle: attempted for every runtime row owned by the task. - Startup reconciliation is allowed to recover from a previous backend crash by reattempting cleanup for stale runtime rows. +- A typed-not-found, confirmed-dead local row removed during one startup does not + reappear or produce the same cleanup warning on the next startup. Rows with + Unknown liveness remain durable until Kandev can prove safe removal. - Pending and retryable task cleanup jobs survive restart and resume independently of whether optional scheduled storage maintenance is enabled. - Terminal failed task cleanup jobs survive restart for diagnosis but do not @@ -275,6 +293,18 @@ The durable cleanup job wraps that resource lifecycle: - **GIVEN** the backend restarts with an `executors_running` row for an archived task, **WHEN** startup reconciliation runs, **THEN** it attempts cleanup for the row instead of treating the archived task as active. +- **GIVEN** a missing, terminal, or failed session whose local runtime is already + absent and whose persisted PID is confirmed dead, **WHEN** startup + reconciliation receives a typed not-found result, **THEN** it safely removes or + repairs the row without an individual warning, and a second reconciliation has + no stale row to process. +- **GIVEN** a typed not-found result for an alive local row, a local row without a + liveness handle, or a remote row, **WHEN** startup reconciliation runs, **THEN** + it preserves the row and includes the safe disposition in a bounded startup + summary. +- **GIVEN** a stop fails with a generic error, **WHEN** startup reconciliation + runs, **THEN** it preserves the row and emits an individual structured warning + rather than reclassifying the runtime as absent. - **GIVEN** an `executors_running` row whose task session is missing and whose local liveness handle refers to a dead host process, **WHEN** startup reconciliation stops it and the stop reports the execution is not found, **THEN** @@ -353,3 +383,7 @@ The durable cleanup job wraps that resource lifecycle: - New user-facing archive/delete controls. - Changing the task/session state model beyond the cleanup guarantees described here. + +## Implementation plan + +[Backend failure containment](../../plans/backend-failure-containment/plan.md) diff --git a/docs/specs/ui/entity-reference-composer.md b/docs/specs/ui/entity-reference-composer.md index d6d0979ad8..999e7ddb6a 100644 --- a/docs/specs/ui/entity-reference-composer.md +++ b/docs/specs/ui/entity-reference-composer.md @@ -1,6 +1,7 @@ --- status: shipped created: 2026-07-21 +updated: 2026-08-08 owner: kandev --- @@ -25,6 +26,8 @@ Users discuss external work items in agent chat, but plain ticket keys and pull- - Results are grouped by provider and work-item kind. Provider order and ordering within each group are deterministic; Kandev does not present unrelated provider scores as globally comparable relevance. Sources that are not configured or cannot search the active workspace are omitted from the menu rather than rendered as empty unavailable sections. - Search groups carry provider-neutral display descriptors. Composer, message, and queue code use normalized fields and a generic fallback rather than exhaustive switches over native integrations, so a future plugin bridge can supply the same contract. - Search is debounced, cancels or ignores stale requests, and caps work per provider and configured workspace/project scope. Each connected provider continues examining available in-scope results until it finds the requested number of matches, has no more results, reaches a search bound, or the request is canceled. One disconnected, slow, rate-limited, or failing provider never hides successful results from other providers. +- Cancellation of a superseded search is a normal client outcome. The backend + does not report it as an internal server failure. - Arrow Up and Arrow Down move the active result. Tab or Enter selects it. Pointer or touch selection has the same behavior. Escape closes the menu without changing or sending the draft. - Selection replaces only the active `#query` range with an inline reference chip, appends a trailing space, keeps focus in the composer, and never submits or queues the message. - A chip displays a stable key when one exists (for example `#ENG-123` or `#123`) and otherwise a concise title. It retains a title snapshot and source label for disambiguation. @@ -101,7 +104,13 @@ The editor's `entityReference` atom stores the same presentation and identity fi - Group status is one of `ok`, `not_configured`, `unauthorized`, `rate_limited`, `timeout`, `upstream_error`, or `unsupported_scope`. - `source` is an opaque stable source ID. `display_name` and `kind_label` let the UI render a provider without importing its native model; unknown providers use a generic work-item icon. - Sources with `not_configured` or `unsupported_scope` status are omitted from the menu, but their safe status remains available in the response. Transient failures from configured sources may be shown non-disruptively. Raw provider errors are never returned. -- Invalid query/limit returns 400. An unknown workspace returns 404. Aggregate infrastructure failure returns 500. +- Invalid query/limit returns 400. An unknown workspace returns 404. A request + canceled by the client returns 499. Aggregate infrastructure failure returns + 500, and every 500 has one corresponding structured backend error record. +- Search diagnostics can include the workspace ID, stable failure stage, source, + provider, kind, and safe status. They never include the query, prompt, + credentials, tokens, or raw provider payloads. Provider-level failures remain + represented as safe groups in an HTTP 200 aggregate response. ### Message submission @@ -129,6 +138,11 @@ Typing updates Primed/Searching/Results/Empty. Escape, moving outside the trigge ## Failure modes - A provider timeout, auth failure, rate limit, malformed upstream response, or unsupported workspace scope produces a safe group status while other groups remain selectable. +- A canceled request stops provider work and returns 499. It is not logged as an + unexpected HTTP 500. +- A workspace repository failure or another unexpected aggregate search failure + returns 500 only after the handler writes one structured error with the safe + workspace and failure-stage context. - If the aggregate endpoint is unreachable, the menu preserves the draft and shows a retryable error state. It never submits the draft. - Stale responses after continued typing, session changes, or menu dismissal are ignored. - Invalid or unsafe returned URLs are discarded. URLs must use HTTP(S) and match the configured provider origin; internal routes are rejected. @@ -151,6 +165,16 @@ Typing updates Primed/Searching/Results/Empty. Escape, moving outside the trigge - **GIVEN** the menu is open on a phone viewport, **WHEN** the user taps a result, **THEN** the same chip is inserted and every required row is touch reachable without horizontal document overflow. - **GIVEN** GitHub times out while Jira returns hits, **WHEN** search finishes, **THEN** Jira results remain selectable and GitHub exposes only a safe timeout state. - **GIVEN** the user types another character before an earlier request returns, **WHEN** the older response arrives last, **THEN** only results for the newest query are displayed. +- **GIVEN** the browser cancels a superseded search request, **WHEN** the backend + observes cancellation, **THEN** provider work stops and the request completes + as 499 without an internal-server-error record. +- **GIVEN** workspace validation or aggregate search fails unexpectedly, **WHEN** + the endpoint returns 500, **THEN** one structured backend error identifies the + workspace and stable failure stage without recording the query or provider + secrets. +- **GIVEN** one provider search fails while another succeeds, **WHEN** aggregation + completes, **THEN** HTTP 200 contains a safe status for the failed provider and + retains the successful results. - **GIVEN** a connected provider's first available results do not match the query but later in-scope results do, **WHEN** the user searches, **THEN** matching results are returned up to the requested per-source limit unless the provider has no more results, a workspace/project search bound is reached, or the request is canceled. - **GIVEN** a selected external reference followed by ordinary text, **WHEN** the user explicitly sends, **THEN** the visible message contains a clickable chip/Markdown fallback and its stable identity is stored in message metadata and supplied to the agent as sanitized reference context. - **GIVEN** the agent is busy, **WHEN** the user queues a message containing two references, **THEN** both references survive queue display, backend restart, drain, and sent-message rendering. @@ -172,3 +196,7 @@ Typing updates Primed/Searching/Results/Empty. Escape, moving outside the trigge - Raw provider query-language syntax, advanced filters, user-controlled result pagination, or a full-screen global search page. - Implementing the plugin manifest contribution, workspace permission/grant, Kandev-to-plugin search RPC, or loading plugin-provided sources in this release. The normalized registry and DTO MUST remain transport-neutral so a later plugin bridge can implement the same provider contract without changing composer/message formats. Search-provider contribution is distinct from the plugin-to-Kandev Host data API and its `api_read` capabilities. - Cross-workspace search. + +## Implementation plan + +[Backend failure containment](../../plans/backend-failure-containment/plan.md)