diff --git a/apps/backend/internal/agent/runtime/agentctl/availability.go b/apps/backend/internal/agent/runtime/agentctl/availability.go
new file mode 100644
index 0000000000..fa3946e96b
--- /dev/null
+++ b/apps/backend/internal/agent/runtime/agentctl/availability.go
@@ -0,0 +1,115 @@
+package client
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/kandev/kandev/internal/common/logger"
+ "github.com/kandev/kandev/internal/events"
+ "github.com/kandev/kandev/internal/events/bus"
+ "go.uber.org/zap"
+)
+
+const (
+ AvailabilityStatusAvailable = "available"
+ AvailabilityStatusUnavailable = "unavailable"
+ AvailabilityReasonAgentctlExited = "agentctl_exited"
+)
+
+// AvailabilitySnapshot is the sanitized, install-wide agent runtime state
+// exposed to authenticated clients. It intentionally contains no process or
+// launcher details.
+type AvailabilitySnapshot struct {
+ Status string `json:"status"`
+ Reason string `json:"reason,omitempty"`
+ OccurredAt *time.Time `json:"occurred_at,omitempty"`
+}
+
+// Availability owns the monotonic runtime availability state. A runtime is
+// unpublished until its health check and authentication handshake succeed;
+// after that point, an unexpected launcher exit is a terminal unavailable
+// state for the lifetime of the backend process.
+type Availability struct {
+ mu sync.RWMutex
+ publishMu sync.Mutex
+ eventBus bus.EventBus
+ logger *logger.Logger
+ snapshot AvailabilitySnapshot
+ published bool
+}
+
+// NewAvailability creates an unpublished runtime availability tracker.
+func NewAvailability(eventBus bus.EventBus, log *logger.Logger) *Availability {
+ return &Availability{eventBus: eventBus, logger: log}
+}
+
+// Snapshot returns a copy of the current published state.
+func (a *Availability) Snapshot() (AvailabilitySnapshot, bool) {
+ a.mu.RLock()
+ defer a.mu.RUnlock()
+ if !a.published {
+ return AvailabilitySnapshot{}, false
+ }
+
+ snapshot := a.snapshot
+ if a.snapshot.OccurredAt != nil {
+ occurredAt := *a.snapshot.OccurredAt
+ snapshot.OccurredAt = &occurredAt
+ }
+ return snapshot, true
+}
+
+// MarkAvailable publishes the first healthy-and-authenticated state.
+func (a *Availability) MarkAvailable() {
+ a.publishMu.Lock()
+ defer a.publishMu.Unlock()
+
+ a.mu.Lock()
+ if a.published {
+ a.mu.Unlock()
+ return
+ }
+ a.snapshot = AvailabilitySnapshot{Status: AvailabilityStatusAvailable}
+ a.published = true
+ snapshot := a.snapshot
+ a.mu.Unlock()
+
+ a.publish(snapshot)
+}
+
+// MarkUnavailable records an unexpected agentctl exit. The transition is
+// intentionally fail-closed and cannot be reversed until the backend restarts.
+func (a *Availability) MarkUnavailable() {
+ a.publishMu.Lock()
+ defer a.publishMu.Unlock()
+
+ a.mu.Lock()
+ if !a.published || a.snapshot.Status == AvailabilityStatusUnavailable {
+ a.mu.Unlock()
+ return
+ }
+ now := time.Now().UTC()
+ a.snapshot = AvailabilitySnapshot{
+ Status: AvailabilityStatusUnavailable,
+ Reason: AvailabilityReasonAgentctlExited,
+ OccurredAt: &now,
+ }
+ snapshot := a.snapshot
+ a.mu.Unlock()
+
+ a.publish(snapshot)
+}
+
+func (a *Availability) publish(snapshot AvailabilitySnapshot) {
+ if a.eventBus == nil {
+ return
+ }
+ if err := a.eventBus.Publish(
+ context.Background(),
+ events.AgentRuntimeAvailabilityChanged,
+ bus.NewEvent(events.AgentRuntimeAvailabilityChanged, "agentctl", snapshot),
+ ); err != nil && a.logger != nil {
+ a.logger.Warn("failed to publish agent runtime availability", zap.Error(err))
+ }
+}
diff --git a/apps/backend/internal/agent/runtime/agentctl/availability_test.go b/apps/backend/internal/agent/runtime/agentctl/availability_test.go
new file mode 100644
index 0000000000..8e01f41ff4
--- /dev/null
+++ b/apps/backend/internal/agent/runtime/agentctl/availability_test.go
@@ -0,0 +1,162 @@
+package client
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/kandev/kandev/internal/events"
+ "github.com/kandev/kandev/internal/events/bus"
+)
+
+func TestAvailabilityPublishesSanitizedMonotonicSnapshots(t *testing.T) {
+ eventBus := bus.NewMemoryEventBus(newTestLogger())
+ t.Cleanup(eventBus.Close)
+ var published []*bus.Event
+ _, err := eventBus.Subscribe(events.AgentRuntimeAvailabilityChanged, func(_ context.Context, event *bus.Event) error {
+ published = append(published, event)
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("subscribe availability events: %v", err)
+ }
+
+ availability := NewAvailability(eventBus, newTestLogger())
+ if _, ok := availability.Snapshot(); ok {
+ t.Fatal("startup availability must not be published before health and auth")
+ }
+
+ availability.MarkAvailable()
+ snapshot, ok := availability.Snapshot()
+ if !ok || snapshot.Status != AvailabilityStatusAvailable {
+ t.Fatalf("available snapshot = %+v, published=%v", snapshot, ok)
+ }
+ if snapshot.Reason != "" || snapshot.OccurredAt != nil {
+ t.Fatalf("available snapshot exposes failure details: %+v", snapshot)
+ }
+
+ availability.MarkUnavailable()
+ unavailable, ok := availability.Snapshot()
+ if !ok || unavailable.Status != AvailabilityStatusUnavailable {
+ t.Fatalf("unavailable snapshot = %+v, published=%v", unavailable, ok)
+ }
+ if unavailable.Reason != AvailabilityReasonAgentctlExited || unavailable.OccurredAt == nil {
+ t.Fatalf("unavailable snapshot = %+v", unavailable)
+ }
+ occurredAt := *unavailable.OccurredAt
+
+ availability.MarkUnavailable()
+ availability.MarkAvailable()
+ final, ok := availability.Snapshot()
+ if !ok || final.Status != AvailabilityStatusUnavailable || final.Reason != AvailabilityReasonAgentctlExited {
+ t.Fatalf("availability was not monotonic: %+v, published=%v", final, ok)
+ }
+ if final.OccurredAt == nil || !final.OccurredAt.Equal(occurredAt) {
+ t.Fatalf("unavailable occurrence changed: got %v, want %v", final.OccurredAt, occurredAt)
+ }
+ if len(published) != 2 {
+ t.Fatalf("published event count = %d, want available and unavailable", len(published))
+ }
+}
+
+func TestAvailabilitySnapshotReadsAreSafeDuringTransition(t *testing.T) {
+ availability := NewAvailability(nil, newTestLogger())
+ availability.MarkAvailable()
+
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ for index := 0; index < 16; index++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ for count := 0; count < 1000; count++ {
+ _, _ = availability.Snapshot()
+ }
+ }()
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ availability.MarkUnavailable()
+ }()
+ close(start)
+ wg.Wait()
+
+ snapshot, ok := availability.Snapshot()
+ if !ok || snapshot.Status != AvailabilityStatusUnavailable {
+ t.Fatalf("final snapshot = %+v, published=%v", snapshot, ok)
+ }
+}
+
+func TestAvailabilityPublishesTransitionsInCommitOrder(t *testing.T) {
+ eventBus := bus.NewMemoryEventBus(newTestLogger())
+ t.Cleanup(eventBus.Close)
+ var published []AvailabilitySnapshot
+ var publishedMu sync.Mutex
+ firstPublishStarted := make(chan struct{})
+ releaseFirstPublish := make(chan struct{})
+ var publishCalls atomic.Int32
+ _, err := eventBus.Subscribe(events.AgentRuntimeAvailabilityChanged, func(_ context.Context, event *bus.Event) error {
+ if event.Data == nil {
+ return nil
+ }
+ if publishCalls.Add(1) == 1 {
+ close(firstPublishStarted)
+ <-releaseFirstPublish
+ }
+ snapshot, ok := event.Data.(AvailabilitySnapshot)
+ if !ok {
+ t.Fatalf("availability event data = %T, want AvailabilitySnapshot", event.Data)
+ }
+ publishedMu.Lock()
+ published = append(published, snapshot)
+ publishedMu.Unlock()
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("subscribe availability events: %v", err)
+ }
+
+ availability := NewAvailability(eventBus, newTestLogger())
+ availableDone := make(chan struct{})
+ go func() {
+ availability.MarkAvailable()
+ close(availableDone)
+ }()
+ <-firstPublishStarted
+
+ unavailableDone := make(chan struct{})
+ go func() {
+ availability.MarkUnavailable()
+ close(unavailableDone)
+ }()
+
+ select {
+ case <-unavailableDone:
+ t.Fatal("unavailable transition completed before available publication was released")
+ case <-time.After(25 * time.Millisecond):
+ }
+
+ close(releaseFirstPublish)
+ select {
+ case <-availableDone:
+ case <-time.After(time.Second):
+ t.Fatal("available transition did not complete")
+ }
+ select {
+ case <-unavailableDone:
+ case <-time.After(time.Second):
+ t.Fatal("unavailable transition did not complete")
+ }
+
+ publishedMu.Lock()
+ defer publishedMu.Unlock()
+ if len(published) != 2 || published[0].Status != AvailabilityStatusAvailable ||
+ published[1].Status != AvailabilityStatusUnavailable {
+ t.Fatalf("published availability transitions = %+v, want available then unavailable", published)
+ }
+}
diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go
index ad514cad75..8b580885bf 100644
--- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go
+++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go
@@ -25,10 +25,11 @@ import (
// Launcher manages an agentctl subprocess.
type Launcher struct {
- binaryPath string
- host string
- port int
- logger *logger.Logger
+ binaryPath string
+ host string
+ port int
+ logger *logger.Logger
+ onUnexpectedExit func()
cmd *exec.Cmd
exited chan struct{}
@@ -59,9 +60,10 @@ type Launcher struct {
// Config holds configuration for the launcher.
type Config struct {
- BinaryPath string // Path to agentctl binary (auto-detected if empty)
- Host string // Host to bind to (default: localhost)
- Port int // Control port (default: 39429)
+ BinaryPath string // Path to agentctl binary (auto-detected if empty)
+ Host string // Host to bind to (default: localhost)
+ Port int // Control port (default: 39429)
+ OnUnexpectedExit func() // Called once when the child exits without Stop.
}
// New creates a new Launcher.
@@ -77,11 +79,12 @@ func New(cfg Config, log *logger.Logger) *Launcher {
}
return &Launcher{
- binaryPath: cfg.BinaryPath,
- host: cfg.Host,
- port: cfg.Port,
- logger: log.WithFields(zap.String("component", "agentctl-launcher")),
- exited: make(chan struct{}),
+ binaryPath: cfg.BinaryPath,
+ host: cfg.Host,
+ port: cfg.Port,
+ onUnexpectedExit: cfg.OnUnexpectedExit,
+ logger: log.WithFields(zap.String("component", "agentctl-launcher")),
+ exited: make(chan struct{}),
}
}
@@ -484,6 +487,9 @@ func (l *Launcher) monitorExit() {
zap.Int("pid", l.cmd.Process.Pid),
zap.Int("exit_code", l.cmd.ProcessState.ExitCode()))
}
+ if !stopping && l.onUnexpectedExit != nil {
+ l.onUnexpectedExit()
+ }
close(l.exited)
}
diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go
index 1f1979712b..0025b97a2f 100644
--- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go
+++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_test.go
@@ -1,10 +1,97 @@
package launcher
import (
+ "fmt"
+ "io"
"os"
+ "os/exec"
+ "reflect"
+ "runtime"
+ "strconv"
"testing"
+ "time"
+
+ "github.com/kandev/kandev/internal/common/logger"
)
+func TestLauncherConfigSupportsUnexpectedExitCallback(t *testing.T) {
+ field, ok := reflect.TypeOf(Config{}).FieldByName("OnUnexpectedExit")
+ if !ok {
+ t.Fatal("launcher config has no unexpected-exit callback")
+ }
+ if field.Type != reflect.TypeOf((func())(nil)) {
+ t.Fatalf("unexpected-exit callback type = %v, want func()", field.Type)
+ }
+}
+
+func TestLauncherUnexpectedExitCallbackRunsForCleanAndFailedExit(t *testing.T) {
+ for _, exitCode := range []int{0, 7} {
+ t.Run(fmt.Sprintf("exit-%d", exitCode), func(t *testing.T) {
+ called := startLauncherMonitorTest(t, exitCode, false)
+ if !called {
+ t.Fatalf("unexpected-exit callback was not called for exit code %d", exitCode)
+ }
+ })
+ }
+}
+
+func TestLauncherUnexpectedExitCallbackSkipsIntentionalStop(t *testing.T) {
+ if called := startLauncherMonitorTest(t, 7, true); called {
+ t.Fatal("unexpected-exit callback ran during intentional stop")
+ }
+}
+
+func startLauncherMonitorTest(t *testing.T, exitCode int, stopping bool) bool {
+ t.Helper()
+ cmd := launcherExitCommand(exitCode)
+ cmd.Stdout = io.Discard
+ cmd.Stderr = io.Discard
+ if err := cmd.Start(); err != nil {
+ t.Fatalf("start launcher child: %v", err)
+ }
+
+ callback := make(chan struct{}, 1)
+ launcher := &Launcher{
+ cmd: cmd,
+ exited: make(chan struct{}),
+ logger: newUnexpectedExitTestLogger(t),
+ onUnexpectedExit: func() { callback <- struct{}{} },
+ }
+ launcher.mu.Lock()
+ launcher.stopping = stopping
+ launcher.mu.Unlock()
+ go launcher.monitorExit()
+
+ select {
+ case <-launcher.exited:
+ case <-time.After(time.Second):
+ t.Fatal("launcher monitor did not finish")
+ }
+ select {
+ case <-callback:
+ return true
+ default:
+ return false
+ }
+}
+
+func launcherExitCommand(exitCode int) *exec.Cmd {
+ code := strconv.Itoa(exitCode)
+ if runtime.GOOS == "windows" {
+ return exec.Command("cmd.exe", "/c", "exit", code)
+ }
+ return exec.Command("sh", "-c", "exit "+code)
+}
+
+func newUnexpectedExitTestLogger(t *testing.T) *logger.Logger {
+ t.Helper()
+ log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"})
+ if err != nil {
+ t.Fatalf("new logger: %v", err)
+ }
+ return log
+}
+
func TestCloseParentPipe_ClosesAndNils(t *testing.T) {
r, w, err := os.Pipe()
if err != nil {
diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go
index b485d96703..588af116fc 100644
--- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go
+++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go
@@ -534,6 +534,7 @@ func (a *Adapter) sendUpdate(event AgentEvent) {
if lifetimeCtx == nil {
lifetimeCtx = context.Background()
}
+ event.NormalizedPayload = event.NormalizedPayload.Snapshot()
closedCh := a.closedCh
if closedCh != nil {
a.updateSendWg.Add(1)
@@ -558,6 +559,7 @@ func (a *Adapter) sendUpdateLocked(event AgentEvent) bool {
if a.closed {
return false
}
+ event.NormalizedPayload = event.NormalizedPayload.Snapshot()
select {
case a.updatesCh <- event:
return true
diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go
index e908a32c68..193b0377e2 100644
--- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go
+++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_tools.go
@@ -546,36 +546,7 @@ func (a *Adapter) clearCodexSubagentCorrelationsLocked(sessionID string) {
}
func cloneSubagentPayload(payload *streams.NormalizedPayload) *streams.NormalizedPayload {
- if payload == nil || payload.SubagentTask() == nil {
- return payload
- }
- src := payload.SubagentTask()
- clone := streams.NewSubagentTask(src.Description, src.Prompt, src.SubagentType)
- dst := clone.SubagentTask()
- dst.Status = src.Status
- dst.AgentID = src.AgentID
- dst.Model = src.Model
- dst.ChildSessionID = src.ChildSessionID
- dst.DurationMs = src.DurationMs
- dst.TotalTokens = src.TotalTokens
- if src.ToolUseCount != nil {
- count := *src.ToolUseCount
- dst.ToolUseCount = &count
- }
- dst.ResultText = src.ResultText
- dst.IsAsync = src.IsAsync
- dst.OutputFile = src.OutputFile
- dst.CanReadOutputFile = src.CanReadOutputFile
- dst.SetIsAuggie(src.IsAuggie())
- if background := payload.BackgroundWork(); background != nil {
- clone.SetBackgroundWorkIdentity(
- background.Kind,
- background.WorkID,
- background.Detached,
- background.Ended,
- )
- }
- return clone
+ return payload.Snapshot()
}
func mergeCodexSubagentPayload(current, candidate *streams.NormalizedPayload) {
diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go
index ae0563e6fe..e62ae0325b 100644
--- a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go
+++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go
@@ -2,6 +2,7 @@ package acp
import (
"encoding/json"
+ "sync"
"testing"
"time"
@@ -64,6 +65,106 @@ func TestSendUpdateBackpressureIsCanceledOnClose(t *testing.T) {
}
}
+func TestSendUpdateSnapshotsNormalizedPayloadBeforeQueueing(t *testing.T) {
+ a := newTestAdapter()
+ t.Cleanup(func() { _ = a.Close() })
+
+ nested := map[string]any{"value": "before"}
+ payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested})
+ a.sendUpdate(AgentEvent{
+ Type: streams.EventTypeToolCall,
+ NormalizedPayload: payload,
+ })
+
+ event := <-a.updatesCh
+ nested["value"] = "after"
+
+ data, err := json.Marshal(event.NormalizedPayload)
+ if err != nil {
+ t.Fatalf("marshal queued payload: %v", err)
+ }
+ want := `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}`
+ if string(data) != want {
+ t.Fatalf("queued payload changed after source mutation: got %s, want %s", data, want)
+ }
+}
+
+func TestSendUpdateLockedSnapshotsNormalizedPayloadBeforeQueueing(t *testing.T) {
+ a := newTestAdapter()
+ t.Cleanup(func() { _ = a.Close() })
+
+ nested := map[string]any{"value": "before"}
+ payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested})
+ a.mu.Lock()
+ if !a.sendUpdateLocked(AgentEvent{
+ Type: streams.EventTypeToolCall,
+ NormalizedPayload: payload,
+ }) {
+ a.mu.Unlock()
+ t.Fatal("sendUpdateLocked did not enqueue event")
+ }
+ a.mu.Unlock()
+
+ event := <-a.updatesCh
+ nested["value"] = "after"
+ data, err := json.Marshal(event.NormalizedPayload)
+ if err != nil {
+ t.Fatalf("marshal queued payload: %v", err)
+ }
+ if !json.Valid(data) || string(data) == "" {
+ t.Fatalf("queued payload is not valid JSON: %s", data)
+ }
+ if string(data) != `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}` {
+ t.Fatalf("queued payload changed after source mutation: %s", data)
+ }
+}
+
+func TestQueuedNormalizedPayloadCanMarshalDuringSourceMutation(t *testing.T) {
+ a := newTestAdapter()
+ t.Cleanup(func() { _ = a.Close() })
+
+ nested := map[string]any{"value": "before"}
+ payload := streams.NewGeneric("custom_tool", map[string]any{"nested": nested})
+ a.sendUpdate(AgentEvent{
+ Type: streams.EventTypeToolCall,
+ NormalizedPayload: payload,
+ })
+ event := <-a.updatesCh
+
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ <-start
+ for index := 0; index < 10000; index++ {
+ nested["value"] = index
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ <-start
+ for index := 0; index < 10000; index++ {
+ if _, err := json.Marshal(event.NormalizedPayload); err != nil {
+ t.Errorf("marshal queued payload: %v", err)
+ }
+ }
+ }()
+ close(start)
+ wg.Wait()
+
+ data, err := json.Marshal(event.NormalizedPayload)
+ if err != nil {
+ t.Fatalf("marshal final queued payload: %v", err)
+ }
+ if !json.Valid(data) || string(data) == "" {
+ t.Fatalf("queued payload is not valid JSON: %s", data)
+ }
+ if string(data) != `{"kind":"generic","generic":{"name":"custom_tool","input":{"nested":{"value":"before"}}}}` {
+ t.Fatalf("queued payload changed during source mutation: %s", data)
+ }
+}
+
func TestConvertACPConfigOptions_PreservesDescriptions(t *testing.T) {
description := "Controls how much reasoning the model performs."
valueDescription := "Uses more reasoning for complex tasks."
diff --git a/apps/backend/internal/agentctl/types/streams/tool_payload.go b/apps/backend/internal/agentctl/types/streams/tool_payload.go
index 6d83d915f3..e8bd9c4de9 100644
--- a/apps/backend/internal/agentctl/types/streams/tool_payload.go
+++ b/apps/backend/internal/agentctl/types/streams/tool_payload.go
@@ -1,6 +1,9 @@
package streams
-import "encoding/json"
+import (
+ "encoding/json"
+ "reflect"
+)
// ToolKind categorizes the normalized tool operation
type ToolKind string
@@ -83,6 +86,209 @@ func (p *NormalizedPayload) ShowPlan() *ShowPlanPayload { return p.showP
func (p *NormalizedPayload) ManageTodos() *ManageTodosPayload { return p.manageTodos }
func (p *NormalizedPayload) Misc() *MiscPayload { return p.misc }
+// Snapshot returns a detached copy of the normalized payload. A snapshot owns
+// every pointer, slice, and JSON-compatible value reachable from the payload so
+// it can safely cross an asynchronous event boundary.
+func (p *NormalizedPayload) Snapshot() *NormalizedPayload {
+ if p == nil {
+ return nil
+ }
+ return &NormalizedPayload{
+ kind: p.kind,
+ readFile: cloneReadFilePayload(p.readFile),
+ modifyFile: cloneModifyFilePayload(p.modifyFile),
+ shellExec: cloneShellExecPayload(p.shellExec),
+ codeSearch: cloneCodeSearchPayload(p.codeSearch),
+ httpRequest: cloneHttpRequestPayload(p.httpRequest),
+ generic: cloneGenericPayload(p.generic),
+ createTask: cloneCreateTaskPayload(p.createTask),
+ subagentTask: cloneSubagentTaskPayload(p.subagentTask),
+ showPlan: cloneShowPlanPayload(p.showPlan),
+ manageTodos: cloneManageTodosPayload(p.manageTodos),
+ misc: cloneMiscPayload(p.misc),
+ backgroundWork: cloneBackgroundWorkPayload(p.backgroundWork),
+ monitor: cloneMonitorPayload(p.monitor),
+ }
+}
+
+func cloneReadFilePayload(src *ReadFilePayload) *ReadFilePayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ if src.Output != nil {
+ output := *src.Output
+ clone.Output = &output
+ }
+ return &clone
+}
+
+func cloneModifyFilePayload(src *ModifyFilePayload) *ModifyFilePayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ clone.Mutations = append([]FileMutation(nil), src.Mutations...)
+ return &clone
+}
+
+func cloneShellExecPayload(src *ShellExecPayload) *ShellExecPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ if src.Output != nil {
+ output := *src.Output
+ clone.Output = &output
+ }
+ return &clone
+}
+
+func cloneCodeSearchPayload(src *CodeSearchPayload) *CodeSearchPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ if src.Output != nil {
+ output := *src.Output
+ output.Files = append([]string(nil), src.Output.Files...)
+ clone.Output = &output
+ }
+ return &clone
+}
+
+func cloneHttpRequestPayload(src *HttpRequestPayload) *HttpRequestPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ return &clone
+}
+
+func cloneGenericPayload(src *GenericPayload) *GenericPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ clone.Input = cloneJSONValue(src.Input)
+ clone.Output = cloneJSONValue(src.Output)
+ return &clone
+}
+
+func cloneCreateTaskPayload(src *CreateTaskPayload) *CreateTaskPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ return &clone
+}
+
+func cloneSubagentTaskPayload(src *SubagentTaskPayload) *SubagentTaskPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ if src.ToolUseCount != nil {
+ count := *src.ToolUseCount
+ clone.ToolUseCount = &count
+ }
+ return &clone
+}
+
+func cloneShowPlanPayload(src *ShowPlanPayload) *ShowPlanPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ clone.Steps = append([]string(nil), src.Steps...)
+ return &clone
+}
+
+func cloneManageTodosPayload(src *ManageTodosPayload) *ManageTodosPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ clone.Items = append([]TodoItem(nil), src.Items...)
+ return &clone
+}
+
+func cloneMiscPayload(src *MiscPayload) *MiscPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ clone.Details = cloneJSONValue(src.Details)
+ return &clone
+}
+
+func cloneBackgroundWorkPayload(src *BackgroundWorkPayload) *BackgroundWorkPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ return &clone
+}
+
+func cloneMonitorPayload(src *MonitorPayload) *MonitorPayload {
+ if src == nil {
+ return nil
+ }
+ clone := *src
+ return &clone
+}
+
+func cloneJSONValue(value any) any {
+ if value == nil {
+ return nil
+ }
+ return cloneJSONReflectValue(reflect.ValueOf(value)).Interface()
+}
+
+func cloneJSONReflectValue(value reflect.Value) reflect.Value {
+ if !value.IsValid() {
+ return value
+ }
+ switch value.Kind() {
+ case reflect.Interface:
+ if value.IsNil() {
+ return reflect.Zero(value.Type())
+ }
+ clone := cloneJSONReflectValue(value.Elem())
+ result := reflect.New(value.Type()).Elem()
+ result.Set(clone)
+ return result
+ case reflect.Map:
+ if value.IsNil() {
+ return reflect.Zero(value.Type())
+ }
+ result := reflect.MakeMapWithSize(value.Type(), value.Len())
+ iter := value.MapRange()
+ for iter.Next() {
+ result.SetMapIndex(iter.Key(), cloneJSONReflectValue(iter.Value()))
+ }
+ return result
+ case reflect.Slice:
+ if value.IsNil() {
+ return reflect.Zero(value.Type())
+ }
+ result := reflect.MakeSlice(value.Type(), value.Len(), value.Len())
+ for index := 0; index < value.Len(); index++ {
+ result.Index(index).Set(cloneJSONReflectValue(value.Index(index)))
+ }
+ return result
+ case reflect.Pointer:
+ if value.IsNil() {
+ return reflect.Zero(value.Type())
+ }
+ result := reflect.New(value.Type().Elem())
+ result.Elem().Set(cloneJSONReflectValue(value.Elem()))
+ return result
+ default:
+ return value
+ }
+}
+
// MarshalJSON implements custom JSON marshaling for NormalizedPayload.
func (p *NormalizedPayload) MarshalJSON() ([]byte, error) {
type jsonPayload struct {
diff --git a/apps/backend/internal/agentctl/types/streams/tool_payload_test.go b/apps/backend/internal/agentctl/types/streams/tool_payload_test.go
index 6f9a7ff205..c120a77ce6 100644
--- a/apps/backend/internal/agentctl/types/streams/tool_payload_test.go
+++ b/apps/backend/internal/agentctl/types/streams/tool_payload_test.go
@@ -85,3 +85,186 @@ func TestNormalizedPayloadMarshalRoundTrip(t *testing.T) {
})
}
}
+
+func TestNormalizedPayloadSnapshotDetachesMutablePayloads(t *testing.T) {
+ tests := []struct {
+ name string
+ payload *NormalizedPayload
+ check func(*NormalizedPayload)
+ }{
+ {
+ name: "read file output",
+ payload: func() *NormalizedPayload {
+ payload := NewReadFile("before.go", 1, 2)
+ payload.ReadFile().Output = &ReadFileOutput{Content: "before"}
+ return payload
+ }(),
+ },
+ {
+ name: "modify file mutations",
+ payload: NewModifyFile("before.go", []FileMutation{{Diff: "before"}}),
+ },
+ {
+ name: "shell output",
+ payload: func() *NormalizedPayload {
+ payload := NewShellExec("echo before", "/tmp", "before", 1, false)
+ payload.ShellExec().Output = &ShellExecOutput{Stdout: "before", RawTerminalOutput: "before"}
+ return payload
+ }(),
+ },
+ {
+ name: "code search output",
+ payload: func() *NormalizedPayload {
+ payload := NewCodeSearch("before", "before", "/tmp", "*.go")
+ payload.CodeSearch().Output = &CodeSearchOutput{Files: []string{"before.go"}}
+ return payload
+ }(),
+ },
+ {
+ name: "http request",
+ payload: &NormalizedPayload{
+ kind: ToolKindHttpRequest,
+ httpRequest: &HttpRequestPayload{URL: "https://before", Response: "before"},
+ },
+ },
+ {
+ name: "generic nested JSON",
+ payload: func() *NormalizedPayload {
+ rawMessage := json.RawMessage(`{"nested":["before"]}`)
+ return &NormalizedPayload{
+ kind: ToolKindGeneric,
+ generic: &GenericPayload{
+ Name: "before",
+ Input: map[string]any{
+ "nested": map[string]any{"values": []any{rawMessage}},
+ },
+ Output: []any{map[string]any{"value": "before"}},
+ },
+ }
+ }(),
+ },
+ {
+ name: "create task",
+ payload: &NormalizedPayload{kind: ToolKindCreateTask, createTask: &CreateTaskPayload{Title: "before"}},
+ },
+ {
+ name: "subagent result",
+ payload: func() *NormalizedPayload {
+ count := 1
+ payload := NewSubagentTask("before", "before", "before")
+ payload.SubagentTask().ToolUseCount = &count
+ payload.SubagentTask().SetIsAuggie(true)
+ return payload
+ }(),
+ check: func(snapshot *NormalizedPayload) {
+ if !snapshot.SubagentTask().IsAuggie() {
+ t.Error("subagent adapter marker was not copied")
+ }
+ },
+ },
+ {
+ name: "plan steps",
+ payload: &NormalizedPayload{
+ kind: ToolKindShowPlan,
+ showPlan: &ShowPlanPayload{Summary: "before", Steps: []string{"before"}},
+ },
+ },
+ {
+ name: "todo items",
+ payload: &NormalizedPayload{
+ kind: ToolKindManageTodos,
+ manageTodos: &ManageTodosPayload{Operation: "before", Items: []TodoItem{{Description: "before"}}},
+ },
+ },
+ {
+ name: "misc nested JSON",
+ payload: func() *NormalizedPayload {
+ rawMessage := json.RawMessage(`{"nested":["before"]}`)
+ return &NormalizedPayload{
+ kind: ToolKindMisc,
+ misc: &MiscPayload{Label: "before", Details: map[string]any{
+ "nested": []any{rawMessage},
+ }},
+ }
+ }(),
+ },
+ {
+ name: "adapter provenance",
+ payload: func() *NormalizedPayload {
+ payload := &NormalizedPayload{
+ kind: ToolKindMisc,
+ misc: &MiscPayload{Label: "before", Details: map[string]any{"value": "before"}},
+ }
+ payload.SetMonitorIdentity("before-task", false)
+ return payload
+ }(),
+ check: func(snapshot *NormalizedPayload) {
+ if snapshot.Monitor().TaskID != "before-task" || snapshot.BackgroundWork().WorkID != "before-task" {
+ t.Error("adapter provenance was not copied")
+ }
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ before, err := json.Marshal(tt.payload.Snapshot())
+ if err != nil {
+ t.Fatalf("marshal snapshot before mutation: %v", err)
+ }
+ snapshot := tt.payload.Snapshot()
+ mutateSnapshotSource(tt.payload)
+ after, err := json.Marshal(snapshot)
+ if err != nil {
+ t.Fatalf("marshal snapshot after mutation: %v", err)
+ }
+ if string(after) != string(before) {
+ t.Fatalf("snapshot changed after source mutation: got %s, want %s", after, before)
+ }
+ if tt.check != nil {
+ tt.check(snapshot)
+ }
+ })
+ }
+}
+
+func mutateSnapshotSource(payload *NormalizedPayload) {
+ switch payload.Kind() {
+ case ToolKindReadFile:
+ payload.ReadFile().Output.Content = "after"
+ case ToolKindModifyFile:
+ payload.ModifyFile().Mutations[0].Diff = "after"
+ case ToolKindShellExec:
+ payload.ShellExec().Output.Stdout = "after"
+ payload.ShellExec().Output.RawTerminalOutput = "after"
+ case ToolKindCodeSearch:
+ payload.CodeSearch().Output.Files[0] = "after.go"
+ case ToolKindHttpRequest:
+ payload.HttpRequest().Response = "after"
+ case ToolKindGeneric:
+ payload.Generic().Input.(map[string]any)["nested"].(map[string]any)["values"].([]any)[0].(json.RawMessage)[0] = 'x'
+ payload.Generic().Output.([]any)[0].(map[string]any)["value"] = "after"
+ if payload.Monitor() != nil {
+ payload.Monitor().TaskID = "after-task"
+ payload.BackgroundWork().WorkID = "after-task"
+ }
+ case ToolKindCreateTask:
+ payload.CreateTask().Title = "after"
+ case ToolKindSubagentTask:
+ payload.SubagentTask().Description = "after"
+ *payload.SubagentTask().ToolUseCount = 2
+ case ToolKindShowPlan:
+ payload.ShowPlan().Steps[0] = "after"
+ case ToolKindManageTodos:
+ payload.ManageTodos().Items[0].Description = "after"
+ case ToolKindMisc:
+ details := payload.Misc().Details.(map[string]any)
+ if nested, ok := details["nested"]; ok {
+ nested.([]any)[0].(json.RawMessage)[0] = 'x'
+ }
+ if payload.Monitor() != nil {
+ payload.Monitor().TaskID = "after-task"
+ payload.BackgroundWork().WorkID = "after-task"
+ }
+ }
+}
diff --git a/apps/backend/internal/backendapp/agent_runtime_boot_test.go b/apps/backend/internal/backendapp/agent_runtime_boot_test.go
new file mode 100644
index 0000000000..07a65902cb
--- /dev/null
+++ b/apps/backend/internal/backendapp/agent_runtime_boot_test.go
@@ -0,0 +1,57 @@
+package backendapp
+
+import (
+ "context"
+ "net/http/httptest"
+ "testing"
+
+ client "github.com/kandev/kandev/internal/agent/runtime/agentctl"
+ "github.com/kandev/kandev/internal/webapp"
+)
+
+func TestBootInitialStateHydratesAgentRuntimeAvailability(t *testing.T) {
+ availability := client.NewAvailability(nil, testLogger(t))
+ availability.MarkAvailable()
+
+ state := bootInitialState(
+ context.Background(),
+ nil,
+ routeParams{agentRuntimeAvailability: availability},
+ webapp.ClassifyRoute("/"),
+ )
+
+ runtimeState, ok := state["agentRuntime"].(client.AvailabilitySnapshot)
+ if !ok {
+ t.Fatalf("agentRuntime state = %#v, want AvailabilitySnapshot", state["agentRuntime"])
+ }
+ if runtimeState.Status != client.AvailabilityStatusAvailable || runtimeState.Reason != "" {
+ t.Fatalf("agentRuntime state = %+v", runtimeState)
+ }
+}
+
+func TestBootInitialStateDoesNotExposeRuntimeToUnauthenticatedVisitors(t *testing.T) {
+ availability := client.NewAvailability(nil, testLogger(t))
+ availability.MarkAvailable()
+ availability.MarkUnavailable()
+ request := httptest.NewRequest("GET", "/", nil)
+
+ state := bootInitialState(
+ context.Background(),
+ request,
+ routeParams{
+ authSvc: newSSOTestAuthService(t),
+ agentRuntimeAvailability: availability,
+ },
+ webapp.ClassifyRoute("/"),
+ )
+
+ if _, ok := state["agentRuntime"]; ok {
+ t.Fatalf("unauthenticated boot state exposed agentRuntime: %#v", state)
+ }
+ if _, ok := state["features"]; !ok {
+ t.Fatal("unauthenticated boot state lost features")
+ }
+ if _, ok := state["auth"]; !ok {
+ t.Fatal("unauthenticated boot state lost auth")
+ }
+}
diff --git a/apps/backend/internal/backendapp/agentctl.go b/apps/backend/internal/backendapp/agentctl.go
index 2ae83c5065..8f4a8d44fd 100644
--- a/apps/backend/internal/backendapp/agentctl.go
+++ b/apps/backend/internal/backendapp/agentctl.go
@@ -22,14 +22,21 @@ type agentctlLauncherResult struct {
// for agent execution on the host machine.
// If the configured port is unavailable, the launcher may fall back to an OS-assigned
// port. In that case, cfg.Agent.StandalonePort is updated to reflect the actual port.
-func provideAgentctlLauncher(ctx context.Context, cfg *config.Config, log *logger.Logger) (*agentctlLauncherResult, error) {
+func provideAgentctlLauncher(
+ ctx context.Context,
+ cfg *config.Config,
+ log *logger.Logger,
+ availability *agentctlclient.Availability,
+) (*agentctlLauncherResult, error) {
l, cleanup, err := launcher.Provide(ctx, launcher.Config{
- Host: cfg.Agent.StandaloneHost,
- Port: cfg.Agent.StandalonePort,
+ Host: cfg.Agent.StandaloneHost,
+ Port: cfg.Agent.StandalonePort,
+ OnUnexpectedExit: availability.MarkUnavailable,
}, log)
if err != nil {
return nil, err
}
+ availability.MarkAvailable()
// Update config with the actual port (may differ if fallback was used)
if actualPort := l.Port(); actualPort != cfg.Agent.StandalonePort {
log.Info("agentctl port changed from configured value",
diff --git a/apps/backend/internal/backendapp/boot_state.go b/apps/backend/internal/backendapp/boot_state.go
index 7fa361ebe6..81e326f13a 100644
--- a/apps/backend/internal/backendapp/boot_state.go
+++ b/apps/backend/internal/backendapp/boot_state.go
@@ -75,6 +75,11 @@ func bootInitialState(
return state
}
}
+ if p.agentRuntimeAvailability != nil {
+ if snapshot, ok := p.agentRuntimeAvailability.Snapshot(); ok {
+ state["agentRuntime"] = snapshot
+ }
+ }
if route.Route == webapp.RouteSettings {
builder.addWorkspaceState(ctx, state, nil)
diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go
index e75659d024..c218ba4358 100644
--- a/apps/backend/internal/backendapp/helpers.go
+++ b/apps/backend/internal/backendapp/helpers.go
@@ -543,6 +543,7 @@ type routeParams struct {
secretStore secrets.SecretStore
mcpConfigSvc *mcpconfig.Service
authSvc *auth.Service
+ agentRuntimeAvailability *client.Availability
addCleanup func(func() error)
repoCloner *repoclone.Cloner
version string
diff --git a/apps/backend/internal/backendapp/httpserver_test.go b/apps/backend/internal/backendapp/httpserver_test.go
index a92142c4d8..1bbb802ce3 100644
--- a/apps/backend/internal/backendapp/httpserver_test.go
+++ b/apps/backend/internal/backendapp/httpserver_test.go
@@ -22,7 +22,7 @@ func TestBuildHTTPServerAbortsWhenInterlockTokenGenerationFails(t *testing.T) {
newInterimSettingsInterlockToken = func() (string, error) { return "", wantErr }
t.Cleanup(func() { newInterimSettingsInterlockToken = original })
- server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
if !errors.Is(err, wantErr) {
t.Fatalf("buildHTTPServer error = %v, want %v", err, wantErr)
}
diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go
index c6a4df3080..a4e2f4a680 100644
--- a/apps/backend/internal/backendapp/main.go
+++ b/apps/backend/internal/backendapp/main.go
@@ -363,7 +363,8 @@ func startServices( //nolint:cyclop
// ============================================
// AGENTCTL LAUNCHER (for standalone mode)
// ============================================
- agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log)
+ agentRuntimeAvailability := agentctlclient.NewAvailability(eventBus, log)
+ agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log, agentRuntimeAvailability)
if err != nil {
log.Error("Failed to start agentctl subprocess", zap.Error(err))
return false
@@ -386,7 +387,8 @@ func startServices( //nolint:cyclop
agentctlBinaryPath = agentctlResult.binaryPath
}
- return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups)
+ return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, agentRuntimeAvailability,
+ dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups)
}
// startAgentInfrastructure initializes the agent lifecycle manager, worktree, orchestrator,
@@ -399,6 +401,7 @@ func startAgentInfrastructure(
log *logger.Logger,
addCleanup func(func() error),
eventBus bus.EventBus,
+ agentRuntimeAvailability *agentctlclient.Availability,
dbPool *db.Pool,
repos *Repositories,
services *Services,
@@ -643,7 +646,7 @@ func startAgentInfrastructure(
log.Info("Automation scheduler and evaluator started")
}
- return startGatewayAndServe(ctx, cfg, log, eventBus, dbPool, repos, services,
+ return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services,
agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, addCleanup, runCleanups)
}
@@ -656,6 +659,7 @@ func startGatewayAndServe(
cfg *config.Config,
log *logger.Logger,
eventBus bus.EventBus,
+ agentRuntimeAvailability *agentctlclient.Availability,
dbPool *db.Pool,
repos *Repositories,
services *Services,
@@ -842,13 +846,20 @@ func startGatewayAndServe(
systemSvc.StartBackground(ctx)
addCleanup(func() error { systemSvc.StopBackground(); return nil })
gateways.RegisterSystemNotifications(ctx, eventBus, gateway.Hub, log)
+ gateways.RegisterAgentRuntimeNotifications(ctx, eventBus, gateway.Hub, func() (any, bool) {
+ if agentRuntimeAvailability == nil {
+ return nil, false
+ }
+ snapshot, ok := agentRuntimeAvailability.Snapshot()
+ return snapshot, ok
+ }, log)
// ============================================
// HTTP SERVER
// ============================================
server, err := buildHTTPServer(cfg, log, gateway, repos, services, agentSettingsController,
lifecycleMgr, eventBus, orchestratorSvc, notificationCtrl, msgCreator, agentRegistry, hostUtilityMgr,
- addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer, dbPool)
+ addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer, dbPool, agentRuntimeAvailability)
if err != nil {
log.Error("Failed to build HTTP server", zap.Error(err))
return false
@@ -1750,6 +1761,7 @@ func buildHTTPServer(
systemSvc *systemsvc.Service,
workspaceRestorer taskhandlers.WorkspaceQuarantineRestorer,
dbPool *db.Pool,
+ agentRuntimeAvailability *agentctlclient.Availability,
) (*http.Server, error) {
gin.SetMode(gin.ReleaseMode)
router := gin.New()
@@ -1827,6 +1839,7 @@ func buildHTTPServer(
secretStore: userSecretStore,
mcpConfigSvc: mcpconfig.NewService(repos.AgentSettings),
authSvc: services.Auth,
+ agentRuntimeAvailability: agentRuntimeAvailability,
addCleanup: addCleanup,
repoCloner: repoCloner,
version: Version,
diff --git a/apps/backend/internal/backendapp/mentions.go b/apps/backend/internal/backendapp/mentions.go
index 305c02829f..7d9d57eb5c 100644
--- a/apps/backend/internal/backendapp/mentions.go
+++ b/apps/backend/internal/backendapp/mentions.go
@@ -10,6 +10,7 @@ import (
"github.com/gin-gonic/gin"
+ "github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/entityrefs"
"github.com/kandev/kandev/internal/gitlab"
"github.com/kandev/kandev/internal/mentions"
@@ -24,6 +25,7 @@ type MentionComponents struct {
Registry *mentions.Registry
Search mentions.Searcher
Submission entityrefs.SubmissionValidator
+ Log *logger.Logger
}
type mentionWorkspaceResolver interface {
@@ -35,6 +37,7 @@ type mentionConversationResolver interface {
}
func newMentionComponents(
+ log *logger.Logger,
workspaces mentionWorkspaceResolver,
conversations mentionConversationResolver,
providers ...mentions.MentionProvider,
@@ -47,12 +50,13 @@ func newMentionComponents(
}
search := &workspaceValidatingMentionSearcher{
workspaces: workspaces,
- searcher: mentions.NewService(registry),
+ searcher: mentions.NewService(registry, mentions.WithLogger(log)),
}
return &MentionComponents{
Registry: registry,
Search: search,
Submission: entityrefs.NewSubmissionService(conversations, registry),
+ Log: log,
}, nil
}
@@ -70,14 +74,14 @@ func (s *workspaceValidatingMentionSearcher) Search(
return nil, mentions.ErrInvalidRequest
}
if s == nil || s.workspaces == nil || s.searcher == nil {
- return nil, errors.New("mention search is unavailable")
+ return nil, mentions.NewSearchFailure(mentions.FailureStageSearch, mentions.FailureClassInternal, errors.New("mention search is unavailable"))
}
workspace, err := s.workspaces.GetWorkspace(ctx, request.WorkspaceID)
if err != nil {
if errors.Is(err, repository.ErrWorkspaceNotFound) {
return nil, mentions.ErrWorkspaceNotFound
}
- return nil, fmt.Errorf("validate mention workspace: %w", err)
+ return nil, mentions.NewSearchFailure(mentions.FailureStageWorkspaceValidation, mentions.FailureClassWorkspaceLookup, err)
}
if workspace == nil || workspace.ID != request.WorkspaceID {
return nil, mentions.ErrWorkspaceNotFound
@@ -298,5 +302,5 @@ func registerMentionRoutes(router gin.IRoutes, components *MentionComponents) {
if router == nil || components == nil || components.Search == nil {
return
}
- mentions.NewHandler(components.Search).RegisterRoutes(router)
+ mentions.NewHandler(components.Search, components.Log).RegisterRoutes(router)
}
diff --git a/apps/backend/internal/backendapp/mentions_test.go b/apps/backend/internal/backendapp/mentions_test.go
index fff8ce0bb4..b69ced31e3 100644
--- a/apps/backend/internal/backendapp/mentions_test.go
+++ b/apps/backend/internal/backendapp/mentions_test.go
@@ -4,16 +4,21 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
"time"
"github.com/gin-gonic/gin"
+ "github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/mentions"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository"
apiv1 "github.com/kandev/kandev/pkg/api/v1"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
)
type fakeMentionScopeResolver struct {
@@ -112,7 +117,7 @@ func TestMentionHTTPCompositionValidatesWorkspaceBeforeMixedProviderFanout(t *te
resolver := &fakeMentionScopeResolver{workspaces: map[string]*models.Workspace{
"workspace-1": {ID: "workspace-1", Name: "One"},
}}
- components, err := newMentionComponents(resolver, resolver, failure, timeout, success)
+ components, err := newMentionComponents(testLogger(t), resolver, resolver, failure, timeout, success)
if err != nil {
t.Fatalf("compose mentions: %v", err)
}
@@ -190,7 +195,7 @@ func TestMentionHTTPCompositionPropagatesRequestCancellation(t *testing.T) {
resolver := &fakeMentionScopeResolver{workspaces: map[string]*models.Workspace{
"workspace-1": {ID: "workspace-1", Name: "One"},
}}
- components, err := newMentionComponents(resolver, resolver, provider)
+ components, err := newMentionComponents(testLogger(t), resolver, resolver, provider)
if err != nil {
t.Fatalf("compose mentions: %v", err)
}
@@ -226,14 +231,55 @@ func TestMentionHTTPCompositionPropagatesRequestCancellation(t *testing.T) {
t.Fatal("provider did not observe request cancellation")
}
- if response.Code != http.StatusInternalServerError {
- t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ if response.Code != 499 {
+ t.Fatalf("status = %d, want 499; body = %s", response.Code, response.Body.String())
}
if !provider.canceled {
t.Fatal("provider did not observe request cancellation")
}
}
+func TestMentionHTTPCompositionLogsWorkspaceFailureOnceWithSafeFields(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ core, observed := observer.New(zap.DebugLevel)
+ log, err := logger.NewFromZap(zap.New(core))
+ if err != nil {
+ t.Fatalf("create observer logger: %v", err)
+ }
+ resolver := &fakeMentionScopeResolver{err: errors.New("secret database address")}
+ components, err := newMentionComponents(log, resolver, resolver)
+ if err != nil {
+ t.Fatalf("compose mentions: %v", err)
+ }
+ router := gin.New()
+ registerMentionRoutes(router, components)
+
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, httptest.NewRequest(
+ http.MethodGet,
+ "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query",
+ nil,
+ ))
+
+ if response.Code != http.StatusInternalServerError {
+ t.Fatalf("status = %d, want %d", response.Code, http.StatusInternalServerError)
+ }
+ entries := observed.All()
+ if len(entries) != 1 || entries[0].Level != zap.ErrorLevel {
+ t.Fatalf("workspace failure diagnostics = %#v", entries)
+ }
+ fields := entries[0].ContextMap()
+ if fields["workspace_id"] != "workspace-1" ||
+ fields["failure_stage"] != mentions.FailureStageWorkspaceValidation ||
+ fields["failure_class"] != mentions.FailureClassWorkspaceLookup {
+ t.Fatalf("workspace failure fields = %#v", fields)
+ }
+ if strings.Contains(fmt.Sprint(fields), "secret-query") ||
+ strings.Contains(fmt.Sprint(fields), "secret database address") {
+ t.Fatalf("workspace failure diagnostics leaked sensitive detail: %#v", fields)
+ }
+}
+
func TestBuiltinMentionProvidersRemainDescriptorDrivenWhenIntegrationsAreNil(t *testing.T) {
providers := builtinMentionProviders(&Services{}, nil)
want := []struct {
diff --git a/apps/backend/internal/backendapp/services.go b/apps/backend/internal/backendapp/services.go
index f32cb7c8e8..a19ddbfac2 100644
--- a/apps/backend/internal/backendapp/services.go
+++ b/apps/backend/internal/backendapp/services.go
@@ -212,6 +212,7 @@ func provideServices(cfg *config.Config, log *logger.Logger, repos *Repositories
Notification: nil,
}
mentionComponents, err := newMentionComponents(
+ log,
taskSvc,
taskSvc,
builtinMentionProviders(services, repos.Task)...,
diff --git a/apps/backend/internal/events/types.go b/apps/backend/internal/events/types.go
index 6ce6ec775a..09be4404bd 100644
--- a/apps/backend/internal/events/types.go
+++ b/apps/backend/internal/events/types.go
@@ -170,7 +170,8 @@ const PluginUserStateUpdated = "plugin.user-state.updated"
// create/restore, disk walk). Published by internal/system/jobs.Tracker on
// every state transition and broadcast to all WebSocket clients.
const (
- SystemJobUpdate = "system.job.update"
+ SystemJobUpdate = "system.job.update"
+ AgentRuntimeAvailabilityChanged = "system.agent_runtime.availability_changed"
)
// Event types for environments
diff --git a/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go b/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go
new file mode 100644
index 0000000000..517325fb37
--- /dev/null
+++ b/apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go
@@ -0,0 +1,93 @@
+package websocket
+
+import (
+ "context"
+ "encoding/json"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/kandev/kandev/internal/events"
+ "github.com/kandev/kandev/internal/events/bus"
+ ws "github.com/kandev/kandev/pkg/websocket"
+)
+
+func TestAgentRuntimeNotificationsBroadcastAndReplaySanitizedState(t *testing.T) {
+ hub := newTestHub(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ go hub.Run(ctx)
+
+ eventBus := bus.NewMemoryEventBus(testLoggerForUserNotifications(t))
+ t.Cleanup(eventBus.Close)
+ payload := map[string]any{"status": "unavailable", "reason": "agentctl_exited"}
+ broadcaster := RegisterAgentRuntimeNotifications(ctx, eventBus, hub, func() (any, bool) {
+ return payload, true
+ }, testLoggerForUserNotifications(t))
+ t.Cleanup(broadcaster.Close)
+
+ liveClient := newTestClient("live")
+ registerTestClient(hub, liveClient)
+ if err := eventBus.Publish(ctx, events.AgentRuntimeAvailabilityChanged,
+ bus.NewEvent(events.AgentRuntimeAvailabilityChanged, "test", payload)); err != nil {
+ t.Fatalf("publish availability event: %v", err)
+ }
+ liveMessage := readNotification(t, liveClient)
+ if liveMessage.Action != ws.ActionSystemAgentRuntimeStatusChanged {
+ t.Fatalf("live action = %q, want %q", liveMessage.Action, ws.ActionSystemAgentRuntimeStatusChanged)
+ }
+
+ replayClient := newTestClient("replay")
+ registerTestClient(hub, replayClient)
+ hub.SubscribeToUser(replayClient, "user-1")
+ replayMessage := readNotification(t, replayClient)
+ if replayMessage.Action != ws.ActionSystemAgentRuntimeStatusChanged {
+ t.Fatalf("replay action = %q, want %q", replayMessage.Action, ws.ActionSystemAgentRuntimeStatusChanged)
+ }
+ var replayPayload map[string]any
+ if err := json.Unmarshal(replayMessage.Payload, &replayPayload); err != nil {
+ t.Fatalf("decode replay payload: %v", err)
+ }
+ if replayPayload["status"] != "unavailable" || replayPayload["reason"] != "agentctl_exited" {
+ t.Fatalf("replay payload = %#v", replayPayload)
+ }
+}
+
+func TestSystemEventBroadcasterCloseIsIdempotent(t *testing.T) {
+ log := testLoggerForUserNotifications(t)
+ eventBus := bus.NewMemoryEventBus(log)
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ hub := newTestHub(t)
+ broadcaster := RegisterSystemNotifications(ctx, eventBus, hub, log)
+
+ var wg sync.WaitGroup
+ for index := 0; index < 16; index++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ broadcaster.Close()
+ }()
+ }
+ wg.Wait()
+
+ if len(broadcaster.subscriptions) != 0 {
+ t.Fatalf("subscriptions after concurrent Close = %d, want 0", len(broadcaster.subscriptions))
+ }
+ eventBus.Close()
+}
+
+func readNotification(t *testing.T, client *Client) ws.Message {
+ t.Helper()
+ select {
+ case frame := <-client.send:
+ var message ws.Message
+ if err := json.Unmarshal(frame, &message); err != nil {
+ t.Fatalf("decode websocket frame: %v", err)
+ }
+ return message
+ case <-time.After(time.Second):
+ t.Fatal("timed out waiting for websocket notification")
+ return ws.Message{}
+ }
+}
diff --git a/apps/backend/internal/gateway/websocket/system_notifications.go b/apps/backend/internal/gateway/websocket/system_notifications.go
index 91df4f4e50..e853a227bc 100644
--- a/apps/backend/internal/gateway/websocket/system_notifications.go
+++ b/apps/backend/internal/gateway/websocket/system_notifications.go
@@ -2,6 +2,7 @@ package websocket
import (
"context"
+ "sync"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
@@ -18,6 +19,7 @@ type SystemEventBroadcaster struct {
hub *Hub
subscriptions []bus.Subscription
logger *logger.Logger
+ closeOnce sync.Once
}
// RegisterSystemNotifications wires the system event broadcaster onto the
@@ -41,14 +43,56 @@ func RegisterSystemNotifications(ctx context.Context, eventBus bus.EventBus, hub
return b
}
+// RegisterAgentRuntimeNotifications forwards the install-wide agent runtime
+// state and replays the latest snapshot when a client subscribes to its user.
+// The snapshot provider keeps the gateway independent from the launcher
+// package and ensures replay uses the same sanitized state as boot hydration.
+func RegisterAgentRuntimeNotifications(
+ ctx context.Context,
+ eventBus bus.EventBus,
+ hub *Hub,
+ snapshotProvider func() (any, bool),
+ log *logger.Logger,
+) *SystemEventBroadcaster {
+ b := &SystemEventBroadcaster{
+ hub: hub,
+ logger: log.WithFields(zap.String("component", "ws-agent-runtime-broadcaster")),
+ }
+ if eventBus != nil {
+ b.subscribe(eventBus, events.AgentRuntimeAvailabilityChanged, ws.ActionSystemAgentRuntimeStatusChanged)
+ }
+ if snapshotProvider != nil {
+ hub.AddUserSubscriptionListener(func(userID string) {
+ payload, ok := snapshotProvider()
+ if !ok {
+ return
+ }
+ msg, err := ws.NewNotification(ws.ActionSystemAgentRuntimeStatusChanged, payload)
+ if err != nil {
+ b.logger.Error("failed to build agent runtime replay notification", zap.Error(err))
+ return
+ }
+ hub.BroadcastToUser(userID, msg)
+ })
+ }
+
+ go func() {
+ <-ctx.Done()
+ b.Close()
+ }()
+ return b
+}
+
// Close unsubscribes from all event-bus subscriptions.
func (b *SystemEventBroadcaster) Close() {
- for _, sub := range b.subscriptions {
- if sub != nil && sub.IsValid() {
- _ = sub.Unsubscribe()
+ b.closeOnce.Do(func() {
+ for _, sub := range b.subscriptions {
+ if sub != nil && sub.IsValid() {
+ _ = sub.Unsubscribe()
+ }
}
- }
- b.subscriptions = nil
+ b.subscriptions = nil
+ })
}
func (b *SystemEventBroadcaster) subscribe(eventBus bus.EventBus, subject, action string) {
@@ -58,6 +102,7 @@ func (b *SystemEventBroadcaster) subscribe(eventBus bus.EventBus, subject, actio
b.logger.Error("failed to build websocket notification", zap.String("action", action), zap.Error(err))
return nil
}
+ //ws:global install-wide system and runtime notifications.
b.hub.Broadcast(msg)
return nil
})
diff --git a/apps/backend/internal/mentions/handler.go b/apps/backend/internal/mentions/handler.go
index 6134e6335f..4c3c02d4ed 100644
--- a/apps/backend/internal/mentions/handler.go
+++ b/apps/backend/internal/mentions/handler.go
@@ -9,7 +9,9 @@ import (
"github.com/gin-gonic/gin"
+ "github.com/kandev/kandev/internal/common/logger"
apiv1 "github.com/kandev/kandev/pkg/api/v1"
+ "go.uber.org/zap"
)
const errorResponseField = "error"
@@ -22,10 +24,11 @@ type Searcher interface {
// Handler exposes mention search over HTTP.
type Handler struct {
searcher Searcher
+ log *logger.Logger
}
-func NewHandler(searcher Searcher) *Handler {
- return &Handler{searcher: searcher}
+func NewHandler(searcher Searcher, log *logger.Logger) *Handler {
+ return &Handler{searcher: searcher, log: log}
}
func (h *Handler) RegisterRoutes(router gin.IRoutes) {
@@ -35,7 +38,7 @@ func (h *Handler) RegisterRoutes(router gin.IRoutes) {
func (h *Handler) search(c *gin.Context) {
limit, err := parseLimit(c.Query("limit"))
if err != nil {
- writeSearchError(c, ErrInvalidRequest)
+ h.writeSearchError(c, ErrInvalidRequest)
return
}
response, err := h.searcher.Search(c.Request.Context(), SearchRequest{
@@ -44,7 +47,7 @@ func (h *Handler) search(c *gin.Context) {
Limit: limit,
})
if err != nil {
- writeSearchError(c, err)
+ h.writeSearchError(c, err)
return
}
c.JSON(http.StatusOK, response)
@@ -68,13 +71,29 @@ func parseLimit(rawLimit string) (int, error) {
return limit, nil
}
-func writeSearchError(c *gin.Context, err error) {
+func (h *Handler) writeSearchError(c *gin.Context, err error) {
switch {
case errors.Is(err, ErrInvalidRequest):
c.JSON(http.StatusBadRequest, gin.H{errorResponseField: "invalid mention search request"})
case errors.Is(err, ErrWorkspaceNotFound):
c.JSON(http.StatusNotFound, gin.H{errorResponseField: "mention search workspace not found"})
+ case errors.Is(err, context.Canceled):
+ c.JSON(499, gin.H{errorResponseField: "mention search request canceled"})
default:
+ h.logUnexpectedFailure(c, err)
c.JSON(http.StatusInternalServerError, gin.H{errorResponseField: "mention search failed"})
}
}
+
+func (h *Handler) logUnexpectedFailure(c *gin.Context, err error) {
+ if h.log == nil {
+ return
+ }
+ stage, class := SearchFailureMetadata(err)
+ h.log.Error(
+ "mention search failed",
+ zap.String("workspace_id", c.Param("id")),
+ zap.String("failure_stage", stage),
+ zap.String("failure_class", class),
+ )
+}
diff --git a/apps/backend/internal/mentions/handler_test.go b/apps/backend/internal/mentions/handler_test.go
index b167afa283..422a609529 100644
--- a/apps/backend/internal/mentions/handler_test.go
+++ b/apps/backend/internal/mentions/handler_test.go
@@ -11,8 +11,11 @@ import (
"testing"
"github.com/gin-gonic/gin"
+ "github.com/kandev/kandev/internal/common/logger"
apiv1 "github.com/kandev/kandev/pkg/api/v1"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
)
type fakeSearcher struct {
@@ -25,7 +28,7 @@ func TestHandlerRegisterRoutesUsesExistingWorkspaceWildcardName(t *testing.T) {
router.GET("/api/v1/workspaces/:id", func(c *gin.Context) { c.Status(http.StatusNoContent) })
handler := NewHandler(fakeSearcher{search: func(_ context.Context, request SearchRequest) (*apiv1.MentionSearchResponse, error) {
return &apiv1.MentionSearchResponse{Query: request.Query}, nil
- }})
+ }}, nil)
var recovered any
func() {
@@ -56,7 +59,7 @@ func TestHandlerSearch_ReturnsNormalizedResponse(t *testing.T) {
Results: []apiv1.EntityReference{},
}},
}, nil
- }}).RegisterRoutes(router)
+ }}, nil).RegisterRoutes(router)
request := httptest.NewRequest(http.MethodGet,
"/api/v1/workspaces/workspace-1/mentions/search?q=auth&limit=99", nil)
@@ -81,7 +84,7 @@ func TestHandlerSearch_RejectsInvalidLimitWithoutSearching(t *testing.T) {
NewHandler(fakeSearcher{search: func(_ context.Context, _ SearchRequest) (*apiv1.MentionSearchResponse, error) {
t.Fatal("search called for invalid limit")
return nil, nil
- }}).RegisterRoutes(router)
+ }}, nil).RegisterRoutes(router)
request := httptest.NewRequest(http.MethodGet,
"/api/v1/workspaces/workspace-1/mentions/search?q=auth&limit=not-a-number", nil)
@@ -110,7 +113,7 @@ func TestHandlerSearch_MapsSafeErrorsWithoutLeaking(t *testing.T) {
router := gin.New()
NewHandler(fakeSearcher{search: func(_ context.Context, _ SearchRequest) (*apiv1.MentionSearchResponse, error) {
return nil, test.err
- }}).RegisterRoutes(router)
+ }}, nil).RegisterRoutes(router)
request := httptest.NewRequest(http.MethodGet,
"/api/v1/workspaces/workspace-1/mentions/search?q=auth", nil)
@@ -126,3 +129,69 @@ func TestHandlerSearch_MapsSafeErrorsWithoutLeaking(t *testing.T) {
})
}
}
+
+func TestHandlerSearch_MapsClientCancellationTo499WithoutLogging(t *testing.T) {
+ log, observed := newMentionObserverLogger(t)
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ NewHandler(fakeSearcher{search: func(context.Context, SearchRequest) (*apiv1.MentionSearchResponse, error) {
+ return nil, context.Canceled
+ }}, log).RegisterRoutes(router)
+
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, httptest.NewRequest(
+ http.MethodGet,
+ "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query",
+ nil,
+ ))
+
+ if response.Code != 499 {
+ t.Fatalf("status = %d, want 499; body = %s", response.Code, response.Body.String())
+ }
+ if observed.Len() != 0 {
+ t.Fatalf("cancellation emitted %d log records, want none", observed.Len())
+ }
+}
+
+func TestHandlerSearchLogsUnexpectedFailureOnceWithSafeFields(t *testing.T) {
+ log, observed := newMentionObserverLogger(t)
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ NewHandler(fakeSearcher{search: func(context.Context, SearchRequest) (*apiv1.MentionSearchResponse, error) {
+ return nil, errors.New("secret database address")
+ }}, log).RegisterRoutes(router)
+
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, httptest.NewRequest(
+ http.MethodGet,
+ "/api/v1/workspaces/workspace-1/mentions/search?q=secret-query",
+ nil,
+ ))
+
+ if response.Code != http.StatusInternalServerError {
+ t.Fatalf("status = %d, want %d", response.Code, http.StatusInternalServerError)
+ }
+ entries := observed.All()
+ if len(entries) != 1 || entries[0].Level != zap.ErrorLevel {
+ t.Fatalf("unexpected error records = %#v", entries)
+ }
+ fields := entries[0].ContextMap()
+ if fields["workspace_id"] != "workspace-1" || fields["failure_stage"] != "search" ||
+ fields["failure_class"] != "internal" {
+ t.Fatalf("safe failure fields = %#v", fields)
+ }
+ serialized := entries[0].Message + response.Body.String() + fmt.Sprint(entries[0].ContextMap())
+ if strings.Contains(serialized, "secret-query") || strings.Contains(serialized, "secret database address") {
+ t.Fatalf("sensitive mention-search detail leaked: %s", serialized)
+ }
+}
+
+func newMentionObserverLogger(t *testing.T) (*logger.Logger, *observer.ObservedLogs) {
+ t.Helper()
+ core, observed := observer.New(zap.DebugLevel)
+ log, err := logger.NewFromZap(zap.New(core))
+ if err != nil {
+ t.Fatalf("create observer logger: %v", err)
+ }
+ return log, observed
+}
diff --git a/apps/backend/internal/mentions/service.go b/apps/backend/internal/mentions/service.go
index 69007f506f..5daf9731d5 100644
--- a/apps/backend/internal/mentions/service.go
+++ b/apps/backend/internal/mentions/service.go
@@ -13,8 +13,10 @@ import (
"unicode"
"unicode/utf8"
+ "github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/entityrefs"
apiv1 "github.com/kandev/kandev/pkg/api/v1"
+ "go.uber.org/zap"
)
type registeredProvider struct {
@@ -155,6 +157,7 @@ func (r *Registry) snapshot() []registeredProvider {
// Service validates and aggregates every registered provider.
type Service struct {
registry *Registry
+ log *logger.Logger
providerTimeout time.Duration
maxConcurrency int
providerSlots chan struct{}
@@ -186,6 +189,12 @@ func WithMaxConcurrency(limit int) Option {
}
}
+func WithLogger(log *logger.Logger) Option {
+ return func(service *Service) {
+ service.log = log
+ }
+}
+
func NewService(registry *Registry, options ...Option) *Service {
service := &Service{
registry: registry,
@@ -274,7 +283,7 @@ func (s *Service) searchProvider(ctx context.Context, registered registeredProvi
defer cancelProvider()
result := make(chan apiv1.MentionGroup, 1)
go func() {
- group := searchProviderWithinContext(providerCtx, registered, request)
+ group := searchProviderWithinContext(providerCtx, registered, request, s.log)
// The provider slot belongs to the underlying call. A provider that ignores
// cancellation stays quarantined without consuming shared search capacity.
<-providerSlot
@@ -308,11 +317,22 @@ func searchProviderWithinContext(
ctx context.Context,
registered registeredProvider,
request SearchRequest,
+ log *logger.Logger,
) apiv1.MentionGroup {
group := groupFromDescriptor(registered.descriptor)
candidates, err := registered.provider.Search(ctx, request)
if err != nil {
- group.Status = classifyProviderError(err)
+ status := classifyProviderError(err)
+ if log != nil && !errors.Is(err, context.Canceled) {
+ log.Warn(
+ "mention provider search failed",
+ zap.String("source", registered.descriptor.Source),
+ zap.String("provider", registered.descriptor.Provider),
+ zap.String("kind", registered.descriptor.Kind),
+ zap.String("status", string(status)),
+ )
+ }
+ group.Status = status
return group
}
if ctx.Err() != nil {
diff --git a/apps/backend/internal/mentions/service_test.go b/apps/backend/internal/mentions/service_test.go
index 14afb2a9fd..bec76fcbaa 100644
--- a/apps/backend/internal/mentions/service_test.go
+++ b/apps/backend/internal/mentions/service_test.go
@@ -9,7 +9,10 @@ import (
"testing"
"time"
+ "github.com/kandev/kandev/internal/common/logger"
apiv1 "github.com/kandev/kandev/pkg/api/v1"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
)
type fakeProvider struct {
@@ -96,6 +99,50 @@ func TestServiceSearch_RegistryOwnsNamespacedProviderIdentity(t *testing.T) {
}
}
+func TestServiceSearchLogsProviderFailureWithSafeIdentity(t *testing.T) {
+ core, observed := observer.New(zap.DebugLevel)
+ log, err := logger.NewFromZap(zap.New(core))
+ if err != nil {
+ t.Fatalf("create observer logger: %v", err)
+ }
+ registry := NewRegistry()
+ if err := registry.Register(fakeProvider{
+ descriptor: ProviderDescriptor{
+ Source: "acme_issues", Provider: "acme", Kind: "issue",
+ DisplayName: "Acme issues", KindLabel: "Issue",
+ },
+ search: func(context.Context, SearchRequest) ([]Candidate, error) {
+ return nil, NewProviderError(StatusUnauthorized, errors.New("secret upstream detail"))
+ },
+ }); err != nil {
+ t.Fatalf("register provider: %v", err)
+ }
+
+ response, err := NewService(registry, WithLogger(log)).Search(context.Background(), SearchRequest{
+ WorkspaceID: "workspace-1",
+ Query: "secret-query",
+ })
+ if err != nil {
+ t.Fatalf("search: %v", err)
+ }
+ if response.Groups[0].Status != StatusUnauthorized {
+ t.Fatalf("provider status = %q, want %q", response.Groups[0].Status, StatusUnauthorized)
+ }
+ entries := observed.All()
+ if len(entries) != 1 || entries[0].Level != zap.WarnLevel {
+ t.Fatalf("provider diagnostics = %#v", entries)
+ }
+ fields := entries[0].ContextMap()
+ if fields["source"] != "acme_issues" || fields["provider"] != "acme" ||
+ fields["kind"] != "issue" || fields["status"] != string(StatusUnauthorized) {
+ t.Fatalf("provider diagnostic fields = %#v", fields)
+ }
+ if strings.Contains(fmt.Sprint(fields), "secret-query") ||
+ strings.Contains(fmt.Sprint(fields), "secret upstream detail") {
+ t.Fatalf("provider diagnostic leaked sensitive detail: %#v", fields)
+ }
+}
+
func TestRegistryRegister_RejectsUnsafeAndDuplicateSources(t *testing.T) {
provider := func(source string) fakeProvider {
return fakeProvider{
diff --git a/apps/backend/internal/mentions/types.go b/apps/backend/internal/mentions/types.go
index 2b237e4047..d79d990ddd 100644
--- a/apps/backend/internal/mentions/types.go
+++ b/apps/backend/internal/mentions/types.go
@@ -31,6 +31,55 @@ var (
type Status = apiv1.MentionStatus
+const (
+ FailureStageSearch = "search"
+ FailureStageWorkspaceValidation = "workspace_validation"
+ FailureClassInternal = "internal"
+ FailureClassWorkspaceLookup = "workspace_lookup"
+)
+
+// SearchFailure carries safe classification metadata across the composition
+// boundary without requiring handlers to inspect an implementation error.
+type SearchFailure struct {
+ stage string
+ class string
+ cause error
+}
+
+func (e *SearchFailure) Error() string {
+ if e == nil || e.cause == nil {
+ return "mention search failed"
+ }
+ return e.cause.Error()
+}
+
+func (e *SearchFailure) Unwrap() error {
+ if e == nil {
+ return nil
+ }
+ return e.cause
+}
+
+// NewSearchFailure attaches stable, non-sensitive diagnostic classification.
+func NewSearchFailure(stage, class string, cause error) error {
+ if stage == "" {
+ stage = FailureStageSearch
+ }
+ if class == "" {
+ class = FailureClassInternal
+ }
+ return &SearchFailure{stage: stage, class: class, cause: cause}
+}
+
+// SearchFailureMetadata returns safe fields for structured HTTP diagnostics.
+func SearchFailureMetadata(err error) (stage, class string) {
+ var failure *SearchFailure
+ if errors.As(err, &failure) && failure != nil {
+ return failure.stage, failure.class
+ }
+ return FailureStageSearch, FailureClassInternal
+}
+
const (
StatusOK = apiv1.MentionStatusOK
StatusNotConfigured = apiv1.MentionStatusNotConfigured
diff --git a/apps/backend/internal/office/costs/modelsdev/client.go b/apps/backend/internal/office/costs/modelsdev/client.go
index 94d6ec1636..8528b48301 100644
--- a/apps/backend/internal/office/costs/modelsdev/client.go
+++ b/apps/backend/internal/office/costs/modelsdev/client.go
@@ -15,6 +15,7 @@ import (
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/office/shared"
+ "golang.org/x/sync/singleflight"
"go.uber.org/zap"
)
@@ -43,7 +44,8 @@ type Client struct {
httpClient *http.Client
logger *logger.Logger
- once sync.Once
+ once sync.Once
+ refreshGroup singleflight.Group
mu sync.RWMutex
index map[string]shared.ModelPricing
@@ -162,7 +164,7 @@ func (c *Client) warmFromDisk(ctx context.Context) {
}
// File missing on first boot — schedule a refresh; lookup
// returns miss this turn.
- go c.refreshSafe(ctx)
+ c.startBackgroundRefresh(ctx)
return
}
buf, err := os.ReadFile(c.cachePath)
@@ -176,7 +178,7 @@ func (c *Client) warmFromDisk(ctx context.Context) {
c.loadedAt = stat.ModTime()
c.mu.Unlock()
if time.Since(stat.ModTime()) >= c.ttl {
- go c.refreshSafe(ctx)
+ c.startBackgroundRefresh(context.WithoutCancel(ctx))
}
}
@@ -220,35 +222,56 @@ func (c *Client) parseModelInfoFromBuffer(key string) (ModelInfo, bool) {
return info, true
}
-// maybeRefresh fires a background refresh when the loaded buffer is
-// stale. No-op when refresh is in progress (sync.Once-style guard
-// implicitly enforced by atomicity of the staleness check + ttl).
+// maybeRefresh fires a background refresh when the loaded buffer is stale.
+// Refresh's per-client singleflight guard coalesces all callers, including
+// concurrent pricing and metadata lookups.
func (c *Client) maybeRefresh(ctx context.Context) {
c.mu.RLock()
stale := c.loadedAt.IsZero() || time.Since(c.loadedAt) >= c.ttl
c.mu.RUnlock()
if stale {
- go c.refreshSafe(ctx)
+ c.startBackgroundRefresh(context.WithoutCancel(ctx))
}
}
-// refreshSafe wraps Refresh in a panic guard + warning log; safe to
-// run from a goroutine.
-func (c *Client) refreshSafe(ctx context.Context) {
- defer func() {
- if r := recover(); r != nil {
- c.logger.Warn("models.dev refresh panicked", zap.Any("recover", r))
+// startBackgroundRefresh registers the refresh synchronously so every stale
+// lookup joins the same in-flight singleflight call before returning. The
+// result is still observed asynchronously, so stale lookups remain nonblocking.
+// Callers choose whether the refresh should inherit or detach from their context.
+func (c *Client) startBackgroundRefresh(ctx context.Context) {
+ result := c.refreshResult(ctx)
+ go func() {
+ if err := (<-result).Err; err != nil {
+ c.logger.Warn("models.dev refresh failed", zap.Error(err))
}
}()
- if err := c.Refresh(ctx); err != nil {
- c.logger.Warn("models.dev refresh failed", zap.Error(err))
- }
}
// Refresh pulls the latest dataset from models.dev and atomically
// swaps the cache file. Network or write errors leave the existing
// file (and in-memory index) untouched.
func (c *Client) Refresh(ctx context.Context) error {
+ result := c.refreshResult(ctx)
+ return (<-result).Err
+}
+
+func (c *Client) refreshResult(ctx context.Context) <-chan singleflight.Result {
+ return c.refreshGroup.DoChan("modelsdev-refresh", func() (interface{}, error) {
+ return c.refreshPhysicalSafely(ctx)
+ })
+}
+
+func (c *Client) refreshPhysicalSafely(ctx context.Context) (value interface{}, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ c.logger.Warn("models.dev refresh panicked", zap.Any("recover", recovered))
+ err = fmt.Errorf("models.dev refresh panicked")
+ }
+ }()
+ return nil, c.refreshPhysical(ctx)
+}
+
+func (c *Client) refreshPhysical(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, nil)
if err != nil {
return fmt.Errorf("build request: %w", err)
@@ -271,24 +294,22 @@ func (c *Client) Refresh(ctx context.Context) error {
}
c.mu.Lock()
- c.cacheBuf = buf
- c.loadedAt = time.Now()
- // Re-populate every entry already in the in-memory index so a
- // stale price doesn't survive the refresh.
+ newIndex := make(map[string]shared.ModelPricing, len(c.index))
for k := range c.index {
if pricing, ok := lookupInDataset(buf, k); ok {
- c.index[k] = pricing
- } else {
- delete(c.index, k)
+ newIndex[k] = pricing
}
}
+ newInfo := make(map[string]ModelInfo, len(c.info))
for k := range c.info {
if info, ok := lookupModelInfoInDataset(buf, k); ok {
- c.info[k] = info
- } else {
- delete(c.info, k)
+ newInfo[k] = info
}
}
+ c.cacheBuf = buf
+ c.index = newIndex
+ c.info = newInfo
+ c.loadedAt = time.Now().UTC()
c.mu.Unlock()
return nil
}
@@ -300,11 +321,27 @@ func (c *Client) writeCacheAtomic(buf []byte) error {
if err := os.MkdirAll(filepath.Dir(c.cachePath), 0o755); err != nil {
return fmt.Errorf("mkdir cache dir: %w", err)
}
- tmp := c.cachePath + ".tmp"
- if err := os.WriteFile(tmp, buf, 0o644); err != nil {
- return fmt.Errorf("write cache tmp: %w", err)
+ tmp, err := os.CreateTemp(filepath.Dir(c.cachePath), "."+filepath.Base(c.cachePath)+".tmp-*")
+ if err != nil {
+ return fmt.Errorf("create cache tmp: %w", err)
}
- if err := os.Rename(tmp, c.cachePath); err != nil {
+ tmpName := tmp.Name()
+ defer func() {
+ _ = tmp.Close()
+ _ = os.Remove(tmpName)
+ }()
+ if written, err := tmp.Write(buf); err != nil {
+ return fmt.Errorf("write cache tmp: %w", err)
+ } else if written != len(buf) {
+ return fmt.Errorf("write cache tmp: %w", io.ErrShortWrite)
+ }
+ if err := tmp.Sync(); err != nil {
+ return fmt.Errorf("sync cache tmp: %w", err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("close cache tmp: %w", err)
+ }
+ if err := os.Rename(tmpName, c.cachePath); err != nil {
return fmt.Errorf("rename cache: %w", err)
}
return nil
diff --git a/apps/backend/internal/office/costs/modelsdev/client_test.go b/apps/backend/internal/office/costs/modelsdev/client_test.go
index 2e4610693c..b3f1c53666 100644
--- a/apps/backend/internal/office/costs/modelsdev/client_test.go
+++ b/apps/backend/internal/office/costs/modelsdev/client_test.go
@@ -6,6 +6,8 @@ import (
"net/http/httptest"
"os"
"path/filepath"
+ "sync"
+ "sync/atomic"
"testing"
"time"
@@ -236,3 +238,289 @@ func TestClient_FirstBootMissesGracefully(t *testing.T) {
t.Error("expected miss on cold-boot lookup")
}
}
+
+type requestGate struct {
+ server *httptest.Server
+ started chan struct{}
+ release chan struct{}
+ requests atomic.Int32
+}
+
+func newRequestGate(t *testing.T, body string) *requestGate {
+ t.Helper()
+ gate := &requestGate{
+ started: make(chan struct{}),
+ release: make(chan struct{}),
+ }
+ gate.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ count := gate.requests.Add(1)
+ if count == 1 {
+ close(gate.started)
+ }
+ select {
+ case <-gate.release:
+ case <-r.Context().Done():
+ return
+ }
+ _, _ = w.Write([]byte(body))
+ }))
+ t.Cleanup(gate.server.Close)
+ return gate
+}
+
+func (g *requestGate) waitForFirstRequest(t *testing.T) {
+ t.Helper()
+ select {
+ case <-g.started:
+ case <-time.After(time.Second):
+ t.Fatal("timed out waiting for the first models.dev request")
+ }
+}
+
+func TestClient_ConcurrentRefreshCallsShareOneFetch(t *testing.T) {
+ dir := t.TempDir()
+ cachePath := filepath.Join(dir, "models-dev.json")
+ gate := newRequestGate(t, sampleDataset)
+ c := modelsdev.New(modelsdev.Config{
+ CachePath: cachePath,
+ URL: gate.server.URL,
+ TTL: time.Hour,
+ HTTPClient: gate.server.Client(),
+ }, logger.Default())
+
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ for index := 0; index < 8; index++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ if err := c.Refresh(context.Background()); err != nil {
+ t.Errorf("Refresh: %v", err)
+ }
+ }()
+ }
+ close(start)
+ gate.waitForFirstRequest(t)
+ close(gate.release)
+ wg.Wait()
+
+ if got := gate.requests.Load(); got != 1 {
+ t.Fatalf("models.dev request count = %d, want 1", got)
+ }
+}
+
+func TestClient_ConcurrentLookupsShareOneBackgroundFetch(t *testing.T) {
+ dir := t.TempDir()
+ cachePath := filepath.Join(dir, "models-dev.json")
+ if err := os.WriteFile(cachePath, []byte(sampleDataset), 0o644); err != nil {
+ t.Fatalf("seed cache: %v", err)
+ }
+ old := time.Now().Add(-time.Hour)
+ if err := os.Chtimes(cachePath, old, old); err != nil {
+ t.Fatalf("age cache: %v", err)
+ }
+ gate := newRequestGate(t, sampleDataset)
+ c := modelsdev.New(modelsdev.Config{
+ CachePath: cachePath,
+ URL: gate.server.URL,
+ TTL: time.Millisecond,
+ HTTPClient: gate.server.Client(),
+ }, logger.Default())
+
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ for index := 0; index < 12; index++ {
+ wg.Add(1)
+ go func(index int) {
+ defer wg.Done()
+ <-start
+ if index%2 == 0 {
+ _, _ = c.LookupForModel(context.Background(), "gpt-5.3-codex-spark")
+ return
+ }
+ _, _ = c.LookupModelInfo(context.Background(), "gpt-5.3-codex-spark")
+ }(index)
+ }
+ close(start)
+ wg.Wait()
+ gate.waitForFirstRequest(t)
+ if got := gate.requests.Load(); got != 1 {
+ t.Fatalf("background models.dev request count before release = %d, want 1", got)
+ }
+ refreshDone := make(chan error, 1)
+ go func() { refreshDone <- c.Refresh(context.Background()) }()
+ close(gate.release)
+ select {
+ case err := <-refreshDone:
+ if err != nil {
+ t.Fatalf("joined background Refresh: %v", err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("joined background Refresh did not complete")
+ }
+ if got := gate.requests.Load(); got != 1 {
+ t.Fatalf("background models.dev request count = %d, want 1", got)
+ }
+}
+
+func TestClient_CanceledStaleLookupDoesNotCancelJoinedRefresh(t *testing.T) {
+ dir := t.TempDir()
+ cachePath := filepath.Join(dir, "models-dev.json")
+ if err := os.WriteFile(cachePath, []byte(sampleDataset), 0o644); err != nil {
+ t.Fatalf("seed cache: %v", err)
+ }
+ old := time.Now().Add(-time.Hour)
+ if err := os.Chtimes(cachePath, old, old); err != nil {
+ t.Fatalf("age cache: %v", err)
+ }
+ gate := newRequestGate(t, sampleDataset)
+ c := modelsdev.New(modelsdev.Config{
+ CachePath: cachePath,
+ URL: gate.server.URL,
+ TTL: time.Millisecond,
+ HTTPClient: gate.server.Client(),
+ }, logger.Default())
+
+ lookupCtx, cancelLookup := context.WithCancel(context.Background())
+ _, _ = c.LookupForModel(lookupCtx, "gpt-5.3-codex-spark")
+ gate.waitForFirstRequest(t)
+ cancelLookup()
+
+ refreshDone := make(chan error, 1)
+ go func() { refreshDone <- c.Refresh(context.Background()) }()
+ select {
+ case err := <-refreshDone:
+ t.Fatalf("joined refresh returned before the shared request completed: %v", err)
+ case <-time.After(25 * time.Millisecond):
+ }
+
+ close(gate.release)
+ select {
+ case err := <-refreshDone:
+ if err != nil {
+ t.Fatalf("joined Refresh: %v", err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("joined Refresh did not complete")
+ }
+ if got := gate.requests.Load(); got != 1 {
+ t.Fatalf("models.dev request count after canceled lookup = %d, want 1", got)
+ }
+}
+
+func TestClient_FailedRefreshPreservesCacheAndAllowsRetry(t *testing.T) {
+ dir := t.TempDir()
+ cachePath := filepath.Join(dir, "models-dev.json")
+ var fail atomic.Bool
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests.Add(1)
+ if fail.Load() {
+ http.Error(w, "upstream unavailable", http.StatusBadGateway)
+ return
+ }
+ _, _ = w.Write([]byte(sampleDataset))
+ }))
+ t.Cleanup(server.Close)
+ c := modelsdev.New(modelsdev.Config{
+ CachePath: cachePath,
+ URL: server.URL,
+ TTL: time.Hour,
+ HTTPClient: server.Client(),
+ }, logger.Default())
+
+ if err := c.Refresh(context.Background()); err != nil {
+ t.Fatalf("initial Refresh: %v", err)
+ }
+ before, err := os.ReadFile(cachePath)
+ if err != nil {
+ t.Fatalf("read initial cache: %v", err)
+ }
+ if _, ok := c.LookupForModel(context.Background(), "claude-opus-4-7"); !ok {
+ t.Fatal("initial cache lookup failed")
+ }
+
+ fail.Store(true)
+ if err := c.Refresh(context.Background()); err == nil {
+ t.Fatal("failed Refresh unexpectedly succeeded")
+ }
+ after, err := os.ReadFile(cachePath)
+ if err != nil {
+ t.Fatalf("read cache after failed refresh: %v", err)
+ }
+ if string(after) != string(before) {
+ t.Fatal("failed refresh replaced the valid cache")
+ }
+ if _, ok := c.LookupForModel(context.Background(), "claude-opus-4-7"); !ok {
+ t.Fatal("failed refresh discarded valid in-memory data")
+ }
+
+ fail.Store(false)
+ if err := c.Refresh(context.Background()); err != nil {
+ t.Fatalf("retry Refresh: %v", err)
+ }
+ if got := requests.Load(); got != 3 {
+ t.Fatalf("request count after failed retry = %d, want 3", got)
+ }
+ assertNoCacheTemps(t, cachePath)
+}
+
+func TestClient_CanceledRefreshAllowsRetryAndCleansTemporaryFile(t *testing.T) {
+ dir := t.TempDir()
+ cachePath := filepath.Join(dir, "models-dev.json")
+ firstStarted := make(chan struct{})
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if requests.Add(1) == 1 {
+ close(firstStarted)
+ <-r.Context().Done()
+ return
+ }
+ _, _ = w.Write([]byte(sampleDataset))
+ }))
+ t.Cleanup(server.Close)
+ c := modelsdev.New(modelsdev.Config{
+ CachePath: cachePath,
+ URL: server.URL,
+ TTL: time.Hour,
+ HTTPClient: server.Client(),
+ }, logger.Default())
+
+ ctx, cancel := context.WithCancel(context.Background())
+ refreshDone := make(chan error, 1)
+ go func() { refreshDone <- c.Refresh(ctx) }()
+ select {
+ case <-firstStarted:
+ case <-time.After(time.Second):
+ t.Fatal("timed out waiting for cancelable refresh")
+ }
+ cancel()
+ select {
+ case err := <-refreshDone:
+ if err == nil {
+ t.Fatal("canceled Refresh unexpectedly succeeded")
+ }
+ case <-time.After(time.Second):
+ t.Fatal("canceled Refresh did not return")
+ }
+
+ if err := c.Refresh(context.Background()); err != nil {
+ t.Fatalf("retry Refresh: %v", err)
+ }
+ if got := requests.Load(); got != 2 {
+ t.Fatalf("request count after canceled retry = %d, want 2", got)
+ }
+ assertNoCacheTemps(t, cachePath)
+}
+
+func assertNoCacheTemps(t *testing.T, cachePath string) {
+ t.Helper()
+ matches, err := filepath.Glob(filepath.Join(filepath.Dir(cachePath), "."+filepath.Base(cachePath)+".tmp-*"))
+ if err != nil {
+ t.Fatalf("glob cache temps: %v", err)
+ }
+ if len(matches) != 0 {
+ t.Fatalf("cache temporary files remain: %v", matches)
+ }
+}
diff --git a/apps/backend/internal/orchestrator/reconcile_liveness.go b/apps/backend/internal/orchestrator/reconcile_liveness.go
index aca749b350..c9295a0cec 100644
--- a/apps/backend/internal/orchestrator/reconcile_liveness.go
+++ b/apps/backend/internal/orchestrator/reconcile_liveness.go
@@ -3,9 +3,13 @@ package orchestrator
import (
"context"
"errors"
+ "fmt"
"go.uber.org/zap"
+ runtimeapi "github.com/kandev/kandev/internal/agent/runtime"
+ "github.com/kandev/kandev/internal/common/logger"
+ "github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/task/models"
)
@@ -20,6 +24,114 @@ type rowLivenessProber interface {
RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness
}
+type startupCleanupDisposition string
+
+const (
+ startupCleanupDispositionStopped startupCleanupDisposition = "stopped"
+ startupCleanupDispositionAlreadyAbsent startupCleanupDisposition = "already_absent"
+ startupCleanupDispositionPreserved startupCleanupDisposition = "preserved"
+)
+
+type startupCleanupDecision struct {
+ disposition startupCleanupDisposition
+ liveness string
+ stopErrorClass string
+ proceed bool
+ expected bool
+}
+
+// classifyStartupCleanup is the single decision matrix for startup stop
+// results. Only a typed runtime absence plus confirmed-dead local liveness
+// permits cleanup to continue; every uncertain result remains preserved.
+func classifyStartupCleanup(err error, liveness models.ProcessLiveness) startupCleanupDecision {
+ decision := startupCleanupDecision{
+ disposition: startupCleanupDispositionPreserved,
+ liveness: processLivenessClass(liveness),
+ stopErrorClass: "none",
+ }
+ if err == nil {
+ decision.disposition = startupCleanupDispositionStopped
+ decision.proceed = true
+ return decision
+ }
+ if stopReportsRuntimeAbsent(err) {
+ decision.stopErrorClass = "runtime_not_found"
+ if liveness == models.ProcessLivenessDead {
+ decision.disposition = startupCleanupDispositionAlreadyAbsent
+ decision.proceed = true
+ return decision
+ }
+ decision.expected = true
+ return decision
+ }
+ decision.stopErrorClass = "unexpected"
+ return decision
+}
+
+func processLivenessClass(liveness models.ProcessLiveness) string {
+ switch liveness {
+ case models.ProcessLivenessAlive:
+ return "alive"
+ case models.ProcessLivenessDead:
+ return "dead"
+ default:
+ return "unknown"
+ }
+}
+
+type startupCleanupReport struct {
+ expectedPreservedCount int
+ expectedOutcomes map[string]int
+}
+
+func newStartupCleanupReport() *startupCleanupReport {
+ return &startupCleanupReport{expectedOutcomes: make(map[string]int)}
+}
+
+func (r *startupCleanupReport) recordExpected(running *models.ExecutorRunning, decision startupCleanupDecision) {
+ if r == nil {
+ return
+ }
+ key := fmt.Sprintf("disposition=%s,liveness=%s,stop_error=%s,local_pid=%t",
+ decision.disposition, decision.liveness, decision.stopErrorClass,
+ running != nil && running.LocalPID > 0)
+ r.expectedPreservedCount++
+ r.expectedOutcomes[key]++
+}
+
+func (r *startupCleanupReport) flush(log *logger.Logger) {
+ if r == nil || r.expectedPreservedCount == 0 {
+ return
+ }
+ log.Warn("startup runtime cleanup summary",
+ zap.Int("expected_preserved_count", r.expectedPreservedCount),
+ zap.Any("expected_preserved_outcomes", r.expectedOutcomes))
+}
+
+func startupCleanupFields(running *models.ExecutorRunning, decision startupCleanupDecision) []zap.Field {
+ fields := []zap.Field{
+ zap.String("liveness", decision.liveness),
+ zap.String("stop_error_class", decision.stopErrorClass),
+ zap.String("disposition", string(decision.disposition)),
+ zap.Bool("local_pid_present", running != nil && running.LocalPID > 0),
+ }
+ if running == nil {
+ return fields
+ }
+ return append(fields,
+ zap.String("session_id", running.SessionID),
+ zap.String("task_id", running.TaskID),
+ zap.String("execution_id", running.AgentExecutionID),
+ )
+}
+
+// stopReportsRuntimeAbsent accepts only typed runtime/executor absence
+// sentinels. Generic lookup errors remain retryable failures.
+func stopReportsRuntimeAbsent(err error) bool {
+ return errors.Is(err, runtimeapi.ErrNotFound) ||
+ errors.Is(err, executor.ErrExecutionNotFound)
+}
+
// rowLiveness classifies a row's backing-process liveness, returning Unknown when
// no prober is wired (unit tests, degraded startup) so a caller never mistakes
// "can't probe" for "dead". The probe is runtime-aware: a local process check
diff --git a/apps/backend/internal/orchestrator/reconcile_liveness_test.go b/apps/backend/internal/orchestrator/reconcile_liveness_test.go
new file mode 100644
index 0000000000..cbcf3a5590
--- /dev/null
+++ b/apps/backend/internal/orchestrator/reconcile_liveness_test.go
@@ -0,0 +1,145 @@
+package orchestrator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ runtimeapi "github.com/kandev/kandev/internal/agent/runtime"
+ "github.com/kandev/kandev/internal/agentruntime"
+ "github.com/kandev/kandev/internal/common/logger"
+ "github.com/kandev/kandev/internal/task/models"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
+)
+
+func TestClassifyStartupCleanupDecision(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ liveness models.ProcessLiveness
+ wantDisposition startupCleanupDisposition
+ wantProceed bool
+ wantExpected bool
+ }{
+ {
+ name: "successful stop",
+ liveness: models.ProcessLivenessAlive,
+ wantDisposition: startupCleanupDispositionStopped,
+ wantProceed: true,
+ },
+ {
+ name: "not found dead local",
+ err: runtimeapi.ErrNotFound,
+ liveness: models.ProcessLivenessDead,
+ wantDisposition: startupCleanupDispositionAlreadyAbsent,
+ wantProceed: true,
+ },
+ {
+ name: "not found alive local",
+ err: runtimeapi.ErrNotFound,
+ liveness: models.ProcessLivenessAlive,
+ wantDisposition: startupCleanupDispositionPreserved,
+ wantExpected: true,
+ },
+ {
+ name: "not found unknown",
+ err: runtimeapi.ErrNotFound,
+ liveness: models.ProcessLivenessUnknown,
+ wantDisposition: startupCleanupDispositionPreserved,
+ wantExpected: true,
+ },
+ {
+ name: "unexpected stop failure",
+ err: errors.New("database unavailable"),
+ liveness: models.ProcessLivenessDead,
+ wantDisposition: startupCleanupDispositionPreserved,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ decision := classifyStartupCleanup(tt.err, tt.liveness)
+ if decision.disposition != tt.wantDisposition {
+ t.Fatalf("disposition = %q, want %q", decision.disposition, tt.wantDisposition)
+ }
+ if decision.proceed != tt.wantProceed {
+ t.Fatalf("proceed = %v, want %v", decision.proceed, tt.wantProceed)
+ }
+ if decision.expected != tt.wantExpected {
+ t.Fatalf("expected = %v, want %v", decision.expected, tt.wantExpected)
+ }
+ })
+ }
+}
+
+func TestStartupCleanupReportAggregatesExpectedPreservationWithoutSecrets(t *testing.T) {
+ core, logs := observer.New(zap.WarnLevel)
+ log, err := logger.NewFromZap(zap.New(core))
+ if err != nil {
+ t.Fatalf("NewFromZap: %v", err)
+ }
+ report := newStartupCleanupReport()
+ row := &models.ExecutorRunning{
+ SessionID: "session-safe",
+ TaskID: "task-safe",
+ AgentExecutionID: "exec-safe",
+ ResumeToken: "resume-token-must-not-log",
+ }
+ decision := classifyStartupCleanup(runtimeapi.ErrNotFound, models.ProcessLivenessUnknown)
+ report.recordExpected(row, decision)
+ report.recordExpected(row, decision)
+ report.flush(log)
+
+ if logs.Len() != 1 {
+ t.Fatalf("summary log count = %d, want 1", logs.Len())
+ }
+ entry := logs.All()[0]
+ if entry.Message != "startup runtime cleanup summary" {
+ t.Fatalf("summary message = %q", entry.Message)
+ }
+ got := fmt.Sprint(entry.ContextMap())
+ if !containsAll(got, "expected_preserved_count", "runtime_not_found", "unknown") {
+ t.Fatalf("summary fields = %s", got)
+ }
+ if containsAll(got, "resume-token-must-not-log") {
+ t.Fatal("summary leaked a resume token")
+ }
+}
+
+func TestStopRuntimeForStartupCleanupPreservesUnknownRowsWithoutHandle(t *testing.T) {
+ repo := setupTestRepo(t)
+ ctx := context.Background()
+ now := time.Now().UTC()
+ seedTaskAndSession(t, repo, "taskUnknown", "sessionUnknown", models.TaskSessionStateCancelled)
+ if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
+ ID: "sessionUnknown", SessionID: "sessionUnknown", TaskID: "taskUnknown",
+ Runtime: agentruntime.RuntimeSSH, Status: models.ExecutorRunningStatusRunning,
+ PID: 9191, CreatedAt: now, UpdatedAt: now,
+ }); err != nil {
+ t.Fatalf("upsert unknown row: %v", err)
+ }
+
+ service := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), &mockAgentManager{
+ rowLivenessFn: func(*models.ExecutorRunning) models.ProcessLiveness {
+ return models.ProcessLivenessUnknown
+ },
+ })
+ service.reconcileSessionsOnStartup(ctx)
+
+ if _, err := repo.GetExecutorRunningBySessionID(ctx, "sessionUnknown"); err != nil {
+ t.Fatalf("unknown row without a stop handle must be preserved: %v", err)
+ }
+}
+
+func containsAll(value string, needles ...string) bool {
+ for _, needle := range needles {
+ if !strings.Contains(value, needle) {
+ return false
+ }
+ }
+ return true
+}
diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go
index 886d0956e0..d44be00c13 100644
--- a/apps/backend/internal/orchestrator/service.go
+++ b/apps/backend/internal/orchestrator/service.go
@@ -24,7 +24,6 @@ import (
"go.uber.org/zap"
- runtimeapi "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
@@ -1739,6 +1738,7 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) {
s.logger.Info("reconciling sessions on startup (lazy recovery)", zap.Int("count", len(runningExecutors)))
+ report := newStartupCleanupReport()
var remoteRecords []executor.RemoteStatusPollRequest
for _, running := range runningExecutors {
if models.IsRemoteExecutorType(models.ExecutorType(running.Runtime)) {
@@ -1750,8 +1750,9 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) {
Metadata: running.Metadata,
})
}
- s.reconcileOneSessionOnStartup(ctx, running)
+ s.reconcileOneSessionOnStartup(ctx, running, report)
}
+ report.flush(s.logger)
// Pre-poll remote executor status so task lists show accurate state
if len(remoteRecords) > 0 && s.agentManager != nil {
@@ -1760,7 +1761,7 @@ func (s *Service) reconcileSessionsOnStartup(ctx context.Context) {
}
// reconcileOneSessionOnStartup adjusts DB state for a single session without launching agents.
-func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning) {
+func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) {
sessionID := running.SessionID
if sessionID == "" {
return
@@ -1769,7 +1770,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil {
if isTaskSessionNotFound(err) {
- s.handleMissingSessionOnStartup(ctx, running)
+ s.handleMissingSessionOnStartup(ctx, running, report)
return
}
s.logger.Warn("failed to load session for reconciliation; preserving executor record",
@@ -1781,7 +1782,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
previousState := session.State
// Handle terminal and never-started states (reuse existing cleanup logic)
- if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState); skip {
+ if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState, report); skip {
return
}
if previousState == models.TaskSessionStateCreated {
@@ -1819,6 +1820,15 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
return
}
+ s.reconcileActiveSessionOnStartup(ctx, running, sessionID, previousState)
+}
+
+func (s *Service) reconcileActiveSessionOnStartup(
+ ctx context.Context,
+ running *models.ExecutorRunning,
+ sessionID string,
+ previousState models.TaskSessionState,
+) {
// Active states: STARTING, RUNNING, WAITING_FOR_INPUT
// Set session to WAITING_FOR_INPUT (idle, ready for lazy resume when user opens it)
if previousState != models.TaskSessionStateWaitingForInput {
@@ -1892,13 +1902,19 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
zap.Bool("has_worktree", running.WorktreePath != ""))
}
-func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning) {
+func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) {
sessionID := running.SessionID
executionID := strings.TrimSpace(running.AgentExecutionID)
if executionID == "" || s.agentManager == nil {
- s.logger.Warn("executor record has no session and no stoppable runtime handle; preserving record",
- zap.String("session_id", sessionID),
- zap.String("task_id", running.TaskID))
+ decision := startupCleanupDecision{
+ disposition: startupCleanupDispositionPreserved,
+ liveness: processLivenessClass(s.rowLiveness(running)),
+ stopErrorClass: "missing_stop_handle",
+ expected: true,
+ }
+ report.recordExpected(running, decision)
+ s.logger.Debug("executor record has no stoppable runtime handle; preserving record",
+ startupCleanupFields(running, decision)...)
return
}
if err := s.agentManager.StopAgentWithReason(ctx, executionID, "startup missing session cleanup", true); err != nil {
@@ -1908,18 +1924,20 @@ func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *mo
// invariant, so the orphan row does not survive restarts forever. Any
// other error (alive/unknown/remote row, or a non-not-found failure) is
// preserved and left for a later attempt.
- if !stopReportsRuntimeAbsent(err) || s.rowLiveness(running) != models.ProcessLivenessDead {
- s.logger.Warn("failed to stop missing-session runtime; preserving executor record",
- zap.String("session_id", sessionID),
- zap.String("task_id", running.TaskID),
- zap.String("execution_id", executionID),
- zap.Error(err))
+ decision := classifyStartupCleanup(err, s.rowLiveness(running))
+ if !decision.proceed {
+ if decision.expected {
+ report.recordExpected(running, decision)
+ s.logger.Debug("failed to stop missing-session runtime; preserving executor record",
+ startupCleanupFields(running, decision)...)
+ } else {
+ s.logger.Warn("failed to stop missing-session runtime; preserving executor record",
+ startupCleanupFields(running, decision)...)
+ }
return
}
s.logger.Info("missing-session runtime already gone (confirmed dead); pruning executor record",
- zap.String("session_id", sessionID),
- zap.String("task_id", running.TaskID),
- zap.String("execution_id", executionID))
+ startupCleanupFields(running, decision)...)
s.pruneOrRepairExecutorRow(ctx, running, models.TaskSessionStateCancelled)
return
}
@@ -1931,17 +1949,6 @@ func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *mo
}
}
-// stopReportsRuntimeAbsent reports whether a stop error means the runtime is
-// already gone (a typed not-found sentinel), as opposed to a transient failure.
-// Only a typed sentinel counts — a generic store/lookup error must never be
-// reinterpreted as an absent runtime. The runtime seam (lifecycleAdapter)
-// normalizes the backend lifecycle not-found sentinel to runtimeapi.ErrNotFound,
-// so reconciliation depends only on the public runtime and executor sentinels.
-func stopReportsRuntimeAbsent(err error) bool {
- return errors.Is(err, runtimeapi.ErrNotFound) ||
- errors.Is(err, executor.ErrExecutionNotFound)
-}
-
func (s *Service) abandonOpenTurnsOnStartup(ctx context.Context, sessionID, reason string) {
if s.turnService == nil {
return
@@ -1961,7 +1968,7 @@ func isTaskSessionNotFound(err error) bool {
// handleTerminalSessionOnStartup processes sessions in terminal states during startup.
// Returns true if the session should be skipped (no further processing needed).
-func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState) bool {
+func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState, report *startupCleanupReport) bool {
sessionID := session.ID
switch previousState {
case models.TaskSessionStateCompleted, models.TaskSessionStateCancelled:
@@ -1970,7 +1977,7 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m
zap.String("task_id", session.TaskID),
zap.String("state", string(previousState)))
s.abandonOpenTurnsOnStartup(ctx, sessionID, "terminal session cleanup")
- if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup") {
+ if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup", report) {
return true
}
// Resume-safety invariant: prune the row only if it is not still resumable
@@ -1980,14 +1987,14 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m
s.pruneOrRepairExecutorRow(ctx, running, previousState)
return true
case models.TaskSessionStateFailed:
- s.handleFailedSessionOnStartup(ctx, session, running)
+ s.handleFailedSessionOnStartup(ctx, session, running, report)
return true
}
return false
}
// handleFailedSessionOnStartup handles a failed session during startup recovery.
-func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning) {
+func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, report *startupCleanupReport) {
sessionID := session.ID
s.abandonOpenTurnsOnStartup(ctx, sessionID, "failed session cleanup")
// If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS)
@@ -2015,7 +2022,7 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod
s.logger.Info("stopping failed session runtime before cleaning up executor record",
zap.String("session_id", sessionID),
zap.String("task_id", session.TaskID))
- if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup") {
+ if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup", report) {
return
}
// Prune only subject to the resume-safety invariant (a lingering
@@ -2031,30 +2038,50 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod
// leaking stale rows: a not-found stop for a confirmed-dead LOCAL runtime means
// the runtime is already gone and cleanup proceeds; any other error
// (alive/unknown/remote row, or a non-not-found failure) preserves the row for a
-// later attempt. A row with no stoppable handle is already stoppable-complete.
-func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string) bool {
+// later attempt. A row with no stoppable handle proceeds only when its local
+// process liveness is confirmed dead; alive and Unknown rows remain durable.
+func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string, report *startupCleanupReport) bool {
executionID := strings.TrimSpace(running.AgentExecutionID)
if executionID == "" || s.agentManager == nil {
+ liveness := s.rowLiveness(running)
+ decision := startupCleanupDecision{
+ disposition: startupCleanupDispositionPreserved,
+ liveness: processLivenessClass(liveness),
+ stopErrorClass: "missing_stop_handle",
+ expected: true,
+ }
+ if liveness == models.ProcessLivenessDead {
+ decision.disposition = startupCleanupDispositionAlreadyAbsent
+ decision.proceed = true
+ decision.expected = false
+ }
+ if !decision.proceed {
+ report.recordExpected(running, decision)
+ s.logger.Debug("session runtime has no stoppable handle; preserving executor record",
+ append(startupCleanupFields(running, decision), zap.String("reason", reason))...)
+ return false
+ }
+ s.logger.Info("session runtime is confirmed dead without a stop handle; proceeding to prune/repair",
+ append(startupCleanupFields(running, decision), zap.String("reason", reason))...)
return true
}
err := s.agentManager.StopAgentWithReason(ctx, executionID, reason, true)
if err == nil {
return true
}
- if !stopReportsRuntimeAbsent(err) || s.rowLiveness(running) != models.ProcessLivenessDead {
- s.logger.Warn("failed to stop session runtime during startup cleanup; preserving executor record",
- zap.String("session_id", running.SessionID),
- zap.String("task_id", running.TaskID),
- zap.String("execution_id", executionID),
- zap.String("reason", reason),
- zap.Error(err))
+ decision := classifyStartupCleanup(err, s.rowLiveness(running))
+ if !decision.proceed {
+ fields := append(startupCleanupFields(running, decision), zap.String("reason", reason))
+ if decision.expected {
+ report.recordExpected(running, decision)
+ s.logger.Debug("failed to stop session runtime during startup cleanup; preserving executor record", fields...)
+ } else {
+ s.logger.Warn("failed to stop session runtime during startup cleanup; preserving executor record", fields...)
+ }
return false
}
s.logger.Info("session runtime already gone (confirmed dead) during startup cleanup; proceeding to prune/repair",
- zap.String("session_id", running.SessionID),
- zap.String("task_id", running.TaskID),
- zap.String("execution_id", executionID),
- zap.String("reason", reason))
+ append(startupCleanupFields(running, decision), zap.String("reason", reason))...)
return true
}
diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go
index dda9ae2323..8837c1634e 100644
--- a/apps/backend/pkg/websocket/actions.go
+++ b/apps/backend/pkg/websocket/actions.go
@@ -335,7 +335,8 @@ const (
// System maintenance jobs (VACUUM, factory reset, snapshot create/restore,
// disk walk). Broadcast to all connected clients so the System pages can
// render progress.
- ActionSystemJobUpdate = "system.job.update"
+ ActionSystemJobUpdate = "system.job.update"
+ ActionSystemAgentRuntimeStatusChanged = "system.agent_runtime.status_changed"
// VS Code server actions
ActionVscodeStart = "vscode.start" // Start code-server for a session
diff --git a/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx b/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx
new file mode 100644
index 0000000000..21773a6001
--- /dev/null
+++ b/apps/web/components/app-status-bar/agent-runtime-unavailable-alert.test.tsx
@@ -0,0 +1,144 @@
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { StateProvider, useAppStoreApi } from "@/components/state-provider";
+import type { RestartCapabilityState } from "@/hooks/domains/system/use-restart-capability";
+import type { KandevRestartController } from "@/hooks/domains/system/use-kandev-restart";
+import { AgentRuntimeUnavailableAlert } from "./agent-runtime-unavailable-alert";
+
+const RESTART_BUTTON_LABEL = "Restart Kandev";
+
+const restartCapability = vi.hoisted(() => ({
+ value: {
+ status: "resolved",
+ capability: { supported: true, mode: "supervisor", adapter: "supervisor" },
+ } as RestartCapabilityState,
+}));
+const restartController = vi.hoisted(() => ({
+ value: {
+ phase: "idle",
+ errorMessage: null,
+ isRestarting: false,
+ start: vi.fn(),
+ dismiss: vi.fn(),
+ } as KandevRestartController,
+}));
+
+vi.mock("@/hooks/domains/system/use-restart-capability", () => ({
+ useRestartCapability: () => restartCapability.value,
+}));
+
+vi.mock("@/hooks/domains/system/use-kandev-restart", () => ({
+ useKandevRestart: () => restartController.value,
+}));
+
+vi.mock("@/components/settings/system/restart-progress-dialog", () => ({
+ RestartProgressDialog: ({ phase }: { phase: string }) =>
+ phase === "idle" ? null :
,
+}));
+
+function RuntimeStateControls() {
+ const store = useAppStoreApi();
+ return (
+ <>
+