From c7cc92382b45f4f45357b577d618f53702a45a52 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 22 Sep 2026 17:31:00 +0800 Subject: [PATCH] feat(executors): add continuous SSH host reachability probing (#3747) Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio --- .../agent/runtime/lifecycle/executor_ssh.go | 6 +- .../lifecycle/executor_ssh_connection.go | 35 +- .../lifecycle/executor_ssh_connection_test.go | 35 ++ .../executor_ssh_reachability_launch_test.go | 45 ++ .../executor_ssh_reachability_probe.go | 188 ++++++ .../executor_ssh_reachability_probe_test.go | 372 +++++++++++ .../agent/runtime/lifecycle/manager.go | 14 + .../agent/runtime/lifecycle/manager_launch.go | 2 + .../manager_launch_reachability_warning.go | 145 +++++ ...anager_launch_reachability_warning_test.go | 305 +++++++++ .../agent/runtime/ssh_reachability_probe.go | 92 +++ .../internal/backendapp/canvas_edit_test.go | 52 +- apps/backend/internal/backendapp/helpers.go | 12 + .../internal/backendapp/httpserver_test.go | 2 +- apps/backend/internal/backendapp/main.go | 40 +- .../backendapp/ssh_reachability_wiring.go | 26 + .../ssh_reachability_wiring_test.go | 108 ++++ .../backend/internal/common/config/catalog.go | 1 + .../internal/common/config/catalog_test.go | 1 + apps/backend/internal/common/config/config.go | 10 + apps/backend/internal/common/config/source.go | 47 +- .../internal/common/config/source_test.go | 68 ++ apps/backend/internal/events/types.go | 24 + .../executors/reachability/goleak_test.go | 15 + .../executors/reachability/interval.go | 51 ++ .../executors/reachability/interval_test.go | 31 + .../executors/reachability/metrics.go | 60 ++ .../internal/executors/reachability/poller.go | 297 +++++++++ .../reachability/poller_publish_test.go | 70 +++ .../executors/reachability/poller_test.go | 445 +++++++++++++ .../internal/executors/reachability/probe.go | 22 + .../executors/reachability/probe_and_wait.go | 102 +++ .../reachability/probe_and_wait_test.go | 232 +++++++ .../executors/reachability/publish.go | 105 ++++ .../executors/reachability/publish_test.go | 158 +++++ .../executors/reachability/save_observer.go | 126 ++++ .../reachability/save_observer_test.go | 249 ++++++++ .../internal/executors/reachability/store.go | 222 +++++++ .../executors/reachability/store_test.go | 322 ++++++++++ .../backend/internal/gateway/websocket/hub.go | 8 + .../gateway/websocket/hub_broadcast_test.go | 40 ++ .../gateway/websocket/launch_warning_cache.go | 50 ++ .../websocket/session_notifications.go | 5 + .../websocket/session_notifications_test.go | 75 +++ .../gateway/websocket/task_notifications.go | 5 + ...otifications_executor_reachability_test.go | 36 ++ .../websocket/task_notifications_test.go | 2 +- .../internal/office/retention/sweep.go | 14 +- apps/backend/internal/ssh/handlers.go | 73 +-- .../internal/ssh/reachability_handlers.go | 228 +++++++ .../ssh/reachability_handlers_test.go | 362 +++++++++++ .../task/handlers/process_handlers_test.go | 18 + .../task/models/executor_reachability.go | 69 ++ .../internal/task/repository/interface.go | 27 + .../task/repository/sqlite/base_migrations.go | 18 + .../task/repository/sqlite/executor.go | 58 +- .../sqlite/executor_reachability.go | 183 ++++++ .../executor_reachability_postgres_test.go | 59 ++ .../sqlite/executor_reachability_test.go | 332 ++++++++++ apps/backend/internal/task/service/service.go | 19 + .../task/service/service_resources.go | 12 + ...e_resources_executors_reachability_test.go | 190 ++++++ apps/backend/pkg/websocket/actions.go | 2 + .../executors/ssh/[executorId]/page.tsx | 2 + .../settings/ssh-reachability-card.test.tsx | 213 +++++++ .../settings/ssh-reachability-card.tsx | 198 ++++++ apps/web/components/settings/ssh-settings.tsx | 2 + .../components/task/launch-warning.test.tsx | 77 +++ apps/web/components/task/launch-warning.tsx | 67 ++ .../task-chat-panel.launch-error.test.tsx | 1 + apps/web/components/task/task-chat-panel.tsx | 110 ++-- apps/web/e2e/pages/SSHSettingsPage.ts | 32 + apps/web/e2e/tests/ssh/reachability.spec.ts | 100 +++ .../domains/settings/use-ssh-reachability.ts | 82 +++ apps/web/lib/api/domains/ssh-api.test.ts | 97 +++ apps/web/lib/api/domains/ssh-api.ts | 34 + apps/web/lib/state/app-state-types.ts | 9 + apps/web/lib/state/default-state.ts | 2 + .../session-runtime/purge-session.test.ts | 21 + .../session-runtime/session-runtime-slice.ts | 10 + .../lib/state/slices/session-runtime/types.ts | 19 + .../slices/settings/settings-slice.test.ts | 101 +++ .../state/slices/settings/settings-slice.ts | 25 + apps/web/lib/state/slices/settings/types.ts | 21 + apps/web/lib/state/store-overrides.ts | 1 + apps/web/lib/types/backend.ts | 8 + apps/web/lib/types/executor-payloads.ts | 11 + apps/web/lib/types/http-ssh.ts | 29 + .../web/lib/ws/handlers/agent-session.test.ts | 2 + apps/web/lib/ws/handlers/agent-session.ts | 8 +- apps/web/lib/ws/handlers/executors.ts | 3 + .../ws/handlers/ssh-launch-warning.test.ts | 89 +++ .../web/lib/ws/handlers/ssh-launch-warning.ts | 20 + apps/web/lib/ws/router.ts | 2 + apps/web/src/locales/en/executors.json | 22 + apps/web/src/locales/en/task.json | 10 + apps/web/src/locales/pseudo/executors.json | 22 + apps/web/src/locales/pseudo/task.json | 10 + apps/web/src/locales/pt-pt/executors.json | 22 + apps/web/src/locales/pt-pt/task.json | 10 + apps/web/src/locales/zh-cn/executors.json | 22 + apps/web/src/locales/zh-cn/task.json | 10 + apps/web/src/locales/zh-hk/executors.json | 22 + apps/web/src/locales/zh-hk/task.json | 10 + apps/web/src/locales/zh-tw/executors.json | 22 + apps/web/src/locales/zh-tw/task.json | 10 + docs/plans/ssh-host-reachability/plan.md | 540 ++++++++++++++++ .../task-01-probe-and-classification.md | 205 ++++++ .../task-02-reachability-persistence.md | 238 +++++++ .../task-03-reachability-poller.md | 330 ++++++++++ .../task-04-reachability-api-and-events.md | 309 +++++++++ .../task-05-launch-non-gating.md | 316 ++++++++++ .../task-06-settings-surface.md | 258 ++++++++ .../task-07-pre-launch-warning.md | 221 +++++++ .../task-08-e2e-and-docs.md | 183 ++++++ docs/public/configuration.md | 1 + docs/public/executors.md | 24 + docs/specs/executors/README.md | 3 + .../requirements/ssh-reachability.md | 178 ++++++ .../ssh-reachability-surfaces.md | 233 +++++++ .../system-design/ssh-reachability.md | 588 ++++++++++++++++++ 121 files changed, 11067 insertions(+), 170 deletions(-) create mode 100644 apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go create mode 100644 apps/backend/internal/agent/runtime/ssh_reachability_probe.go create mode 100644 apps/backend/internal/backendapp/ssh_reachability_wiring.go create mode 100644 apps/backend/internal/backendapp/ssh_reachability_wiring_test.go create mode 100644 apps/backend/internal/executors/reachability/goleak_test.go create mode 100644 apps/backend/internal/executors/reachability/interval.go create mode 100644 apps/backend/internal/executors/reachability/interval_test.go create mode 100644 apps/backend/internal/executors/reachability/metrics.go create mode 100644 apps/backend/internal/executors/reachability/poller.go create mode 100644 apps/backend/internal/executors/reachability/poller_publish_test.go create mode 100644 apps/backend/internal/executors/reachability/poller_test.go create mode 100644 apps/backend/internal/executors/reachability/probe.go create mode 100644 apps/backend/internal/executors/reachability/probe_and_wait.go create mode 100644 apps/backend/internal/executors/reachability/probe_and_wait_test.go create mode 100644 apps/backend/internal/executors/reachability/publish.go create mode 100644 apps/backend/internal/executors/reachability/publish_test.go create mode 100644 apps/backend/internal/executors/reachability/save_observer.go create mode 100644 apps/backend/internal/executors/reachability/save_observer_test.go create mode 100644 apps/backend/internal/executors/reachability/store.go create mode 100644 apps/backend/internal/executors/reachability/store_test.go create mode 100644 apps/backend/internal/gateway/websocket/launch_warning_cache.go create mode 100644 apps/backend/internal/gateway/websocket/task_notifications_executor_reachability_test.go create mode 100644 apps/backend/internal/ssh/reachability_handlers.go create mode 100644 apps/backend/internal/ssh/reachability_handlers_test.go create mode 100644 apps/backend/internal/task/models/executor_reachability.go create mode 100644 apps/backend/internal/task/repository/sqlite/executor_reachability.go create mode 100644 apps/backend/internal/task/repository/sqlite/executor_reachability_postgres_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/executor_reachability_test.go create mode 100644 apps/backend/internal/task/service/service_resources_executors_reachability_test.go create mode 100644 apps/web/components/settings/ssh-reachability-card.test.tsx create mode 100644 apps/web/components/settings/ssh-reachability-card.tsx create mode 100644 apps/web/components/task/launch-warning.test.tsx create mode 100644 apps/web/components/task/launch-warning.tsx create mode 100644 apps/web/e2e/tests/ssh/reachability.spec.ts create mode 100644 apps/web/hooks/domains/settings/use-ssh-reachability.ts create mode 100644 apps/web/lib/api/domains/ssh-api.test.ts create mode 100644 apps/web/lib/ws/handlers/ssh-launch-warning.test.ts create mode 100644 apps/web/lib/ws/handlers/ssh-launch-warning.ts create mode 100644 docs/plans/ssh-host-reachability/plan.md create mode 100644 docs/plans/ssh-host-reachability/task-01-probe-and-classification.md create mode 100644 docs/plans/ssh-host-reachability/task-02-reachability-persistence.md create mode 100644 docs/plans/ssh-host-reachability/task-03-reachability-poller.md create mode 100644 docs/plans/ssh-host-reachability/task-04-reachability-api-and-events.md create mode 100644 docs/plans/ssh-host-reachability/task-05-launch-non-gating.md create mode 100644 docs/plans/ssh-host-reachability/task-06-settings-surface.md create mode 100644 docs/plans/ssh-host-reachability/task-07-pre-launch-warning.md create mode 100644 docs/plans/ssh-host-reachability/task-08-e2e-and-docs.md create mode 100644 docs/specs/executors/requirements/ssh-reachability.md create mode 100644 docs/specs/executors/system-design/ssh-reachability-surfaces.md create mode 100644 docs/specs/executors/system-design/ssh-reachability.md diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go index e5b316d066..5747933c3d 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go @@ -256,7 +256,11 @@ func (r *SSHExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateReq } client, err := dialSSH(baseCtx, target) if err != nil { - return nil, fmt.Errorf("ssh: connect to %s@%s: %w", target.User, target.Host, err) + // Classified from this launch's own attempt, never from a stored + // reachability record — that record can predate this dial by a full + // probe interval and would misattribute a stale cause to a fresh + // failure (see task-05-launch-non-gating.md). + return nil, fmt.Errorf("ssh: connect to %s@%s: %s: %w", target.User, target.Host, ClassifyDialError(err), err) } var runtimeAPITunnel *sshRuntimeAPITunnel released := false diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go index 2344b0bcea..eab48365fa 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go @@ -495,7 +495,7 @@ func dialDirect(ctx context.Context, addr string, cfg *ssh.ClientConfig) (*ssh.C if err != nil { return nil, fmt.Errorf("ssh: tcp dial %s: %w", addr, err) } - sshConn, chans, reqs, err := ssh.NewClientConn(tcpConn, addr, cfg) + sshConn, chans, reqs, err := handshakeWithDeadline(ctx, tcpConn, addr, cfg) if err != nil { _ = tcpConn.Close() return nil, fmt.Errorf("ssh: handshake with %s: %w", addr, err) @@ -530,6 +530,37 @@ func (c *sshProxyJumpConn) Close() error { return c.closeErr } +// handshakeWithDeadline runs the SSH handshake over conn, bounding it by +// ctx's deadline when one is set. ssh.NewClientConn itself takes no context — +// without this, a peer that completes the TCP handshake but never speaks SSH +// (or stalls partway through key exchange) blocks the caller indefinitely. +// Callers that pass a context with no deadline see unchanged behavior. +func handshakeWithDeadline(ctx context.Context, conn net.Conn, addr string, cfg *ssh.ClientConfig) (ssh.Conn, <-chan ssh.NewChannel, <-chan *ssh.Request, error) { + cancelDone := make(chan struct{}) + stopCancellation := context.AfterFunc(ctx, func() { + // ProxyJump tunnels wrap an x/crypto/ssh channel. That channel + // reports unsupported deadlines, so closing the owner transport is + // the only reliable way to unblock NewClientConn on cancellation. + _ = conn.Close() + close(cancelDone) + }) + defer func() { + if !stopCancellation() { + <-cancelDone + } + }() + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + defer func() { _ = conn.SetDeadline(time.Time{}) }() + } + sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg) + if ctxErr := ctx.Err(); ctxErr != nil { + _ = conn.Close() + return nil, nil, nil, ctxErr + } + return sshConn, chans, reqs, err +} + // dialViaJump implements ProxyJump as a single bastion hop. The bastion is // resolved from its own ~/.ssh/config Host block, defaulting to the same // identity source as the target (passes the user's agent / key through). @@ -568,7 +599,7 @@ func dialViaJump(ctx context.Context, target *SSHTarget, finalAddr string, final return nil, fmt.Errorf("ssh: bastion tunnel to %s: %w", finalAddr, err) } jumpConn := &sshProxyJumpConn{Conn: tunnel, bastion: bastionClient} - sshConn, chans, reqs, err := ssh.NewClientConn(jumpConn, finalAddr, finalCfg) + sshConn, chans, reqs, err := handshakeWithDeadline(ctx, jumpConn, finalAddr, finalCfg) if err != nil { _ = jumpConn.Close() return nil, fmt.Errorf("ssh: handshake with %s via %s: %w", finalAddr, target.ProxyJump, err) diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection_test.go index 6307c6ba95..a21f22f29e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection_test.go @@ -2,6 +2,8 @@ package lifecycle import ( "context" + "errors" + "net" "net/http" "os" "os/exec" @@ -17,6 +19,39 @@ import ( "golang.org/x/crypto/ssh" ) +type unsupportedDeadlineConn struct{ net.Conn } + +func (unsupportedDeadlineConn) SetDeadline(time.Time) error { + return errors.New("deadlines are not supported") +} + +func TestHandshakeWithDeadlineClosesConnectionsWhenDeadlinesAreUnsupported(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer func() { _ = serverConn.Close() }() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { + _, _, _, err := handshakeWithDeadline(ctx, unsupportedDeadlineConn{Conn: clientConn}, "test", &ssh.ClientConfig{ + User: "test", + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }) + done <- err + }() + + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("handshake error = %v, want context deadline exceeded", err) + } + case <-time.After(500 * time.Millisecond): + _ = clientConn.Close() + <-done + t.Fatal("handshake did not stop when its context deadline expired") + } +} + func TestSSHControlRequestUsesBearerToken(t *testing.T) { req, err := http.NewRequest(http.MethodPost, "http://127.0.0.1/api/v1/instances", nil) if err != nil { diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go new file mode 100644 index 0000000000..9191f167d5 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go @@ -0,0 +1,45 @@ +package lifecycle + +import ( + "context" + "strconv" + "strings" + "testing" +) + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2 +// +// A launch's own dial failure must be attributed with the target host and +// the same reason ClassifyDialError would produce for a standalone probe — +// not read from any stored record, since the stored record can predate this +// attempt by a full interval. +func TestSSHExecutorCreateInstanceDialErrorNamesHostAndReason(t *testing.T) { + server := newFakeSSHServer(t, nil) + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger()) + t.Cleanup(func() { _ = exec.Close() }) + + _, err := exec.CreateInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + Metadata: map[string]interface{}{ + MetadataKeySSHHost: "127.0.0.1", + MetadataKeySSHPort: strconv.Itoa(server.port()), + MetadataKeySSHUser: "kandev", + MetadataKeySSHHostFingerprint: "SHA256:not-the-server-key", + MetadataKeySSHIdentitySource: string(SSHIdentitySourceFile), + MetadataKeySSHIdentityFile: identity, + }, + }) + + if err == nil { + t.Fatal("expected a dial failure, got nil error") + } + if !strings.Contains(err.Error(), "127.0.0.1") { + t.Fatalf("error = %q, want it to name the target host", err.Error()) + } + if !strings.Contains(err.Error(), string(SSHReachabilityReasonHostKey)) { + t.Fatalf("error = %q, want it to carry the classified reason %q", err.Error(), SSHReachabilityReasonHostKey) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe.go new file mode 100644 index 0000000000..fbae97264d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe.go @@ -0,0 +1,188 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + "unicode/utf8" + + "golang.org/x/crypto/ssh/knownhosts" + + "github.com/kandev/kandev/internal/agent/runtime/routingerr" +) + +const maxSSHReachabilityMessageBytes = 512 + +// SanitizeSSHReachabilityMessage keeps dial diagnostics useful while making +// sure credentials, local paths, and oversized provider text do not cross the +// reachability API boundary. +func SanitizeSSHReachabilityMessage(err error) string { + if err == nil { + return "" + } + message := routingerr.Sanitize(err.Error()) + if len(message) <= maxSSHReachabilityMessageBytes { + return message + } + cut := maxSSHReachabilityMessageBytes - len("…") + for cut > 0 && !utf8.RuneStart(message[cut]) { + cut-- + } + return message[:cut] + "…" +} + +// SSHReachabilityReason classifies why a probe or launch-time SSH dial +// failed. The set is closed: ClassifyDialError always returns one of these +// six values, walking a total order so a deadline is never reported as a +// network failure and a rejected host key is never reported as an +// authentication failure. +type SSHReachabilityReason string + +const ( + SSHReachabilityReasonConfig SSHReachabilityReason = "config" + SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout" + SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key" + SSHReachabilityReasonAuth SSHReachabilityReason = "auth" + SSHReachabilityReasonNetwork SSHReachabilityReason = "network" + SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown" +) + +// SSHProbeOutcome is the result of a single reachability probe, or of a +// launch's own dial attempt classified through the same rules. Host always +// names the target executor's host, even when a failure occurred on a +// bastion hop. Cancelled means the caller's context ended the attempt before +// it produced an observation about the host; callers must discard a +// cancelled outcome rather than recording it as a failure. +type SSHProbeOutcome struct { + Success bool + Cancelled bool + Host string + Reason SSHReachabilityReason + Message string +} + +// ProbeSSHHost opens an authenticated SSH transport to target using its +// pinned fingerprint, closes it without running a remote command, and +// returns a typed outcome. It bounds both the TCP dial and the SSH handshake +// by timeout, and never dials when the target has no pinned fingerprint to +// verify against. +func ProbeSSHHost(ctx context.Context, target *SSHTarget, timeout time.Duration) SSHProbeOutcome { + outcome := SSHProbeOutcome{Host: target.Host} + if strings.TrimSpace(target.PinnedFingerprint) == "" { + outcome.Reason = SSHReachabilityReasonConfig + outcome.Message = "no pinned host key fingerprint is configured" + return outcome + } + + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + client, err := dialSSH(probeCtx, target) + if err != nil { + if errors.Is(err, context.Canceled) { + outcome.Cancelled = true + return outcome + } + outcome.Reason = ClassifyDialError(err) + outcome.Message = SanitizeSSHReachabilityMessage(err) + return outcome + } + defer func() { _ = client.Close() }() + + outcome.Success = true + return outcome +} + +// ClassifyDialError maps a non-nil SSH dial error to exactly one reason from +// the closed set, walking timeout, host_key, auth, network, unknown in that +// order — first match wins. context.Canceled is deliberately not classified +// here: it is not an observation about the host, and callers check for it +// (see ProbeSSHHost) before reaching this function. +func ClassifyDialError(err error) SSHReachabilityReason { + switch { + case isSSHTimeoutError(err): + return SSHReachabilityReasonTimeout + case isSSHHostKeyMismatch(err): + return SSHReachabilityReasonHostKey + case isSSHAuthFailure(err): + return SSHReachabilityReasonAuth + case isSSHNetworkError(err): + return SSHReachabilityReasonNetwork + default: + return SSHReachabilityReasonUnknown + } +} + +func isSSHTimeoutError(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// isSSHHostKeyMismatch confines the host_key discriminator to Kandev's own +// errHostKeyMismatch (the pinned-fingerprint check) and knownhosts.KeyError +// with a non-empty Want (a genuine mismatch, not an unknown-host TOFU +// accept, which returns nil and never reaches this function) — the bastion +// host-key check. golang.org/x/crypto/ssh has no stable exported error type +// for a rejected key; keying off our own types keeps a library upgrade from +// turning a security event into auth or unknown. +func isSSHHostKeyMismatch(err error) bool { + var mismatch *errHostKeyMismatch + if errors.As(err, &mismatch) { + return true + } + var keyErr *knownhosts.KeyError + return errors.As(err, &keyErr) && len(keyErr.Want) > 0 +} + +// isSSHAuthFailure relies on string-matching because the pinned +// golang.org/x/crypto/ssh v0.57.0 dependency has no exported error type for +// an authentication failure — it returns a plain fmt.Errorf. Revisit this +// classifier when upgrading the dependency if it adds a stable type. +func isSSHAuthFailure(err error) bool { + return strings.Contains(err.Error(), "unable to authenticate") +} + +func isSSHNetworkError(err error) bool { + var opErr *net.OpError + return errors.As(err, &opErr) +} + +// SSHTargetFromExecutorConfig projects an SSH executor's persisted config map +// into a resolved SSHTarget, applying the same ~/.ssh/config inheritance as +// ResolveSSHTarget. A missing host or fingerprint fails resolution here, +// which callers should classify as SSHReachabilityReasonConfig without +// attempting a dial. +func SSHTargetFromExecutorConfig(cfg map[string]string) (*SSHTarget, error) { + if cfg == nil { + return nil, fmt.Errorf("ssh executor has no config") + } + fingerprint := strings.TrimSpace(cfg["ssh_host_fingerprint"]) + if fingerprint == "" { + return nil, fmt.Errorf("ssh host fingerprint is required for reachability probes") + } + port := 0 + if p := strings.TrimSpace(cfg["ssh_port"]); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n < 1 || n > 65535 { + return nil, fmt.Errorf("invalid ssh_port %q", p) + } + port = n + } + return ResolveSSHTarget(SSHConnConfig{ + HostAlias: cfg["ssh_host_alias"], + Host: cfg["ssh_host"], + Port: port, + User: cfg["ssh_user"], + IdentitySource: SSHIdentitySource(cfg["ssh_identity_source"]), + IdentityFile: cfg["ssh_identity_file"], + ProxyJump: cfg["ssh_proxy_jump"], + PinnedFingerprint: fingerprint, + }) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go new file mode 100644 index 0000000000..f993adaf5d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go @@ -0,0 +1,372 @@ +package lifecycle + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +// fakeTimeoutError is a minimal net.Error whose Timeout() reports true, +// wrapped inside a *net.OpError below to model a timeout that is also +// structurally shaped like a network dial error — @covers +// AC-EXECUTORS-SSH-REACHABILITY-001.27 (timeout must win the ordering, not +// fall through to network). +type fakeTimeoutError struct{} + +func (fakeTimeoutError) Error() string { return "fake timeout" } +func (fakeTimeoutError) Timeout() bool { return true } +func (fakeTimeoutError) Temporary() bool { return true } + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.27 — classification order +// config, timeout, host_key, auth, network, unknown, first match wins, +// including the two explicit ordering cases. +func TestClassifyDialError(t *testing.T) { + tests := []struct { + name string + err error + want SSHReachabilityReason + }{ + { + name: "a context deadline classifies as timeout", + err: fmt.Errorf("ssh: tcp dial x: %w", context.DeadlineExceeded), + want: SSHReachabilityReasonTimeout, + }, + { + name: "a timeout-shaped dial error wins over the network fallback", + err: &net.OpError{Op: "dial", Net: "tcp", Err: fakeTimeoutError{}}, + want: SSHReachabilityReasonTimeout, + }, + { + name: "a pinned host key mismatch classifies as host_key", + err: fmt.Errorf("ssh: handshake with x: %w", &errHostKeyMismatch{Expected: "A", Got: "B"}), + want: SSHReachabilityReasonHostKey, + }, + { + name: "a bastion known_hosts mismatch classifies as host_key", + err: fmt.Errorf("ssh: bastion dial: %w", &knownhosts.KeyError{Want: []knownhosts.KnownKey{{}}}), + want: SSHReachabilityReasonHostKey, + }, + { + name: "host_key wins over an auth-shaped message in the same error", + err: fmt.Errorf("ssh: handshake failed: ssh: unable to authenticate: %w", + &errHostKeyMismatch{Expected: "A", Got: "B"}), + want: SSHReachabilityReasonHostKey, + }, + { + name: "an authentication failure classifies as auth", + err: errors.New("ssh: unable to authenticate, attempted methods [publickey], no supported methods remain"), + want: SSHReachabilityReasonAuth, + }, + { + name: "a connection-refused dial error classifies as network", + err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")}, + want: SSHReachabilityReasonNetwork, + }, + { + name: "an unrecognized error classifies as unknown", + err: errors.New("something unexpected happened"), + want: SSHReachabilityReasonUnknown, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ClassifyDialError(tt.err); got != tt.want { + t.Fatalf("ClassifyDialError(%v) = %q, want %q", tt.err, got, tt.want) + } + }) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.4, 001.5 +func TestProbeSSHHostSuccess(t *testing.T) { + server := newFakeSSHServer(t, nil) + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: server.hostKeyFingerprint(), + } + + outcome := ProbeSSHHost(context.Background(), target, 5*time.Second) + + if !outcome.Success { + t.Fatalf("outcome = %+v, want Success", outcome) + } + if outcome.Host != "127.0.0.1" { + t.Fatalf("Host = %q", outcome.Host) + } + if outcome.Reason != "" || outcome.Message != "" { + t.Fatalf("a successful outcome must carry no reason/message: %+v", outcome) + } + if len(server.execCalls()) != 0 { + t.Fatalf("execCalls = %v, want the probe to open no session channel", server.execCalls()) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.5 +func TestProbeSSHHostHostKeyMismatch(t *testing.T) { + server := newFakeSSHServer(t, nil) + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: "SHA256:not-the-server-key", + } + + outcome := ProbeSSHHost(context.Background(), target, 5*time.Second) + + if outcome.Success { + t.Fatalf("outcome = %+v, want failure", outcome) + } + if outcome.Reason != SSHReachabilityReasonHostKey { + t.Fatalf("Reason = %q, want host_key", outcome.Reason) + } + if target.PinnedFingerprint != "SHA256:not-the-server-key" { + t.Fatalf("PinnedFingerprint mutated: %q", target.PinnedFingerprint) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.6 +func TestProbeSSHHostEmptyPinDoesNotDial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + dialed := false + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + conn, err := ln.Accept() + if err == nil { + dialed = true + _ = conn.Close() + } + }() + t.Cleanup(func() { + _ = ln.Close() + wg.Wait() + }) + addr := ln.Addr().(*net.TCPAddr) + + target := &SSHTarget{ + Host: addr.IP.String(), Port: addr.Port, User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: "/does/not/matter", + PinnedFingerprint: "", + } + + outcome := ProbeSSHHost(context.Background(), target, 5*time.Second) + + if outcome.Reason != SSHReachabilityReasonConfig { + t.Fatalf("Reason = %q, want config", outcome.Reason) + } + if outcome.Success { + t.Fatal("outcome must not be a success") + } + _ = ln.Close() + wg.Wait() + if dialed { + t.Fatal("ProbeSSHHost dialed TCP despite an empty pinned fingerprint") + } +} + +// newStallHandshakeListener accepts exactly one TCP connection and holds it +// open without ever writing the SSH version banner, so a client's handshake +// blocks on read until its own deadline fires. Used to prove the probe +// bounds the handshake, not only the TCP dial — @covers +// AC-EXECUTORS-SSH-REACHABILITY-001.7. +func newStallHandshakeListener(t *testing.T) *net.TCPAddr { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + conn, err := ln.Accept() + if err != nil { + return + } + <-done + _ = conn.Close() + }() + t.Cleanup(func() { + close(done) + _ = ln.Close() + wg.Wait() + }) + return ln.Addr().(*net.TCPAddr) +} + +func TestProbeSSHHostHandshakeStallIsBoundedByTheDeadline(t *testing.T) { + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + addr := newStallHandshakeListener(t) + + target := &SSHTarget{ + Host: addr.IP.String(), Port: addr.Port, User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: "SHA256:irrelevant", + } + + const probeTimeout = 300 * time.Millisecond + start := time.Now() + done := make(chan SSHProbeOutcome, 1) + go func() { done <- ProbeSSHHost(context.Background(), target, probeTimeout) }() + + select { + case outcome := <-done: + elapsed := time.Since(start) + if elapsed > 5*time.Second { + t.Fatalf("probe took %s, want it bounded near the %s deadline", elapsed, probeTimeout) + } + if outcome.Reason != SSHReachabilityReasonTimeout { + t.Fatalf("Reason = %q, want timeout; message=%q", outcome.Reason, outcome.Message) + } + case <-time.After(5 * time.Second): + t.Fatal("ProbeSSHHost did not return within 5s of a 300ms deadline — handshake is unbounded") + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.29 +func TestProbeSSHHostBastionHostKeyMismatchClassifiesAsHostKey(t *testing.T) { + home := newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + final := newFakeSSHServer(t, nil) + bastion := newFakeSSHServer(t, nil) + + _, otherPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate other key: %v", err) + } + otherSigner, err := ssh.NewSignerFromKey(otherPriv) + if err != nil { + t.Fatalf("other signer: %v", err) + } + writeKnownHosts(t, home, fmt.Sprintf("[127.0.0.1]:%d", bastion.port()), otherSigner.PublicKey()) + + proxyJump := "jump@127.0.0.1:" + strconv.Itoa(bastion.port()) + target := &SSHTarget{ + Host: "127.0.0.1", Port: final.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: final.hostKeyFingerprint(), + ProxyJump: proxyJump, + } + + outcome := ProbeSSHHost(context.Background(), target, 5*time.Second) + + if outcome.Reason != SSHReachabilityReasonHostKey { + t.Fatalf("Reason = %q, want host_key; message=%q", outcome.Reason, outcome.Message) + } + if outcome.Host != "127.0.0.1" { + t.Fatalf("Host = %q, want the target host, not the bastion", outcome.Host) + } + if !strings.Contains(outcome.Message, "bastion") { + t.Fatalf("Message = %q, want it to name the bastion", outcome.Message) + } +} + +func TestProbeSSHHostCancelledContextProducesNoObservation(t *testing.T) { + server := newFakeSSHServer(t, nil) + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: server.hostKeyFingerprint(), + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + outcome := ProbeSSHHost(ctx, target, 5*time.Second) + + if !outcome.Cancelled { + t.Fatalf("outcome = %+v, want Cancelled", outcome) + } + if outcome.Success { + t.Fatal("a cancelled probe must not report success") + } + if outcome.Reason != "" { + t.Fatalf("Reason = %q, want no reason for a cancelled probe", outcome.Reason) + } +} + +func TestSSHTargetFromExecutorConfig(t *testing.T) { + newTestHomeDir(t) + + t.Run("a valid config resolves", func(t *testing.T) { + target, err := SSHTargetFromExecutorConfig(map[string]string{ + "ssh_host": "target.internal", + "ssh_port": "2222", + "ssh_user": "deploy", + "ssh_identity_source": "file", + "ssh_identity_file": "/keys/id_ed25519", + "ssh_host_fingerprint": "SHA256:abc", + }) + if err != nil { + t.Fatalf("SSHTargetFromExecutorConfig: %v", err) + } + if target.Host != "target.internal" || target.Port != 2222 || target.User != "deploy" { + t.Fatalf("target = %+v", target) + } + if target.PinnedFingerprint != "SHA256:abc" { + t.Fatalf("PinnedFingerprint = %q", target.PinnedFingerprint) + } + }) + + t.Run("a nil config fails resolution", func(t *testing.T) { + if _, err := SSHTargetFromExecutorConfig(nil); err == nil { + t.Fatal("expected an error for a nil config") + } + }) + + t.Run("an invalid port fails resolution", func(t *testing.T) { + if _, err := SSHTargetFromExecutorConfig(map[string]string{ + "ssh_host": "target.internal", + "ssh_port": "not-a-port", + }); err == nil { + t.Fatal("expected an error for an invalid port") + } + }) + + t.Run("a missing pinned fingerprint fails resolution", func(t *testing.T) { + if _, err := SSHTargetFromExecutorConfig(map[string]string{ + "ssh_host": "target.internal", + "ssh_user": "deploy", + }); err == nil { + t.Fatal("expected an error when the pinned host fingerprint is missing") + } + }) +} + +func TestSanitizeSSHReachabilityMessageBoundsAndRedacts(t *testing.T) { + secret := "ssh: unable to authenticate, password=super-secret-token " + strings.Repeat("x", 700) + got := SanitizeSSHReachabilityMessage(errors.New(secret)) + if len(got) > maxSSHReachabilityMessageBytes { + t.Fatalf("message length = %d, want at most %d", len(got), maxSSHReachabilityMessageBytes) + } + if strings.Contains(got, "super-secret-token") { + t.Fatalf("message contains the raw credential: %q", got) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager.go b/apps/backend/internal/agent/runtime/lifecycle/manager.go index a171ab30b2..0462dc504c 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager.go @@ -250,6 +250,20 @@ type Manager struct { // SkillDeployer hook. Nil → skill deploy is skipped. agentProfileReader AgentProfileReader + // reachabilityReader resolves an ssh executor's stored reachability + // record for the launch-time session.launch.warning producer. Nil → + // no warning is ever published (feature not wired). See + // manager_launch_reachability_warning.go and SetSSHReachabilityWarningPolicy. + reachabilityReader ReachabilityReader + // reachabilityProbingEnabled mirrors whether the reachability poller's + // periodic sweep is on (interval != 0). When true, a stored unreachable + // record is always warning-eligible regardless of how old checked_at is. + reachabilityProbingEnabled bool + // reachabilityWarningWindowSeconds is 3x the reachability package's own + // default interval (not the configured/effective one), evaluated even + // with probing disabled per AC-EXECUTORS-SSH-REACHABILITY-001.28. + reachabilityWarningWindowSeconds int + // skillDeployer materialises per-profile skills + custom prompt before // the agent process starts. Defaults to a no-op deployer; office wires // its concrete implementation via SetSkillDeployer. diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go index 032177561e..ebba2b6ff9 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go @@ -1096,6 +1096,8 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st return nil, nil, nil, err } + m.scheduleSSHLaunchWarning(launchCtx, reqWithWorktree, metadata, reqWithWorktree.SessionID) + execInstance, err := rt.CreateInstance(launchCtx, execReq) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create execution: %w", err) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning.go new file mode 100644 index 0000000000..d267d1239f --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning.go @@ -0,0 +1,145 @@ +package lifecycle + +import ( + "context" + "strings" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +// ReachabilityReader is the narrow read accessor launchBuildExecutorRequest +// uses to look up an ssh executor's stored reachability record immediately +// before calling CreateInstance. It is declared here — not imported from +// executors/reachability — because that package already imports lifecycle +// for SSH target resolution and dial classification; importing it back here +// would cycle. In production this is satisfied directly by the task +// repository, which already implements this exact method for the +// reachability HTTP routes (see internal/ssh.ReachabilityLister). +type ReachabilityReader interface { + GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) +} + +// SetSSHReachabilityWarningPolicy wires the launch-time session.launch.warning +// producer (see task-05-launch-non-gating.md). reader resolves an ssh +// executor's stored record; probingEnabled mirrors whether the reachability +// poller's periodic sweep is on; warningWindowSeconds is 3x the reachability +// package's own default interval, evaluated even with probing disabled. +// Leaving this unset (reader nil) disables the producer entirely. +func (m *Manager) SetSSHReachabilityWarningPolicy(reader ReachabilityReader, probingEnabled bool, warningWindowSeconds int) { + m.reachabilityReader = reader + m.reachabilityProbingEnabled = probingEnabled + m.reachabilityWarningWindowSeconds = warningWindowSeconds +} + +// LaunchWarningEventPayload is the session.launch.warning payload. It is +// published once, immediately before an SSH launch's CreateInstance call, +// when the target executor's stored reachability record is unreachable and +// still within the probing window. +type LaunchWarningEventPayload struct { + TaskID string `json:"task_id"` + SessionID string `json:"session_id"` + ExecutorID string `json:"executor_id"` + Host string `json:"host"` + State string `json:"state"` + Reason string `json:"reason"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + Timestamp string `json:"timestamp"` +} + +// sshLaunchWarningReadTimeout bounds the best-effort record lookup started by +// a launch. The lookup must not hold up CreateInstance, but it also must not +// leave a database call running forever after a launch has finished. +const sshLaunchWarningReadTimeout = 2 * time.Second + +// GetSessionID satisfies the session-routing accessor +// internal/gateway/websocket.extractSessionID uses to broadcast this event +// to the correct session's subscribers. +func (p LaunchWarningEventPayload) GetSessionID() string { + return p.SessionID +} + +// PublishLaunchWarning publishes a session.launch.warning event to the +// launched session's own event stream. +func (p *EventPublisher) PublishLaunchWarning(sessionID string, payload *LaunchWarningEventPayload) { + if p.eventBus == nil { + return + } + if payload.Timestamp == "" { + payload.Timestamp = time.Now().UTC().Format(time.RFC3339Nano) + } + subject := events.BuildSessionLaunchWarningSubject(sessionID) + event := bus.NewEvent(events.SessionLaunchWarning, "lifecycle", payload) + if err := p.eventBus.Publish(context.Background(), subject, event); err != nil { + p.logger.Error("failed to publish session launch warning event", + zap.String("session_id", sessionID), + zap.Error(err)) + } +} + +// scheduleSSHLaunchWarning starts the best-effort record lookup without +// delaying CreateInstance. The request and metadata values used by the +// goroutine are copied because the launch path may reuse or mutate its input +// maps after it returns. +func (m *Manager) scheduleSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) { + if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) || + m.reachabilityReader == nil || m.eventPublisher == nil { + return + } + warningReq := *req + warningMetadata := map[string]interface{}{ + "executor_id": getMetadataString(metadata, "executor_id"), + } + warningCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sshLaunchWarningReadTimeout) + go func() { + defer cancel() + m.maybePublishSSHLaunchWarning(warningCtx, &warningReq, warningMetadata, sessionID) + }() +} + +// maybePublishSSHLaunchWarning reads the target executor's reachability +// record and, when it is unreachable and still warning-eligible, publishes +// exactly one session.launch.warning. It is a strict read-only accessor call, +// made from launchBuildExecutorRequest alongside its single CreateInstance +// call site, never from inside CreateInstance itself. A read failure or a +// missing record produces no warning. +func (m *Manager) maybePublishSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) { + if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) || + m.reachabilityReader == nil || m.eventPublisher == nil { + return + } + executorID := strings.TrimSpace(getMetadataString(metadata, "executor_id")) + if executorID == "" { + return + } + record, err := m.reachabilityReader.GetExecutorReachability(ctx, executorID) + if err != nil || record == nil || record.State != models.ExecutorReachabilityStateUnreachable { + return + } + if !m.reachabilityProbingEnabled && !withinReachabilityWarningWindow(record.CheckedAt, m.reachabilityWarningWindowSeconds) { + return + } + m.eventPublisher.PublishLaunchWarning(sessionID, &LaunchWarningEventPayload{ + TaskID: req.TaskID, + SessionID: sessionID, + ExecutorID: executorID, + Host: record.Host, + State: string(record.State), + Reason: string(record.Reason), + LastSuccessAt: record.LastSuccessAt, + }) +} + +// withinReachabilityWarningWindow reports whether checkedAt is recent enough +// to still warn even with periodic probing disabled. A nil checkedAt (never +// probed) or a non-positive window never qualifies. +func withinReachabilityWarningWindow(checkedAt *time.Time, windowSeconds int) bool { + if checkedAt == nil || windowSeconds <= 0 { + return false + } + return time.Since(*checkedAt) <= time.Duration(windowSeconds)*time.Second +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go new file mode 100644 index 0000000000..a4201ad718 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go @@ -0,0 +1,305 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +type fakeReachabilityReader struct { + records map[string]*models.ExecutorReachability + err error + calls []string +} + +func (f *fakeReachabilityReader) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) { + f.calls = append(f.calls, executorID) + if f.err != nil { + return nil, f.err + } + return f.records[executorID], nil +} + +// failOnReadReachabilityReader fails the test the moment it is touched at +// all — used to prove a code path performs no reachability read, rather than +// merely counting reads after the fact. +type failOnReadReachabilityReader struct { + t *testing.T +} + +func (f failOnReadReachabilityReader) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) { + f.t.Fatal("reachability record was read from a code path that must never read one") + return nil, nil +} + +type blockingReachabilityReader struct { + started chan struct{} + release chan struct{} +} + +func (r *blockingReachabilityReader) GetExecutorReachability(ctx context.Context, _ string) (*models.ExecutorReachability, error) { + close(r.started) + select { + case <-r.release: + return nil, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func newWarningTestManager(t *testing.T, reader ReachabilityReader, probingEnabled bool, windowSeconds int) (*Manager, *bus.MemoryEventBus) { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("new logger: %v", err) + } + memBus := bus.NewMemoryEventBus(log) + mgr := &Manager{ + logger: log, + eventPublisher: NewEventPublisher(memBus, log), + } + mgr.SetSSHReachabilityWarningPolicy(reader, probingEnabled, windowSeconds) + return mgr, memBus +} + +func subscribeLaunchWarnings(t *testing.T, memBus *bus.MemoryEventBus, sessionID string) <-chan *bus.Event { + t.Helper() + received := make(chan *bus.Event, 1) + sub, err := memBus.Subscribe(events.BuildSessionLaunchWarningSubject(sessionID), func(_ context.Context, ev *bus.Event) error { + received <- ev + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + return received +} + +const warningWindowSeconds = 180 // 3x reachability.DefaultIntervalSeconds + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_UnreachableAndProbingEnabledWarns(t *testing.T) { + old := time.Now().Add(-time.Hour) + lastSuccess := old.Add(-time.Hour) + reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{ + "executor-1": { + ExecutorID: "executor-1", + State: models.ExecutorReachabilityStateUnreachable, + Reason: models.ExecutorReachabilityReasonTimeout, + Host: "build.example", + CheckedAt: &old, + LastSuccessAt: &lastSuccess, + }, + }} + mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeSSH), + TaskID: "task-1", + SessionID: "session-1", + } + metadata := map[string]interface{}{"executor_id": "executor-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, metadata, req.SessionID) + + select { + case ev := <-received: + payload, ok := ev.Data.(*LaunchWarningEventPayload) + if !ok { + t.Fatalf("payload type = %T", ev.Data) + } + if payload.ExecutorID != "executor-1" || payload.Host != "build.example" || + payload.State != string(models.ExecutorReachabilityStateUnreachable) || + payload.Reason != string(models.ExecutorReachabilityReasonTimeout) || + payload.LastSuccessAt == nil || !payload.LastSuccessAt.Equal(lastSuccess) { + t.Fatalf("payload = %+v", payload) + } + default: + t.Fatal("expected a session.launch.warning event, got none") + } + if len(reader.calls) != 1 || reader.calls[0] != "executor-1" { + t.Fatalf("reader.calls = %v", reader.calls) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_ProbingDisabledButRecentlyCheckedWarns(t *testing.T) { + recent := time.Now().Add(-time.Minute) + reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{ + "executor-1": { + ExecutorID: "executor-1", + State: models.ExecutorReachabilityStateUnreachable, + Reason: models.ExecutorReachabilityReasonNetwork, + Host: "build.example", + CheckedAt: &recent, + }, + }} + mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + + select { + case <-received: + default: + t.Fatal("expected a session.launch.warning event when checked_at is within the warning window") + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_ProbingDisabledAndStaleCheckedAtDoesNotWarn(t *testing.T) { + stale := time.Now().Add(-time.Hour) + reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{ + "executor-1": { + ExecutorID: "executor-1", + State: models.ExecutorReachabilityStateUnreachable, + Host: "build.example", + CheckedAt: &stale, + }, + }} + mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + + select { + case ev := <-received: + t.Fatalf("expected no warning for a stale checked_at with probing disabled, got %+v", ev) + default: + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_ReachableRecordDoesNotWarn(t *testing.T) { + reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{ + "executor-1": {ExecutorID: "executor-1", State: models.ExecutorReachabilityStateReachable, Host: "build.example"}, + }} + mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + + select { + case ev := <-received: + t.Fatalf("expected no warning for a reachable record, got %+v", ev) + default: + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_NoRecordDoesNotWarn(t *testing.T) { + reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{}} + mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + + select { + case ev := <-received: + t.Fatalf("expected no warning when no record exists yet, got %+v", ev) + default: + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3 +func TestMaybePublishSSHLaunchWarning_ReadFailureDoesNotWarnOrPanic(t *testing.T) { + reader := &fakeReachabilityReader{err: errors.New("boom")} + mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds) + received := subscribeLaunchWarnings(t, memBus, "session-1") + + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + + select { + case ev := <-received: + t.Fatalf("expected no warning on a read failure, got %+v", ev) + default: + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// Non-ssh launches must never trigger a reachability read at all — the +// failOnReadReachabilityReader fails the test the instant it is touched, +// which is what proves the short-circuit rather than merely observing zero +// warnings (a bug that read the record but decided not to warn would still +// pass a call-count-only assertion). +func TestMaybePublishSSHLaunchWarning_NonSSHExecutorNeverReads(t *testing.T) { + mgr, _ := newWarningTestManager(t, failOnReadReachabilityReader{t: t}, true, warningWindowSeconds) + + req := &LaunchRequest{ExecutorType: "local_docker", SessionID: "session-1"} + mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) +} + +func TestScheduleSSHLaunchWarningDoesNotWaitForRead(t *testing.T) { + reader := &blockingReachabilityReader{started: make(chan struct{}), release: make(chan struct{})} + mgr, _ := newWarningTestManager(t, reader, true, warningWindowSeconds) + req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"} + + started := time.Now() + mgr.scheduleSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID) + if elapsed := time.Since(started); elapsed > 100*time.Millisecond { + t.Fatalf("scheduleSSHLaunchWarning waited %v for a blocked read", elapsed) + } + select { + case <-reader.started: + case <-time.After(time.Second): + t.Fatal("warning read did not start asynchronously") + } + close(reader.release) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// launchBuildExecutorRequest is the single production call site for every +// executor type (verified: it has exactly one caller, launchInternal, which +// every launch entry point — WS-initiated, resume, and workflow/dependency +// launches via orchestrator/executor's LaunchAgent — funnels through). This +// exercises that shared call site directly with a non-ssh backend and proves +// the reachability reader is never touched and CreateInstance still runs. +func TestLaunchBuildExecutorRequestNonSSHExecutorNeverReadsReachability(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameDocker}} + registry.Register(backend) + + memBusLog, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + memBus := bus.NewMemoryEventBus(memBusLog) + mgr := &Manager{ + executorRegistry: registry, + executorFallbackPolicy: ExecutorFallbackDeny, + logger: log, + eventPublisher: NewEventPublisher(memBus, memBusLog), + } + mgr.SetSSHReachabilityWarningPolicy(failOnReadReachabilityReader{t: t}, true, warningWindowSeconds) + + req := &LaunchRequest{ + ExecutorType: "local_docker", + EnvironmentFinalized: true, + SessionID: "session-1", + Metadata: map[string]interface{}{"executor_id": "executor-1"}, + } + + _, _, _, err := mgr.launchBuildExecutorRequest( + context.Background(), "instance-1", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil, + ) + + if err != nil { + t.Fatalf("launchBuildExecutorRequest: %v", err) + } + if backend.lastRequest == nil { + t.Fatal("expected CreateInstance to run") + } +} diff --git a/apps/backend/internal/agent/runtime/ssh_reachability_probe.go b/apps/backend/internal/agent/runtime/ssh_reachability_probe.go new file mode 100644 index 0000000000..253861f930 --- /dev/null +++ b/apps/backend/internal/agent/runtime/ssh_reachability_probe.go @@ -0,0 +1,92 @@ +package runtime + +import ( + "context" + "time" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" +) + +// SSHReachabilityReason classifies why an SSH reachability probe failed. It +// mirrors lifecycle.SSHReachabilityReason so callers outside the runtime +// package never import the lifecycle tier directly. +type SSHReachabilityReason string + +const ( + SSHReachabilityReasonConfig SSHReachabilityReason = "config" + SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout" + SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key" + SSHReachabilityReasonAuth SSHReachabilityReason = "auth" + SSHReachabilityReasonNetwork SSHReachabilityReason = "network" + SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown" +) + +// SSHProbeOutcome is the runtime-facade mirror of lifecycle.SSHProbeOutcome. +type SSHProbeOutcome struct { + Success bool + Cancelled bool + Host string + Reason SSHReachabilityReason + Message string +} + +// SanitizeSSHReachabilityMessage exposes the runtime-safe SSH diagnostic +// projection without making higher-level adapters import lifecycle directly. +func SanitizeSSHReachabilityMessage(err error) string { + return lifecycle.SanitizeSSHReachabilityMessage(err) +} + +// SSHReachabilityProber is the runtime seam for a single SSH reachability +// probe. Higher-level packages depend on this contract instead of the +// lifecycle implementation details. +type SSHReachabilityProber interface { + // Probe resolves an SSH executor's target from its persisted config and + // probes it, bounded by timeout. A resolution failure (missing host, + // invalid port, unresolvable alias) never dials — it is reported as + // SSHReachabilityReasonConfig directly, the same reason a target with no + // pinned fingerprint uses. + Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome + // ValidateConfig reports whether config resolves to a dialable SSH + // target, without dialing it. + ValidateConfig(config map[string]string) error +} + +type sshReachabilityProber struct { + resolveTarget func(map[string]string) (*lifecycle.SSHTarget, error) + probe func(context.Context, *lifecycle.SSHTarget, time.Duration) lifecycle.SSHProbeOutcome +} + +// NewSSHReachabilityProber creates the production adapter over the lifecycle +// SSH target resolver and reachability prober. +func NewSSHReachabilityProber() SSHReachabilityProber { + return &sshReachabilityProber{ + resolveTarget: lifecycle.SSHTargetFromExecutorConfig, + probe: lifecycle.ProbeSSHHost, + } +} + +func (a *sshReachabilityProber) Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome { + target, err := a.resolveTarget(config) + if err != nil { + return SSHProbeOutcome{ + Host: config["ssh_host"], + Reason: SSHReachabilityReasonConfig, + Message: lifecycle.SanitizeSSHReachabilityMessage(err), + } + } + outcome := a.probe(ctx, target, timeout) + return SSHProbeOutcome{ + Success: outcome.Success, + Cancelled: outcome.Cancelled, + Host: outcome.Host, + Reason: SSHReachabilityReason(outcome.Reason), + Message: outcome.Message, + } +} + +func (a *sshReachabilityProber) ValidateConfig(config map[string]string) error { + _, err := a.resolveTarget(config) + return err +} + +var _ SSHReachabilityProber = (*sshReachabilityProber)(nil) diff --git a/apps/backend/internal/backendapp/canvas_edit_test.go b/apps/backend/internal/backendapp/canvas_edit_test.go index 8a6ebb7c0a..8fd9604371 100644 --- a/apps/backend/internal/backendapp/canvas_edit_test.go +++ b/apps/backend/internal/backendapp/canvas_edit_test.go @@ -220,32 +220,32 @@ func stringContains(value, want string) bool { func newCanvasEditTestDependencies() (*fakeCanvasEditTaskStore, *fakeCanvasEditCanvasStore, *fakeCanvasEditReleaseStore, *fakeCanvasEditArtifactStore) { return &fakeCanvasEditTaskStore{ - workspace: &models.Workspace{ - ID: "workspace-a", - DefaultAgentProfileID: stringPointer("profile-default"), - DefaultExecutorID: stringPointer("executor-default"), - }, - }, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{ - ID: "canvas-a", - PluginInstanceID: "instance-a", - WorkspaceID: "workspace-a", - ScopeKind: canvas.ScopeWorkspace, - Status: instances.StatusActive, - ActiveReleaseID: "release-a", - ActiveReleaseStatus: instances.ValidationValid, - EffectiveGrants: []canvas.GrantProjection{{ - PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace, - }}, - }}, &fakeCanvasEditReleaseStore{release: instances.Release{ - ID: "release-a", - InstanceID: "instance-a", - PackageDigest: "digest-a", - ArtifactPath: "releases/digest-a", - ValidationStatus: instances.ValidationValid, - }}, &fakeCanvasEditArtifactStore{files: map[string][]byte{ - "manifest.yaml": []byte("manifest"), - "index.html": []byte("index"), - }} + workspace: &models.Workspace{ + ID: "workspace-a", + DefaultAgentProfileID: stringPointer("profile-default"), + DefaultExecutorID: stringPointer("executor-default"), + }, + }, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{ + ID: "canvas-a", + PluginInstanceID: "instance-a", + WorkspaceID: "workspace-a", + ScopeKind: canvas.ScopeWorkspace, + Status: instances.StatusActive, + ActiveReleaseID: "release-a", + ActiveReleaseStatus: instances.ValidationValid, + EffectiveGrants: []canvas.GrantProjection{{ + PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace, + }}, + }}, &fakeCanvasEditReleaseStore{release: instances.Release{ + ID: "release-a", + InstanceID: "instance-a", + PackageDigest: "digest-a", + ArtifactPath: "releases/digest-a", + ValidationStatus: instances.ValidationValid, + }}, &fakeCanvasEditArtifactStore{files: map[string][]byte{ + "manifest.yaml": []byte("manifest"), + "index.html": []byte("index"), + }} } func stringPointer(value string) *string { return &value } diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 588fe23650..b7b3a8f20d 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -46,6 +46,7 @@ import ( editorhandlers "github.com/kandev/kandev/internal/editors/handlers" "github.com/kandev/kandev/internal/entityrefs" "github.com/kandev/kandev/internal/events/bus" + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" "github.com/kandev/kandev/internal/failedinbox" gateways "github.com/kandev/kandev/internal/gateway/websocket" "github.com/kandev/kandev/internal/github" @@ -729,6 +730,7 @@ type routeParams struct { planCoalesceWindowConfigured bool homeDir string interimSettingsInterlockToken string + sshReachabilityPoller *reachabilitypkg.Poller log *logger.Logger progress *startup.Reporter } @@ -1467,6 +1469,14 @@ func registerSecondaryRoutes( ) p.log.Debug("Registered Kubernetes handlers (HTTP + WebSocket)") + // A nil *reachabilitypkg.Poller must not be passed directly as the + // sshhandlers.ReachabilityProber interface parameter — that would + // produce a non-nil interface holding a nil pointer, defeating the + // handler's own nil check and panicking on first use. + var reachabilityPoller sshhandlers.ReachabilityProber + if p.sshReachabilityPoller != nil { + reachabilityPoller = p.sshReachabilityPoller + } sshhandlers.RegisterRoutes( p.router, p.gateway.Dispatcher, @@ -1475,6 +1485,8 @@ func registerSecondaryRoutes( p.agentRegistry, lifecycle.NewAgentctlResolver(p.log), p.log, + p.taskRepo, + reachabilityPoller, ) p.log.Debug("Registered SSH handlers (HTTP + WebSocket)") } diff --git a/apps/backend/internal/backendapp/httpserver_test.go b/apps/backend/internal/backendapp/httpserver_test.go index 3edf27a60b..7ff5d76c91 100644 --- a/apps/backend/internal/backendapp/httpserver_test.go +++ b/apps/backend/internal/backendapp/httpserver_test.go @@ -22,7 +22,7 @@ func TestBuildHTTPServerAbortsWhenInterlockTokenGenerationFails(t *testing.T) { newInterimSettingsInterlockToken = func() (string, error) { return "", wantErr } t.Cleanup(func() { newInterimSettingsInterlockToken = original }) - server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) if !errors.Is(err, wantErr) { t.Fatalf("buildHTTPServer error = %v, want %v", err, wantErr) } diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index daa9a1250d..202e813320 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -43,6 +43,9 @@ import ( "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" + // SSH executor reachability poller + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" + // GitHub integration azuredevopspkg "github.com/kandev/kandev/internal/azuredevops" githubpkg "github.com/kandev/kandev/internal/github" @@ -881,6 +884,36 @@ func startAgentInfrastructure( log.Info("Office config sync poller started") } + // Start SSH executor reachability poller: sweeps every eligible SSH + // executor on a configurable interval, probing reachability and + // persisting results through a hysteresis-owning write path. Wired with + // a publisher (state/reason changes reach WS clients) and registered as + // the task service's executor-save observer (a changed host resets the + // record and dispatches an off-cycle probe) so both the poller and the + // reachability HTTP routes below share the one running instance. + sshReachabilityPoller := startSSHReachabilityPoller( + ctx, + repos.Task, + cfg.Executors.SSHReachabilityIntervalSeconds, + log, + reachabilitypkg.NewPublisher(eventBus, log), + addRuntimeCleanup, + ) + services.Task.SetExecutorSaveObserver(reachabilitypkg.NewSaveObserver(sshReachabilityPoller)) + + // Launch-time session.launch.warning producer (task 05): repos.Task + // already implements the narrow read accessor (same method used by the + // reachability HTTP routes). probingEnabled mirrors the poller's own + // effective interval so a configured 0 (disabled) keeps the warning + // gated on staleness alone. warningWindowSeconds is 3x the reachability + // package's own default interval, not the configured one, per + // AC-EXECUTORS-SSH-REACHABILITY-001.28. + lifecycleMgr.SetSSHReachabilityWarningPolicy( + repos.Task, + sshReachabilityPoller.EffectiveIntervalSeconds() != 0, + 3*reachabilitypkg.DefaultIntervalSeconds, + ) + // Start the plugin system's event delivery and health monitor // background loops. if services.Plugins != nil { @@ -943,7 +976,7 @@ func startAgentInfrastructure( return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services, agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, sessionCapacityEnvironment, storageStore, func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelWorkers, - restoreCleanups, databaseQuiesce) + restoreCleanups, databaseQuiesce, sshReachabilityPoller) } // startOrchestratorAndAutomationConsumers establishes the startup chain in @@ -1013,6 +1046,7 @@ func startGatewayAndServe( cancelWorkers context.CancelFunc, restoreCleanups []func() error, databaseQuiesce func() error, + sshReachabilityPoller *reachabilitypkg.Poller, ) bool { // ============================================ // WEBSOCKET GATEWAY @@ -1350,7 +1384,7 @@ func startGatewayAndServe( builtServer, err := buildHTTPServer(cfg, log, gateway, repos, services, agentSettingsController, lifecycleMgr, eventBus, orchestratorSvc, notificationCtrl, msgCreator, agentRegistry, hostUtilityMgr, addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer, - storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, startup.FromContext(ctx), persistenceHealth) + storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, sshReachabilityPoller, startup.FromContext(ctx), persistenceHealth) if err != nil { log.Error("Failed to build HTTP server", zap.Error(err)) closeBoundListeners(server, listeners, log) @@ -2599,6 +2633,7 @@ func buildHTTPServer( temporaryArtifacts *tempartifacts.Registry, dbPool *db.Pool, agentRuntimeAvailability *agentctlclient.Availability, + sshReachabilityPoller *reachabilitypkg.Poller, progress *startup.Reporter, persistenceHealth ...*requiredstores.Health, ) (*http.Server, error) { @@ -2730,6 +2765,7 @@ func buildHTTPServer( planCoalesceWindowConfigured: true, homeDir: cfg.ResolvedHomeDir(), interimSettingsInterlockToken: interimSettingsInterlockToken, + sshReachabilityPoller: sshReachabilityPoller, log: log, progress: progress, }) diff --git a/apps/backend/internal/backendapp/ssh_reachability_wiring.go b/apps/backend/internal/backendapp/ssh_reachability_wiring.go new file mode 100644 index 0000000000..b83ed6f13b --- /dev/null +++ b/apps/backend/internal/backendapp/ssh_reachability_wiring.go @@ -0,0 +1,26 @@ +package backendapp + +import ( + "context" + + "github.com/kandev/kandev/internal/common/logger" + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" +) + +// startSSHReachabilityPoller launches the SSH executor reachability poller +// and registers its Stop on addCleanup, mirroring the third-party +// integration poller wiring alongside it in startAgentInfrastructure. +func startSSHReachabilityPoller( + ctx context.Context, + repo reachabilitypkg.Repository, + intervalSeconds int, + log *logger.Logger, + publisher *reachabilitypkg.Publisher, + addCleanup func(func() error) func() error, +) *reachabilitypkg.Poller { + poller := reachabilitypkg.New(repo, intervalSeconds, log) + poller.SetPublisher(publisher) + poller.Start(ctx) + addCleanup(func() error { poller.Stop(); return nil }) + return poller +} diff --git a/apps/backend/internal/backendapp/ssh_reachability_wiring_test.go b/apps/backend/internal/backendapp/ssh_reachability_wiring_test.go new file mode 100644 index 0000000000..580e28153c --- /dev/null +++ b/apps/backend/internal/backendapp/ssh_reachability_wiring_test.go @@ -0,0 +1,108 @@ +package backendapp + +import ( + "context" + "go/ast" + "go/parser" + "go/token" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" + "github.com/kandev/kandev/internal/task/models" +) + +type sshReachabilityWiringRepo struct { + listed chan struct{} +} + +func (r *sshReachabilityWiringRepo) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) { + select { + case r.listed <- struct{}{}: + default: + } + return nil, nil +} + +func (r *sshReachabilityWiringRepo) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) { + return nil, models.ErrExecutorReachabilityNotFound +} + +func (r *sshReachabilityWiringRepo) UpsertExecutorReachability(context.Context, models.ExecutorReachabilityObservation) error { + return nil +} + +func (r *sshReachabilityWiringRepo) ResetExecutorReachability(context.Context, string, string, time.Time) error { + return nil +} + +// TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper mirrors +// TestInitOfficeServicesUsesTaskUsageWiringHelper: the composition root +// (startAgentInfrastructure, too large to exercise directly in a unit test) +// must delegate to the narrow, independently-testable wiring helper exactly +// once rather than inlining the poller construction. +func TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "main.go", nil, 0) + if err != nil { + t.Fatalf("parse main.go: %v", err) + } + + var target *ast.FuncDecl + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if ok && fn.Name.Name == "startAgentInfrastructure" { + target = fn + break + } + } + if target == nil { + t.Fatal("startAgentInfrastructure not found in main.go") + } + + var helperCalls int + ast.Inspect(target, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if name, ok := call.Fun.(*ast.Ident); ok && name.Name == "startSSHReachabilityPoller" { + helperCalls++ + } + return true + }) + if helperCalls != 1 { + t.Fatalf("startSSHReachabilityPoller calls = %d, want exactly one composition call", helperCalls) + } +} + +func TestStartSSHReachabilityPoller_StartsPollerAndRegistersCleanup(t *testing.T) { + repo := &sshReachabilityWiringRepo{listed: make(chan struct{}, 1)} + var cleanups []func() error + addCleanup := func(fn func() error) func() error { + cleanups = append(cleanups, fn) + return fn + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + poller := startSSHReachabilityPoller(ctx, repo, 3600, logger.Default(), (*reachabilitypkg.Publisher)(nil), addCleanup) + if poller == nil { + t.Fatal("startSSHReachabilityPoller returned a nil poller") + } + if len(cleanups) != 1 { + t.Fatalf("cleanups registered = %d, want exactly 1", len(cleanups)) + } + + select { + case <-repo.listed: + case <-time.After(time.Second): + t.Fatal("poller never ran its immediate pass") + } + + if err := cleanups[0](); err != nil { + t.Fatalf("cleanup: %v", err) + } +} diff --git a/apps/backend/internal/common/config/catalog.go b/apps/backend/internal/common/config/catalog.go index 6e006e312a..4b2928c058 100644 --- a/apps/backend/internal/common/config/catalog.go +++ b/apps/backend/internal/common/config/catalog.go @@ -111,6 +111,7 @@ var startupCatalog = []CatalogEntry{ {Key: "launcher.webPort", EnvVars: []string{"KANDEV_WEB_PORT"}, Owner: "launcher", Default: "automatic"}, {Key: "launcher.healthTimeoutMs", EnvVars: []string{"KANDEV_HEALTH_TIMEOUT_MS"}, Owner: "launcher", Default: "45000"}, {Key: "launcher.noBrowser", EnvVars: []string{"KANDEV_NO_BROWSER"}, Owner: "launcher", Default: "false"}, + {Key: "executors.sshReachabilityIntervalSeconds", EnvVars: []string{"KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS"}, Owner: "executors/reachability", Default: "60"}, } var startupExclusions = []CatalogExclusion{ diff --git a/apps/backend/internal/common/config/catalog_test.go b/apps/backend/internal/common/config/catalog_test.go index 8af524a9c0..9daae7328d 100644 --- a/apps/backend/internal/common/config/catalog_test.go +++ b/apps/backend/internal/common/config/catalog_test.go @@ -134,6 +134,7 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment { {envVar: "KANDEV_WEB_PORT", class: "catalog"}, {envVar: "KANDEV_HEALTH_TIMEOUT_MS", class: "catalog"}, {envVar: "KANDEV_NO_BROWSER", class: "catalog"}, + {envVar: "KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", class: "catalog"}, {envVar: InternalConfigFileEnv, class: "exclusion"}, {envVar: InternalConfigHomeFileEnv, class: "exclusion"}, {envVar: InternalAgentctlStartupConfigEnv, class: "exclusion"}, diff --git a/apps/backend/internal/common/config/config.go b/apps/backend/internal/common/config/config.go index 5238658e8a..dbb66f1167 100644 --- a/apps/backend/internal/common/config/config.go +++ b/apps/backend/internal/common/config/config.go @@ -53,6 +53,7 @@ type Config struct { Office OfficeConfig `mapstructure:"office"` Features FeaturesConfig `mapstructure:"features"` GitHubCredentialBroker GitHubCredentialBrokerConfig `mapstructure:"githubCredentialBroker"` + Executors ExecutorsConfig `mapstructure:"executors"` Source ConfigSource `mapstructure:"-" json:"-"` } @@ -71,6 +72,15 @@ type CredentialsConfig struct { File string `mapstructure:"file"` } +// ExecutorsConfig contains executor-related startup settings. +type ExecutorsConfig struct { + // SSHReachabilityIntervalSeconds is the raw configured value, not yet + // clamped into the reachability package's own 15-3600 bound: 0 disables + // the poller, and reachability.ClampInterval owns the rest of the + // normalization at construction time. + SSHReachabilityIntervalSeconds int `mapstructure:"sshReachabilityIntervalSeconds"` +} + // LimitsConfig contains process and protocol capacity limits. type LimitsConfig struct { GHMaxConcurrent int `mapstructure:"ghMaxConcurrent"` diff --git a/apps/backend/internal/common/config/source.go b/apps/backend/internal/common/config/source.go index aea8ea96f1..f7a1efb06d 100644 --- a/apps/backend/internal/common/config/source.go +++ b/apps/backend/internal/common/config/source.go @@ -65,28 +65,29 @@ type configSelection struct { // AutomaticEnv during Unmarshal: legacy invalid environment values must retain // their fallback behavior instead of turning into typed-config startup errors. var yamlOnlyStartupKeys = map[string]struct{}{ - "server.trustedProxies": {}, - "tasks.preparationTimeout": {}, - "tasks.stallDetectionThreshold": {}, - "credentials.file": {}, - "limits.ghMaxConcurrent": {}, - "limits.gitMaxConcurrent": {}, - "limits.lspMaxConnections": {}, - "messageQueue.maxPerSession": {}, - "agentctl.idleTimeout": {}, - "agentctl.idleReaperInterval": {}, - "agentctl.notificationQueueCapacity": {}, - "agentctl.recoveryDeadline": {}, - "agentctl.recoveryReadTimeout": {}, - "agentctl.recoveryReadRetries": {}, - "agentctl.unownedPeriod": {}, - "agentctl.detachedEventLimit": {}, - "planning.coalesceWindowMs": {}, - "office.schedulerTickMs": {}, - "observability.otlpEndpoint": {}, - "launcher.webPort": {}, - "launcher.healthTimeoutMs": {}, - "launcher.noBrowser": {}, + "server.trustedProxies": {}, + "tasks.preparationTimeout": {}, + "tasks.stallDetectionThreshold": {}, + "credentials.file": {}, + "limits.ghMaxConcurrent": {}, + "limits.gitMaxConcurrent": {}, + "limits.lspMaxConnections": {}, + "messageQueue.maxPerSession": {}, + "agentctl.idleTimeout": {}, + "agentctl.idleReaperInterval": {}, + "agentctl.notificationQueueCapacity": {}, + "agentctl.recoveryDeadline": {}, + "agentctl.recoveryReadTimeout": {}, + "agentctl.recoveryReadRetries": {}, + "agentctl.unownedPeriod": {}, + "agentctl.detachedEventLimit": {}, + "planning.coalesceWindowMs": {}, + "office.schedulerTickMs": {}, + "observability.otlpEndpoint": {}, + "launcher.webPort": {}, + "launcher.healthTimeoutMs": {}, + "launcher.noBrowser": {}, + "executors.sshReachabilityIntervalSeconds": {}, } func isYAMLOnlyStartupKey(key string) bool { @@ -313,6 +314,7 @@ func applyStartupDefaults(cfg *Config, yamlKeys map[string]bool, profileDefaults if !yamlKeys["launcher.noBrowser"] { cfg.Launcher.NoBrowser = false } + setDefaultInt("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60) } func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources map[string]SettingSource) { @@ -343,6 +345,7 @@ func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources applyBoundedIntEnv("launcher.webPort", &cfg.Launcher.WebPort, 0, 1, 65535, envSnapshot, sources) applyPositiveIntEnv("launcher.healthTimeoutMs", &cfg.Launcher.HealthTimeoutMs, launcherHealthTimeoutDefault(), envSnapshot, sources) applyBoolEnv("launcher.noBrowser", &cfg.Launcher.NoBrowser, false, envSnapshot, sources) + applyNonNegativeIntEnv("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60, envSnapshot, sources) } func launcherHealthTimeoutDefault() int { diff --git a/apps/backend/internal/common/config/source_test.go b/apps/backend/internal/common/config/source_test.go index 562964eb7a..ca2df3ca42 100644 --- a/apps/backend/internal/common/config/source_test.go +++ b/apps/backend/internal/common/config/source_test.go @@ -307,6 +307,74 @@ func TestHomeConfigInvalidFirstDoesNotFallThrough(t *testing.T) { } } +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14 +// +// The catalog's own fallback-and-bounds logic only ever falls back to the +// default (60): the 0-disables/15-3600-clamp behavior belongs to the +// reachability package's own ClampInterval, not this layer, per the design's +// split between catalog-level normalization and package-level clamping. +func TestExecutorsSSHReachabilityIntervalSecondsEnvironment(t *testing.T) { + tests := []struct { + name string + env string + wantValue int + wantSource SettingSource + }{ + {"absent uses default", "", 60, SourceDefault}, + {"valid value passes through unclamped", "45", 45, SourceEnvironment}, + {"zero passes through as the disable sentinel", "0", 0, SourceEnvironment}, + {"negative falls back to the default", "-5", 60, SourceDefault}, + {"unparsable falls back to the default", "not-a-number", 60, SourceDefault}, + {"a value outside 15-3600 still passes through unclamped here", "9999", 9999, SourceEnvironment}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env != "" { + t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", tt.env) + } else { + t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "") + } + t.Setenv("KANDEV_SERVER_PORT", "") + + cfg, err := LoadWithPath(t.TempDir()) + if err != nil { + t.Fatalf("LoadWithPath: %v", err) + } + if cfg.Executors.SSHReachabilityIntervalSeconds != tt.wantValue { + t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want %d", cfg.Executors.SSHReachabilityIntervalSeconds, tt.wantValue) + } + if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != tt.wantSource { + t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, tt.wantSource) + } + }) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14 +// +// A YAML value is not passed through the environment's fallback-and-bounds +// logic at all — it decodes straight into the typed field, negative values +// included, leaving ClampInterval as the sole place that normalizes it. +func TestExecutorsSSHReachabilityIntervalSecondsYAMLPassesThroughUnclamped(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte("executors:\n sshReachabilityIntervalSeconds: -5\n"), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "") + t.Setenv("KANDEV_SERVER_PORT", "") + + cfg, err := LoadWithPath(dir) + if err != nil { + t.Fatalf("LoadWithPath: %v", err) + } + if cfg.Executors.SSHReachabilityIntervalSeconds != -5 { + t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want -5 (unclamped)", cfg.Executors.SSHReachabilityIntervalSeconds) + } + if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != SourceConfiguration { + t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, SourceConfiguration) + } +} + func nestedField(t *testing.T, value any, names ...string) reflect.Value { t.Helper() current := reflect.ValueOf(value) diff --git a/apps/backend/internal/events/types.go b/apps/backend/internal/events/types.go index d43d3c9a05..72a2a77948 100644 --- a/apps/backend/internal/events/types.go +++ b/apps/backend/internal/events/types.go @@ -192,6 +192,10 @@ const ( ExecutorCreated = "executor.created" ExecutorUpdated = "executor.updated" ExecutorDeleted = "executor.deleted" + // ExecutorReachabilityChanged is published only when a probe or a + // connection-configuration reset actually changes the stored state or + // reason — a steady host never publishes. + ExecutorReachabilityChanged = "executor.reachability.changed" ) // Event types for executor profiles @@ -310,6 +314,16 @@ const ( AvailableCommandsUpdated = "available_commands.updated" // Available slash commands updated ) +// Event types for session launch warnings +const ( + // SessionLaunchWarning is published once, immediately before an SSH + // launch's CreateInstance call, when the target executor's stored + // reachability record is unreachable and the record is still within the + // probing window. It carries Kandev's own attribution of the target + // host, independent of whatever an agent process itself reports. + SessionLaunchWarning = "session.launch.warning" +) + // Event types for session mode const ( SessionModeChanged = "session_mode.changed" // Agent session mode changed @@ -512,6 +526,16 @@ func BuildAgentCapabilitiesSubject(sessionID string) string { return AgentCapabilitiesUpdated + "." + sessionID } +// BuildSessionLaunchWarningSubject creates a session launch warning subject for a specific session +func BuildSessionLaunchWarningSubject(sessionID string) string { + return SessionLaunchWarning + "." + sessionID +} + +// BuildSessionLaunchWarningWildcardSubject creates a wildcard subscription for all session launch warning events +func BuildSessionLaunchWarningWildcardSubject() string { + return SessionLaunchWarning + ".*" +} + // BuildAgentCapabilitiesWildcardSubject creates a wildcard subscription for all agent capabilities events func BuildAgentCapabilitiesWildcardSubject() string { return AgentCapabilitiesUpdated + ".*" diff --git a/apps/backend/internal/executors/reachability/goleak_test.go b/apps/backend/internal/executors/reachability/goleak_test.go new file mode 100644 index 0000000000..52c35b9e7a --- /dev/null +++ b/apps/backend/internal/executors/reachability/goleak_test.go @@ -0,0 +1,15 @@ +package reachability + +import ( + "testing" + + "go.uber.org/goleak" +) + +// TestMain asserts that no goroutines from this package outlive the test +// process. The Poller owns the scheduled loop plus any off-cycle ProbeNow +// call, all registered on one WaitGroup — goleak catches a Start path that +// forgets to register, or a Stop path that returns before every probe drains. +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/apps/backend/internal/executors/reachability/interval.go b/apps/backend/internal/executors/reachability/interval.go new file mode 100644 index 0000000000..b88584a0d0 --- /dev/null +++ b/apps/backend/internal/executors/reachability/interval.go @@ -0,0 +1,51 @@ +// Package reachability sweeps every eligible SSH executor on a configurable +// interval, probing each with bounded concurrency and persisting the result +// through a hysteresis-owning write path. See +// docs/specs/executors/system-design/ssh-reachability.md. +package reachability + +import "time" + +const ( + // probeTimeout bounds a single probe's TCP dial and SSH handshake. + // sshDialTimeout is 30s; a shorter bound here keeps one hung host from + // holding a semaphore slot for half the interval. + probeTimeout = 10 * time.Second + // passConcurrency bounds simultaneous outbound connections and file + // descriptors while keeping a pass short. + passConcurrency = 4 + // failureThreshold is the smallest value that suppresses a single-probe + // blip; worst-case detection is two intervals. + failureThreshold = 2 + + // DefaultIntervalSeconds is the effective interval used when the + // configured value is absent, negative, fractional, or unparsable. + DefaultIntervalSeconds = 60 + // MinIntervalSeconds and MaxIntervalSeconds bound the poller's cadence. + // 0 is a separate "disabled" sentinel and is never clamped into this + // range. + MinIntervalSeconds = 15 + MaxIntervalSeconds = 3600 +) + +// ClampInterval applies this package's bounds to a raw configured interval +// (in seconds) and reports whether the value was adjusted — worth one +// startup log line rather than a refused boot. The environment path in +// internal/common/config already normalizes an absent, empty, fractional, or +// unparsable value to DefaultIntervalSeconds; a YAML value reaches here +// unnormalized; either way this function still owns the below-minimum, +// above-maximum, and negative cases directly. +func ClampInterval(seconds int) (effective int, clamped bool) { + switch { + case seconds == 0: + return 0, false + case seconds < 0: + return DefaultIntervalSeconds, true + case seconds < MinIntervalSeconds: + return MinIntervalSeconds, true + case seconds > MaxIntervalSeconds: + return MaxIntervalSeconds, true + default: + return seconds, false + } +} diff --git a/apps/backend/internal/executors/reachability/interval_test.go b/apps/backend/internal/executors/reachability/interval_test.go new file mode 100644 index 0000000000..bbca4d32db --- /dev/null +++ b/apps/backend/internal/executors/reachability/interval_test.go @@ -0,0 +1,31 @@ +package reachability + +import "testing" + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.23, AC-EXECUTORS-SSH-REACHABILITY-001.24 +func TestClampInterval(t *testing.T) { + tests := []struct { + name string + in int + wantEffect int + wantClamped bool + }{ + {"zero disables and is not a clamp", 0, 0, false}, + {"negative falls back to the default", -5, DefaultIntervalSeconds, true}, + {"below minimum clamps up to 15", 1, MinIntervalSeconds, true}, + {"just below minimum clamps up to 15", 14, MinIntervalSeconds, true}, + {"minimum passes through unchanged", 15, 15, false}, + {"default passes through unchanged", 60, 60, false}, + {"maximum passes through unchanged", 3600, 3600, false}, + {"just above maximum clamps down to 3600", 3601, MaxIntervalSeconds, true}, + {"far above maximum clamps down to 3600", 100000, MaxIntervalSeconds, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + effective, clamped := ClampInterval(tt.in) + if effective != tt.wantEffect || clamped != tt.wantClamped { + t.Errorf("ClampInterval(%d) = (%d, %v), want (%d, %v)", tt.in, effective, clamped, tt.wantEffect, tt.wantClamped) + } + }) + } +} diff --git a/apps/backend/internal/executors/reachability/metrics.go b/apps/backend/internal/executors/reachability/metrics.go new file mode 100644 index 0000000000..ae5fa67e92 --- /dev/null +++ b/apps/backend/internal/executors/reachability/metrics.go @@ -0,0 +1,60 @@ +package reachability + +import ( + "expvar" + "strings" +) + +// expvar counters exposed via /debug/vars, following the office_stall_* and +// routing_* precedent: "key=value;key=value..." labels so a Prometheus +// translation layer can split on ';' and '='. Every name carries the +// executor_ssh_reachability_ prefix. +var ( + probeTotal = expvar.NewMap("executor_ssh_reachability_probe_total") + stateTransitionsTotal = expvar.NewMap("executor_ssh_reachability_state_transitions_total") + passSkippedTotal = expvar.NewInt("executor_ssh_reachability_pass_skipped_total") + writeRefusedTotal = expvar.NewInt("executor_ssh_reachability_write_refused_total") + readBackFailedTotal = expvar.NewInt("executor_ssh_reachability_read_back_failed_total") + publishFailedTotal = expvar.NewInt("executor_ssh_reachability_publish_failed_total") + resetTotal = expvar.NewInt("executor_ssh_reachability_reset_total") + probeDiscardedTotal = expvar.NewInt("executor_ssh_reachability_probe_discarded_total") + probeDurationMsLastRun = expvar.NewInt("executor_ssh_reachability_probe_duration_ms") +) + +// reachabilityLabel builds a "k1=v1;k2=v2;..." expvar map key. Returns an +// empty label for an odd number of arguments rather than guessing. +func reachabilityLabel(pairs ...string) string { + if len(pairs)%2 != 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2) + for i := 0; i < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + return strings.Join(parts, ";") +} + +// recordProbeOutcome increments probeTotal labelled by outcome: "reachable" +// on success, or the failure reason string otherwise. +func recordProbeOutcome(success bool, reason string) { + outcome := reason + if success { + outcome = "reachable" + } + probeTotal.Add(reachabilityLabel("outcome", outcome), 1) +} + +// recordStateTransition increments stateTransitionsTotal labelled by the +// destination state. +func recordStateTransition(destination string) { + stateTransitionsTotal.Add(reachabilityLabel("state", destination), 1) +} + +// RecordReset increments resetTotal. This package never calls +// ResetExecutorReachability itself — task 04's save observer is the only +// caller of that method — but the counter is exported here so it lives with +// the rest of this package's reachability metrics rather than being +// duplicated at the call site. +func RecordReset() { + resetTotal.Add(1) +} diff --git a/apps/backend/internal/executors/reachability/poller.go b/apps/backend/internal/executors/reachability/poller.go new file mode 100644 index 0000000000..63707e58e4 --- /dev/null +++ b/apps/backend/internal/executors/reachability/poller.go @@ -0,0 +1,297 @@ +package reachability + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" + "golang.org/x/sync/singleflight" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" +) + +// probeFunc is the injectable seam over a single probe attempt. defaultProbe +// is the production implementation; tests substitute a fake so poller +// lifecycle/concurrency behavior can be verified under testing/synctest +// without real SSH or network I/O. +type probeFunc func(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome + +// Poller sweeps every eligible SSH executor on intervalSeconds, probing each +// with bounded concurrency and persisting results through store. An +// intervalSeconds of 0 disables the scheduled sweep entirely — the poller +// still starts (so ProbeNow keeps working) but never runs a pass on its own. +// +// Start and Stop are idempotent. Stop cancels every in-flight unit of work +// registered on wg — the scheduled loop, the pass it is running, and any +// off-cycle ProbeNow call — and waits for all of them to drain before +// returning, so a probe cancelled by Stop is guaranteed to never be +// persisted (see probeAndPersist). +type Poller struct { + store *store + log *logger.Logger + intervalSeconds int + probe probeFunc + // publisher is optional: nil until SetPublisher wires the event bus, so + // task 03's own tests and callers that never call it keep working + // unchanged. Set once at construction time in production. + publisher *Publisher + + // inflight coalesces concurrent ProbeAndWait callers for the same + // executor id into a single dial. Zero value is ready to use. + inflight singleflight.Group + + // probeSem is owned by the poller rather than by a scheduled pass. This + // keeps scheduled, manual, and synchronous probes under one cap. + probeSem chan struct{} + + // mu guards started/stopping/ctx/cancel/wg against concurrent + // Start/Stop/ProbeNow calls. acquire() registers new work on wg under mu + // so it can never race a concurrent Stop's wg.Wait() observing the + // counter transition from zero. + mu sync.Mutex + started bool + stopping bool + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + stopDone chan struct{} + + // passMu guards passRunning, the single-flight gate for the scheduled + // pass: an overlapping tick is dropped and counted, never queued. + passMu sync.Mutex + passRunning bool +} + +// New builds a Poller. intervalSeconds is clamped via ClampInterval; a +// clamp is logged once rather than refusing to start. +func New(repo Repository, intervalSeconds int, log *logger.Logger) *Poller { + effective, clamped := ClampInterval(intervalSeconds) + if clamped { + log.Warn("executor ssh reachability: interval clamped", + zap.Int("configured", intervalSeconds), zap.Int("effective", effective)) + } + return &Poller{ + store: &store{repo: repo, log: log}, + log: log, + intervalSeconds: effective, + probe: defaultProbe, + probeSem: make(chan struct{}, passConcurrency), + } +} + +// SetPublisher wires the event publisher used to announce a state/reason +// change. Optional — a Poller with no publisher still probes and persists +// exactly as before, it just never publishes. +func (p *Poller) SetPublisher(publisher *Publisher) { + p.publisher = publisher +} + +// Start launches the background loop. Calling Start more than once without +// an intervening Stop is a no-op. +func (p *Poller) Start(ctx context.Context) { + p.mu.Lock() + if p.started { + p.mu.Unlock() + return + } + runCtx, cancel := context.WithCancel(ctx) + p.started = true + p.stopping = false + p.ctx = runCtx + p.cancel = cancel + p.stopDone = make(chan struct{}) + if p.intervalSeconds > 0 { + p.wg.Add(1) + } + p.mu.Unlock() + + if p.intervalSeconds > 0 { + go p.loop(runCtx) + } +} + +// Stop cancels the loop and every in-flight probe, then waits for all of +// them to drain. Calling Stop before Start, or twice in a row, is a no-op. +func (p *Poller) Stop() { + p.mu.Lock() + if !p.started { + p.mu.Unlock() + return + } + stopDone := p.stopDone + if p.stopping { + p.mu.Unlock() + <-stopDone + return + } + p.stopping = true + cancel := p.cancel + p.mu.Unlock() + + cancel() + p.wg.Wait() + + p.mu.Lock() + p.started = false + p.stopping = false + p.ctx = nil + p.cancel = nil + if p.stopDone == stopDone { + close(stopDone) + p.stopDone = nil + } + p.mu.Unlock() +} + +// ProbeNow runs one off-cycle probe for executor, outside the scheduled +// cadence, tracked on the same WaitGroup/context as the scheduled loop so +// Stop drains it too. Returns false without doing anything when the poller +// is not currently accepting new work. +func (p *Poller) ProbeNow(executor *models.Executor) bool { + ctx, ok := p.acquire() + if !ok { + return false + } + go func() { + defer p.wg.Done() + p.probeAndPersist(ctx, executor) + }() + return true +} + +func (p *Poller) acquireProbeSlot(ctx context.Context) bool { + select { + case p.probeSem <- struct{}{}: + return true + case <-ctx.Done(): + return false + } +} + +func (p *Poller) releaseProbeSlot() { + <-p.probeSem +} + +// acquire registers one unit of in-flight work on wg, gated by mu so it can +// never race a concurrent Stop's wg.Wait(). It refuses when the poller isn't +// started or a Stop is already underway. +func (p *Poller) acquire() (context.Context, bool) { + p.mu.Lock() + defer p.mu.Unlock() + if !p.started || p.stopping { + return nil, false + } + // loop already owns a WaitGroup reference, so this positive Add cannot + // race a Stop waiting for the counter to reach zero. + // Safe without passMu: dispatchPass is called only from loop, which holds + // its own WaitGroup slot, so Stop cannot observe zero before this Add. + p.wg.Add(1) + return p.ctx, true +} + +func (p *Poller) loop(ctx context.Context) { + defer p.wg.Done() + + ticker := time.NewTicker(time.Duration(p.intervalSeconds) * time.Second) + defer ticker.Stop() + + p.dispatchPass(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + p.dispatchPass(ctx) + } + } +} + +// dispatchPass starts one pass if none is currently running. An overlapping +// dispatch is dropped and counted rather than queued, so a slow pass never +// builds up a backlog of pending passes. The pass itself runs on its own +// WaitGroup unit so Stop drains it even after loop itself has returned. +func (p *Poller) dispatchPass(ctx context.Context) { + p.passMu.Lock() + if p.passRunning { + p.passMu.Unlock() + passSkippedTotal.Add(1) + return + } + p.passRunning = true + p.passMu.Unlock() + + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { + p.passMu.Lock() + p.passRunning = false + p.passMu.Unlock() + }() + p.runPass(ctx) + }() +} + +// runPass lists every eligible executor and probes each with bounded +// concurrency. A listing failure abandons the pass without writing anything. +// Cancellation stops dispatching new probes and waits only for the ones +// already started to finish or self-cancel. +func (p *Poller) runPass(ctx context.Context) { + start := time.Now() + executors, err := p.store.repo.ListSSHExecutorsForReachability(ctx) + if err != nil { + p.log.Warn("executor ssh reachability: list failed", zap.Error(err)) + return + } + + var wg sync.WaitGroup + for _, executor := range executors { + if !p.acquireProbeSlot(ctx) { + wg.Wait() + return + } + wg.Add(1) + go func(executor *models.Executor) { + defer wg.Done() + defer p.releaseProbeSlot() + p.probeAndPersistWithSlot(ctx, executor) + }(executor) + } + wg.Wait() + probeDurationMsLastRun.Set(time.Since(start).Milliseconds()) +} + +// probeAndPersist runs one probe and records it, unless the probe was +// cancelled — a cancelled outcome is not an observation about the host and +// must never be written (see agentruntime.SSHProbeOutcome.Cancelled). +func (p *Poller) probeAndPersist(ctx context.Context, executor *models.Executor) { + if !p.acquireProbeSlot(ctx) { + return + } + defer p.releaseProbeSlot() + p.probeAndPersistWithSlot(ctx, executor) +} + +func (p *Poller) probeAndPersistWithSlot(ctx context.Context, executor *models.Executor) { + outcome := p.probe(ctx, executor) + if outcome.Cancelled { + probeDiscardedTotal.Add(1) + return + } + p.observeAndPublish(ctx, executor, outcome) +} + +// observeAndPublish records outcome through store.Observe and, when the +// write actually changed the stored state or reason, announces it through +// the publisher — the one propagation path every probe primitive +// (scheduled pass, ProbeNow, ProbeAndWait) shares. +func (p *Poller) observeAndPublish(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome) observeResult { + result := p.store.Observe(ctx, executor, outcome, time.Now().UTC()) + if result.Changed && p.publisher != nil { + p.publisher.PublishChanged(ctx, result.After, p.EffectiveIntervalSeconds()) + } + return result +} diff --git a/apps/backend/internal/executors/reachability/poller_publish_test.go b/apps/backend/internal/executors/reachability/poller_publish_test.go new file mode 100644 index 0000000000..b9a82f3d63 --- /dev/null +++ b/apps/backend/internal/executors/reachability/poller_publish_test.go @@ -0,0 +1,70 @@ +package reachability + +import ( + "context" + "testing" + "time" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// A scheduled-pass probe (ProbeNow's own code path, probeAndPersist) must +// publish exactly like the immediate-probe route on a real change, and stay +// silent on a repeat that changes nothing — the "one propagation path" rule +// from the system design applies regardless of which primitive ran the +// probe. +func TestPollerProbeNow_PublishesOnlyOnChange(t *testing.T) { + eventBus := bus.NewMemoryEventBus(logger.Default()) + published := make(chan RecordDTO, 4) + _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error { + published <- ev.Data.(RecordDTO) + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + repo := newFakeRepository() + p := New(repo, 60, logger.Default()) + p.SetPublisher(NewPublisher(eventBus)) + p.Start(context.Background()) + defer p.Stop() + + executor := sshExecutor("exec-publish") + + // First success: unknown -> reachable is a change. + p.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]} + } + if !p.ProbeNow(executor) { + t.Fatal("ProbeNow refused") + } + select { + case dto := <-published: + if dto.ExecutorID != executor.ID || dto.State != "reachable" { + t.Fatalf("dto = %+v, want reachable exec-publish", dto) + } + case <-time.After(time.Second): + t.Fatal("no event published for the first, change-carrying probe") + } + + // Second success, same outcome: reachable -> reachable is not a change. + if !p.ProbeNow(executor) { + t.Fatal("ProbeNow refused") + } + // Stop drains the second ProbeNow's probeAndPersist goroutine (it is + // registered on the same WaitGroup) before we check for the absence of a + // publish — a happens-before barrier instead of a raw sleep. + p.Stop() + select { + case dto := <-published: + t.Fatalf("unexpected event published for a repeat outcome: %+v", dto) + default: + } +} diff --git a/apps/backend/internal/executors/reachability/poller_test.go b/apps/backend/internal/executors/reachability/poller_test.go new file mode 100644 index 0000000000..a1f4c1f7f4 --- /dev/null +++ b/apps/backend/internal/executors/reachability/poller_test.go @@ -0,0 +1,445 @@ +package reachability + +import ( + "context" + "errors" + "sort" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" +) + +// fakeRepository is an in-memory Repository double for poller tests: it +// isolates poller mechanics (scheduling, concurrency, lifecycle) from the +// SQL-backed hysteresis contract, which store_test.go already covers +// end-to-end against a real database. +type fakeRepository struct { + mu sync.Mutex + executors []*models.Executor + records map[string]*models.ExecutorReachability + listErr error + upsertErr error + + upsertCalls int32 +} + +// errRepoRefused simulates a repository-level write failure — a test double +// for the last-write-wins WHERE clause silently refusing a write, without +// needing a real SQLite race to reproduce it. +var errRepoRefused = errors.New("write refused") + +func newFakeRepository(executors ...*models.Executor) *fakeRepository { + return &fakeRepository{executors: executors, records: map[string]*models.ExecutorReachability{}} +} + +func (f *fakeRepository) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.listErr != nil { + return nil, f.listErr + } + out := append([]*models.Executor(nil), f.executors...) + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (f *fakeRepository) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) { + f.mu.Lock() + defer f.mu.Unlock() + rec, ok := f.records[executorID] + if !ok { + return nil, models.ErrExecutorReachabilityNotFound + } + cp := *rec + return &cp, nil +} + +func (f *fakeRepository) UpsertExecutorReachability(_ context.Context, obs models.ExecutorReachabilityObservation) error { + f.mu.Lock() + defer f.mu.Unlock() + atomic.AddInt32(&f.upsertCalls, 1) + if f.upsertErr != nil { + return f.upsertErr + } + state := obs.InitialState + failures := obs.InitialFailures + if existing, ok := f.records[obs.ExecutorID]; ok { + if obs.Reason == "" { + failures = 0 + state = models.ExecutorReachabilityStateReachable + } else { + failures = existing.ConsecutiveFailures + 1 + state = existing.State + if isStickyReason(obs.Reason) || failures >= obs.FailureThreshold { + state = models.ExecutorReachabilityStateUnreachable + } + } + } + f.records[obs.ExecutorID] = &models.ExecutorReachability{ + ExecutorID: obs.ExecutorID, State: state, Reason: obs.Reason, Message: obs.Message, + ConsecutiveFailures: failures, Host: obs.Host, CheckedAt: timePtr(obs.CheckedAt), UpdatedAt: time.Now().UTC(), + } + return nil +} + +func (f *fakeRepository) ResetExecutorReachability(_ context.Context, executorID, host string, _ time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + f.records[executorID] = &models.ExecutorReachability{ExecutorID: executorID, State: models.ExecutorReachabilityStateUnknown, Host: host, UpdatedAt: time.Now().UTC()} + return nil +} + +func (f *fakeRepository) upsertCount() int32 { return atomic.LoadInt32(&f.upsertCalls) } + +func timePtr(t time.Time) *time.Time { return &t } + +func sshExecutor(id string) *models.Executor { + return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive, Config: map[string]string{"ssh_host": id}} +} + +// concurrencyProbe blocks on a gate until released, tracking the maximum +// number of simultaneous in-flight calls it observed. +type concurrencyProbe struct { + mu sync.Mutex + inFlight int + maxSeen int + release chan struct{} +} + +func newConcurrencyProbe() *concurrencyProbe { + return &concurrencyProbe{release: make(chan struct{})} +} + +func (c *concurrencyProbe) probe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + c.mu.Lock() + c.inFlight++ + if c.inFlight > c.maxSeen { + c.maxSeen = c.inFlight + } + c.mu.Unlock() + + select { + case <-c.release: + case <-ctx.Done(): + c.mu.Lock() + c.inFlight-- + c.mu.Unlock() + return agentruntime.SSHProbeOutcome{Host: executor.ID, Cancelled: true} + } + + c.mu.Lock() + c.inFlight-- + c.mu.Unlock() + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} +} + +func TestPollerStart_RunsImmediatePass(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a"), sshExecutor("b")) + p := New(repo, 60, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + defer p.Stop() + + synctest.Wait() + + if got := repo.upsertCount(); got != 2 { + t.Fatalf("upsert calls = %d, want 2 (one immediate pass over two executors)", got) + } + }) +} + +func TestPollerStart_IsIdempotent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 60, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.Start(ctx) // second call must be a no-op + + synctest.Wait() + p.Stop() + + if got := repo.upsertCount(); got != 1 { + t.Fatalf("upsert calls = %d, want exactly 1 (a second Start must not spawn a parallel loop)", got) + } + }) +} + +func TestPollerStop_BeforeStart_IsNoOp(t *testing.T) { + p := New(newFakeRepository(), 60, logger.Default()) + p.Stop() // must not block or panic +} + +func TestPollerZeroInterval_DisablesScheduledPass(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 0, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + defer p.Stop() + + synctest.Wait() + // Advance well past what would have been several ticks at the + // default interval, to prove no pass ever runs. + time.Sleep(10 * time.Minute) + synctest.Wait() + + if got := repo.upsertCount(); got != 0 { + t.Fatalf("upsert calls = %d, want 0 (interval=0 must run no scheduled pass)", got) + } + }) +} + +func TestPollerZeroInterval_ProbeNowStillWorks(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 0, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + defer p.Stop() + synctest.Wait() + + if ok := p.ProbeNow(sshExecutor("a")); !ok { + t.Fatal("ProbeNow returned false while the poller is started") + } + synctest.Wait() + + if got := repo.upsertCount(); got != 1 { + t.Fatalf("upsert calls = %d, want 1 from the off-cycle probe", got) + } + }) +} + +func TestPollerPass_NeverExceedsPassConcurrency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var executors []*models.Executor + for i := 0; i < passConcurrency+3; i++ { + executors = append(executors, sshExecutor(string(rune('a'+i)))) + } + repo := newFakeRepository(executors...) + p := New(repo, 60, logger.Default()) + cp := newConcurrencyProbe() + p.probe = cp.probe + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + // Let every worker reach the gate. + synctest.Wait() + close(cp.release) + p.Stop() + + cp.mu.Lock() + defer cp.mu.Unlock() + if cp.maxSeen > passConcurrency { + t.Fatalf("max simultaneous probes = %d, want at most passConcurrency (%d)", cp.maxSeen, passConcurrency) + } + if cp.maxSeen != passConcurrency { + t.Fatalf("max simultaneous probes = %d, want exactly passConcurrency (%d) since more executors than slots were queued", cp.maxSeen, passConcurrency) + } + }) +} + +func TestPollerProbeEntrypointsShareConcurrencyLimit(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var executors []*models.Executor + for i := 0; i < passConcurrency; i++ { + executors = append(executors, sshExecutor(string(rune('a'+i)))) + } + repo := newFakeRepository(executors...) + p := New(repo, 60, logger.Default()) + cp := newConcurrencyProbe() + p.probe = cp.probe + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + synctest.Wait() + + if ok := p.ProbeNow(sshExecutor("manual")); !ok { + t.Fatal("ProbeNow returned false while the poller was running") + } + synctest.Wait() + + cp.mu.Lock() + maxSeen := cp.maxSeen + cp.mu.Unlock() + close(cp.release) + p.Stop() + if maxSeen > passConcurrency { + t.Fatalf("max simultaneous probes = %d, want at most %d across scheduled and manual probes", maxSeen, passConcurrency) + } + }) +} + +func TestPollerConcurrentStopWaitsForTheFirstStop(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 0, logger.Default()) + release := make(chan struct{}) + var releaseOnce sync.Once + var inFlight atomic.Int32 + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + inFlight.Add(1) + defer inFlight.Add(-1) + <-release + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.ID} + } + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + p.Start(context.Background()) + if ok := p.ProbeNow(sshExecutor("stop-race")); !ok { + t.Fatal("ProbeNow returned false while the poller was running") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if inFlight.Load() == 1 { + break + } + time.Sleep(time.Millisecond) + } + + firstDone := make(chan struct{}) + go func() { p.Stop(); close(firstDone) }() + for time.Now().Before(deadline) { + p.mu.Lock() + stopping := p.stopping + p.mu.Unlock() + if stopping { + break + } + time.Sleep(time.Millisecond) + } + secondDone := make(chan struct{}) + go func() { p.Stop(); close(secondDone) }() + select { + case <-secondDone: + t.Fatal("concurrent Stop returned before the first Stop drained the probe") + case <-time.After(20 * time.Millisecond): + } + releaseOnce.Do(func() { close(release) }) + select { + case <-firstDone: + case <-time.After(time.Second): + t.Fatal("first Stop did not finish") + } + select { + case <-secondDone: + case <-time.After(time.Second): + t.Fatal("concurrent Stop did not finish after the first Stop") + } +} + +func TestPollerPass_OverlappingTickIsSkippedNotQueued(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 60, logger.Default()) + cp := newConcurrencyProbe() + p.probe = cp.probe + + before := passSkippedTotal.Value() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + defer func() { + close(cp.release) + p.Stop() + }() + + // The immediate pass is now blocked on the probe gate. Advance past + // a full tick while it is still running. + synctest.Wait() + time.Sleep(61 * time.Second) + synctest.Wait() + + if got := passSkippedTotal.Value() - before; got != 1 { + t.Fatalf("pass_skipped_total delta = %d, want 1 (the overlapping tick must be dropped, not queued)", got) + } + }) +} + +func TestPollerStop_DiscardsCancelledProbeResult(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 60, logger.Default()) + cp := newConcurrencyProbe() + p.probe = cp.probe + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + // The immediate pass is now blocked on the probe gate (never + // released), so Stop must cancel it in flight. + synctest.Wait() + p.Stop() + + if got := repo.upsertCount(); got != 0 { + t.Fatalf("upsert calls = %d, want 0 — a probe cancelled by Stop must never be written", got) + } + }) +} + +func TestPollerProbeNow_RefusedAfterStop(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + p := New(repo, 60, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + p.Stop() + + if ok := p.ProbeNow(sshExecutor("a")); ok { + t.Fatal("ProbeNow returned true after Stop; a stopped poller must refuse new work") + } +} + +func TestPollerListFailure_AbandonsPassWithoutWriting(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := newFakeRepository(sshExecutor("a")) + repo.listErr = context.DeadlineExceeded + p := New(repo, 60, logger.Default()) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true} + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + defer p.Stop() + synctest.Wait() + + if got := repo.upsertCount(); got != 0 { + t.Fatalf("upsert calls = %d, want 0 when listing executors fails", got) + } + }) +} diff --git a/apps/backend/internal/executors/reachability/probe.go b/apps/backend/internal/executors/reachability/probe.go new file mode 100644 index 0000000000..1e9af6aff2 --- /dev/null +++ b/apps/backend/internal/executors/reachability/probe.go @@ -0,0 +1,22 @@ +package reachability + +import ( + "context" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/task/models" +) + +// reachabilityProber is the production seam over the runtime's SSH +// reachability prober. Tests substitute probeFunc directly, so this stays +// unexported and unmocked. +var reachabilityProber = agentruntime.NewSSHReachabilityProber() + +// defaultProbe resolves executor's SSH target from its persisted config and +// probes it. A resolution failure (missing host, invalid port, unresolvable +// alias) never dials — it is reported as +// agentruntime.SSHReachabilityReasonConfig directly, the same reason a +// target with no pinned fingerprint uses. +func defaultProbe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return reachabilityProber.Probe(ctx, executor.Config, probeTimeout) +} diff --git a/apps/backend/internal/executors/reachability/probe_and_wait.go b/apps/backend/internal/executors/reachability/probe_and_wait.go new file mode 100644 index 0000000000..150988230d --- /dev/null +++ b/apps/backend/internal/executors/reachability/probe_and_wait.go @@ -0,0 +1,102 @@ +package reachability + +import ( + "strings" + + "github.com/kandev/kandev/internal/task/models" +) + +// ProbeResult is what ProbeAndWait returns: the durable record when available, +// or the locally projected observation when persistence failed, plus whether +// that specific write actually landed. +type ProbeResult struct { + Record *models.ExecutorReachability + Persisted bool +} + +// probeAndWaitOutcome is the value singleflight.Group shares across every +// coalesced caller for one key. It is deliberately independent of any +// caller's ProbeResult — ProbeAndWait copies out of it before returning, so +// no caller can mutate what another caller already received. +type probeAndWaitOutcome struct { + ran bool + persisted bool + record *models.ExecutorReachability +} + +// ProbeAndWait runs one probe for executor synchronously and returns its +// persisted record, coalescing overlapping callers for the same executor id +// into a single dial via singleflight — the primitive behind the immediate- +// probe HTTP route. Unlike ProbeNow, this does not accept (or run on) any +// caller-supplied context: like every other off-cycle probe it runs on the +// poller's own context/WaitGroup, registered through acquire(), so a +// disconnecting HTTP caller neither cancels it nor affects a caller still +// waiting on the same coalesced result, and Stop still drains it. +// +// ran is false only when the poller itself refused new work (not started, or +// stopping); it does not report whether the probe outcome was a success — +// see ProbeResult.Persisted for whether the result was actually stored. +func (p *Poller) ProbeAndWait(executor *models.Executor) (ProbeResult, bool) { + v, _, _ := p.inflight.Do(probeKey(executor), func() (any, error) { + return p.probeAndWaitOnce(executor), nil + }) + outcome := v.(probeAndWaitOutcome) + if !outcome.ran { + return ProbeResult{}, false + } + var record *models.ExecutorReachability + if outcome.record != nil { + cp := *outcome.record + record = &cp + } + return ProbeResult{Record: record, Persisted: outcome.persisted}, true +} + +// probeKey includes the connection identity, not only the executor ID. A +// save can change the host or its pinned key while an older probe is still in +// flight; those observations must not share a singleflight result. +func probeKey(executor *models.Executor) string { + if executor == nil { + return "" + } + values := make([]string, 0, len(connectionConfigKeys)+1) + values = append(values, executor.ID) + for _, key := range connectionConfigKeys { + values = append(values, key+"="+executor.Config[key]) + } + return strings.Join(values, "\x00") +} + +// probeAndWaitOnce is the singleflight leader's body: acquire a WaitGroup +// slot exactly like every other off-cycle unit of work, probe, and persist. +func (p *Poller) probeAndWaitOnce(executor *models.Executor) probeAndWaitOutcome { + ctx, ok := p.acquire() + if !ok { + return probeAndWaitOutcome{} + } + defer p.wg.Done() + if !p.acquireProbeSlot(ctx) { + return probeAndWaitOutcome{ran: true} + } + defer p.releaseProbeSlot() + + outcome := p.probe(ctx, executor) + if outcome.Cancelled { + probeDiscardedTotal.Add(1) + return probeAndWaitOutcome{ran: true} + } + result := p.observeAndPublish(ctx, executor, outcome) + record := result.After + if record == nil { + record = result.Observed + } + return probeAndWaitOutcome{ran: true, persisted: result.Applied, record: record} +} + +// EffectiveIntervalSeconds returns the clamped interval this poller is +// running with (0 when scheduled probing is disabled), for a caller (the +// reachability DTO) that needs to report probing_enabled/ +// probe_interval_seconds without duplicating ClampInterval's rules. +func (p *Poller) EffectiveIntervalSeconds() int { + return p.intervalSeconds +} diff --git a/apps/backend/internal/executors/reachability/probe_and_wait_test.go b/apps/backend/internal/executors/reachability/probe_and_wait_test.go new file mode 100644 index 0000000000..c962f4b0a9 --- /dev/null +++ b/apps/backend/internal/executors/reachability/probe_and_wait_test.go @@ -0,0 +1,232 @@ +package reachability + +import ( + "context" + "encoding/json" + "sync" + "sync/atomic" + "testing" + "time" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" +) + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2 +// +// Two overlapping immediate-probe callers for the same executor must +// coalesce into exactly one dial and receive one identical record each — the +// acceptance criterion this task adds ProbeAndWait to satisfy. The blocking +// probeFunc plus a short settle delay before releasing mirrors +// golang.org/x/sync/singleflight's own TestDoDupSuppress: there is no faster +// deterministic signal for "the second caller has entered Do" without +// instrumenting production code for the test. +func TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 60, logger.Default()) + p.Start(context.Background()) + defer p.Stop() + + var dialCount int32 + started := make(chan struct{}) + release := make(chan struct{}) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + if atomic.AddInt32(&dialCount, 1) == 1 { + close(started) + } + <-release + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + executor := sshExecutor("exec-coalesce") + + type call struct { + result ProbeResult + ran bool + } + results := make(chan call, 2) + go func() { + result, ran := p.ProbeAndWait(executor) + results <- call{result, ran} + }() + <-started + go func() { + result, ran := p.ProbeAndWait(executor) + results <- call{result, ran} + }() + time.Sleep(100 * time.Millisecond) + close(release) + + first := <-results + second := <-results + + if got := atomic.LoadInt32(&dialCount); got != 1 { + t.Fatalf("dial count = %d, want exactly 1 (coalesced)", got) + } + if !first.ran || !second.ran { + t.Fatalf("ran = %v, %v, want both true", first.ran, second.ran) + } + firstJSON, _ := json.Marshal(first.result) + secondJSON, _ := json.Marshal(second.result) + if string(firstJSON) != string(secondJSON) { + t.Fatalf("responses differ:\n first: %s\n second: %s", firstJSON, secondJSON) + } + if !first.result.Persisted { + t.Fatalf("Persisted = false, want true — the probe outcome was a plain success") + } +} + +func TestPollerProbeAndWaitDoesNotCoalesceDifferentConnectionConfigs(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 0, logger.Default()) + p.Start(context.Background()) + defer p.Stop() + + var dialCount int32 + started := make(chan struct{}, 2) + release := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + atomic.AddInt32(&dialCount, 1) + started <- struct{}{} + <-release + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + first := sshExecutor("same-id") + first.Config["ssh_host"] = "host-a" + second := sshExecutor("same-id") + second.Config["ssh_host"] = "host-b" + results := make(chan ProbeResult, 2) + go func() { result, _ := p.ProbeAndWait(first); results <- result }() + go func() { result, _ := p.ProbeAndWait(second); results <- result }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first probe did not start") + } + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("second connection config incorrectly coalesced with the first") + } + releaseOnce.Do(func() { close(release) }) + <-results + <-results + if got := atomic.LoadInt32(&dialCount); got != 2 { + t.Fatalf("dial count = %d, want 2 for distinct connection configs", got) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2 +// +// The singleflight group shares one result across coalesced callers. Each +// caller must get its own copy of the record, not a shared pointer, so one +// caller mutating its response cannot corrupt what the other already +// received (the risk this task's work order names explicitly). +func TestPollerProbeAndWait_ReturnsIndependentRecordCopies(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 60, logger.Default()) + p.Start(context.Background()) + defer p.Stop() + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + executor := sshExecutor("exec-copy") + result, ran := p.ProbeAndWait(executor) + if !ran { + t.Fatalf("ran = false, want true") + } + if result.Record == nil { + t.Fatalf("Record = nil, want the stored record") + } + originalHost := result.Record.Host + result.Record.Host = "mutated-in-caller" + + second, ran := p.ProbeAndWait(executor) + if !ran { + t.Fatalf("second call: ran = false, want true") + } + if second.Record.Host == "mutated-in-caller" { + t.Fatalf("second caller observed the first caller's mutation") + } + if second.Record.Host != originalHost { + t.Fatalf("second.Record.Host = %q, want %q", second.Record.Host, originalHost) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.28 +// +// The immediate probe must work while the scheduled poller is disabled +// (interval 0) — ProbeAndWait must not depend on the loop ever having run. +func TestPollerProbeAndWait_WorksWhilePollerDisabled(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 0, logger.Default()) + p.Start(context.Background()) + defer p.Stop() + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + result, ran := p.ProbeAndWait(sshExecutor("exec-disabled")) + if !ran { + t.Fatalf("ran = false, want true even though the scheduled poller is disabled") + } + if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("result = %+v, want a persisted reachable record", result) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2 +// +// A probe that ran but was not stored (the repository refused the write) +// still reports ran=true with Persisted=false, never an error — the route +// this backs answers 200 either way. +func TestPollerProbeAndWait_WriteRefusedReportsPersistedFalse(t *testing.T) { + repo := newFakeRepository() + repo.upsertErr = errRepoRefused + p := New(repo, 60, logger.Default()) + p.Start(context.Background()) + defer p.Stop() + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + result, ran := p.ProbeAndWait(sshExecutor("exec-refused")) + if !ran { + t.Fatalf("ran = false, want true") + } + if result.Persisted { + t.Fatalf("Persisted = true, want false — the repository write failed") + } + if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("Record = %+v, want the actual unsaved reachable observation", result.Record) + } +} + +// TestPollerProbeAndWait_RefusedAfterStop mirrors +// TestPollerProbeNow_RefusedAfterStop: once Stop has run, ProbeAndWait must +// not dial at all. +func TestPollerProbeAndWait_RefusedAfterStop(t *testing.T) { + repo := newFakeRepository() + p := New(repo, 60, logger.Default()) + p.Start(context.Background()) + p.Stop() + + dialed := false + p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome { + dialed = true + return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]} + } + + _, ran := p.ProbeAndWait(sshExecutor("exec-stopped")) + if ran { + t.Fatalf("ran = true, want false — the poller was stopped") + } + if dialed { + t.Fatalf("probe was dialed after Stop") + } +} diff --git a/apps/backend/internal/executors/reachability/publish.go b/apps/backend/internal/executors/reachability/publish.go new file mode 100644 index 0000000000..1f1d45b2ae --- /dev/null +++ b/apps/backend/internal/executors/reachability/publish.go @@ -0,0 +1,105 @@ +package reachability + +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +// RecordDTO is the wire shape every reachability HTTP route and the change +// event share, so a client's WS handler and its GET/POST response handler +// stay structurally identical. Reason is always present, even empty — the +// surface's contract is that a reachable/unknown state can carry a non-empty +// reason (a below-threshold failure), so an omitted field would be +// ambiguous with "definitely no reason". +type RecordDTO struct { + ExecutorID string `json:"executor_id"` + State string `json:"state"` + Reason string `json:"reason"` + Message string `json:"message,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures"` + Host string `json:"host,omitempty"` + CheckedAt *time.Time `json:"checked_at"` + LastSuccessAt *time.Time `json:"last_success_at"` + UpdatedAt *time.Time `json:"updated_at"` + ProbingEnabled bool `json:"probing_enabled"` + ProbeIntervalSeconds int `json:"probe_interval_seconds"` + Persisted bool `json:"persisted"` +} + +// BuildRecordDTO projects record into the wire shape. A nil record (an +// eligible executor never probed) synthesizes the "unknown" placeholder with +// every timestamp null, per AC-EXECUTORS-SSH-REACHABILITY-002.5 — this is +// never a 404, so every caller building a GET response for an eligible +// executor can call this unconditionally. +func BuildRecordDTO(executorID string, record *models.ExecutorReachability, effectiveIntervalSeconds int, persisted bool) RecordDTO { + dto := RecordDTO{ + ExecutorID: executorID, + State: string(models.ExecutorReachabilityStateUnknown), + ProbingEnabled: effectiveIntervalSeconds > 0, + ProbeIntervalSeconds: effectiveIntervalSeconds, + Persisted: persisted, + } + if record == nil { + return dto + } + dto.State = string(record.State) + dto.Reason = string(record.Reason) + dto.Message = record.Message + dto.ConsecutiveFailures = record.ConsecutiveFailures + dto.Host = record.Host + dto.CheckedAt = record.CheckedAt + dto.LastSuccessAt = record.LastSuccessAt + if !record.UpdatedAt.IsZero() { + updatedAt := record.UpdatedAt + dto.UpdatedAt = &updatedAt + } + return dto +} + +// Publisher publishes events.ExecutorReachabilityChanged. Constructed once +// at startup wiring and shared by the poller (probe results) and the save +// observer (configuration resets) — one propagation path for both triggers, +// per the system design. +type Publisher struct { + eventBus bus.EventBus + log *logger.Logger +} + +// NewPublisher builds a Publisher. A nil eventBus is accepted so a caller +// that hasn't wired the event bus yet (tests, a degraded boot path) can +// still construct one; PublishChanged becomes a no-op. An optional logger +// records publication failures with the affected executor id. +func NewPublisher(eventBus bus.EventBus, logs ...*logger.Logger) *Publisher { + var log *logger.Logger + if len(logs) > 0 { + log = logs[0] + } + return &Publisher{eventBus: eventBus, log: log} +} + +// PublishChanged publishes record's current shape on +// events.ExecutorReachabilityChanged. Safe to call on a nil *Publisher, with +// a nil record, or with a Publisher built from a nil event bus — every +// combination is a no-op rather than a panic, so callers on the probe/reset +// hot paths don't need a defensive check of their own. +func (p *Publisher) PublishChanged(ctx context.Context, record *models.ExecutorReachability, effectiveIntervalSeconds int) { + if p == nil || p.eventBus == nil || record == nil { + return + } + dto := BuildRecordDTO(record.ExecutorID, record, effectiveIntervalSeconds, true) + event := bus.NewEvent(events.ExecutorReachabilityChanged, "executor-reachability", dto) + if err := p.eventBus.Publish(ctx, events.ExecutorReachabilityChanged, event); err != nil { + publishFailedTotal.Add(1) + if p.log != nil { + p.log.Warn("executor ssh reachability: publish failed", + zap.String("executor_id", record.ExecutorID), zap.Error(err)) + } + } +} diff --git a/apps/backend/internal/executors/reachability/publish_test.go b/apps/backend/internal/executors/reachability/publish_test.go new file mode 100644 index 0000000000..84be6784a5 --- /dev/null +++ b/apps/backend/internal/executors/reachability/publish_test.go @@ -0,0 +1,158 @@ +package reachability + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +type failingPublishEventBus struct { + *bus.MemoryEventBus + err error +} + +func (b *failingPublishEventBus) Publish(context.Context, string, *bus.Event) error { + return b.err +} + +func testEventBusLogger() *bus.MemoryEventBus { + return bus.NewMemoryEventBus(logger.Default()) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// PublishChanged must publish on events.ExecutorReachabilityChanged with a +// payload carrying the record plus the derived probing fields a client needs +// (probing_enabled/probe_interval_seconds) without a second fetch. +func TestPublisherPublishChanged_PublishesRecordDTO(t *testing.T) { + eventBus := testEventBusLogger() + var captured *bus.Event + _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error { + captured = ev + return nil + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + pub := NewPublisher(eventBus) + checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + record := &models.ExecutorReachability{ + ExecutorID: "exec-1", + State: models.ExecutorReachabilityStateUnreachable, + Reason: models.ExecutorReachabilityReasonNetwork, + Message: "connection refused", + Host: "10.0.0.1", + CheckedAt: &checkedAt, + } + pub.PublishChanged(context.Background(), record, 60) + + if captured == nil { + t.Fatal("no event was published") + } + dto, ok := captured.Data.(RecordDTO) + if !ok { + t.Fatalf("event data type = %T, want RecordDTO", captured.Data) + } + if dto.ExecutorID != "exec-1" || dto.State != "unreachable" || dto.Reason != "network" { + t.Fatalf("dto = %+v, want executor-1/unreachable/network", dto) + } + if !dto.ProbingEnabled || dto.ProbeIntervalSeconds != 60 { + t.Fatalf("dto probing fields = %+v, want enabled/60", dto) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14 +// +// probing_enabled must be false exactly when the effective interval is 0 — +// the client's only way to distinguish "not probed yet" from "probing off". +func TestPublisherPublishChanged_ProbingDisabledWhenIntervalIsZero(t *testing.T) { + eventBus := testEventBusLogger() + var captured *bus.Event + _, _ = eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error { + captured = ev + return nil + }) + + pub := NewPublisher(eventBus) + pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 0) + + dto := captured.Data.(RecordDTO) + if dto.ProbingEnabled { + t.Fatalf("ProbingEnabled = true, want false when the effective interval is 0") + } +} + +// A nil publisher, nil record, or nil bus must never panic — PublishChanged +// is called from probe/save-reset paths that don't want a defensive nil +// check at every call site. +func TestPublisherPublishChanged_NilSafe(t *testing.T) { + var nilPub *Publisher + nilPub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60) + + pub := NewPublisher(testEventBusLogger()) + pub.PublishChanged(context.Background(), nil, 60) + + pub2 := NewPublisher(nil) + pub2.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60) +} + +func TestPublisherPublishChangedCountsPublishFailures(t *testing.T) { + before := publishFailedTotal.Value() + pub := NewPublisher(&failingPublishEventBus{ + MemoryEventBus: testEventBusLogger(), + err: errors.New("event bus unavailable"), + }) + pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60) + if got := publishFailedTotal.Value(); got != before+1 { + t.Fatalf("publishFailedTotal = %d, want %d", got, before+1) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14, AC-EXECUTORS-SSH-REACHABILITY-002.5 +// +// BuildRecordDTO is the single place the HTTP routes and the event publisher +// share for projecting a stored (or absent) record into the wire shape — +// including the "no record yet" unknown placeholder with every timestamp +// null, never omitted. +func TestBuildRecordDTO_SynthesizesUnknownShapeWhenRecordIsNil(t *testing.T) { + dto := BuildRecordDTO("exec-1", nil, 60, true) + if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Reason != "" { + t.Fatalf("dto = %+v, want a synthesized unknown/empty-reason shape", dto) + } + if dto.CheckedAt != nil || dto.LastSuccessAt != nil || dto.UpdatedAt != nil { + t.Fatalf("dto = %+v, want every timestamp nil", dto) + } + if dto.ConsecutiveFailures != 0 { + t.Fatalf("ConsecutiveFailures = %d, want 0", dto.ConsecutiveFailures) + } +} + +func TestBuildRecordDTO_ProjectsStoredRecord(t *testing.T) { + checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + record := &models.ExecutorReachability{ + ExecutorID: "exec-1", + State: models.ExecutorReachabilityStateReachable, + ConsecutiveFailures: 0, + Host: "10.0.0.1", + CheckedAt: &checkedAt, + LastSuccessAt: &checkedAt, + UpdatedAt: checkedAt, + } + dto := BuildRecordDTO("exec-1", record, 60, true) + if dto.State != "reachable" || dto.Host != "10.0.0.1" { + t.Fatalf("dto = %+v, want the stored record projected", dto) + } + if dto.CheckedAt == nil || !dto.CheckedAt.Equal(checkedAt) { + t.Fatalf("CheckedAt = %v, want %v", dto.CheckedAt, checkedAt) + } + if !dto.Persisted { + t.Fatalf("Persisted = false, want true") + } +} diff --git a/apps/backend/internal/executors/reachability/save_observer.go b/apps/backend/internal/executors/reachability/save_observer.go new file mode 100644 index 0000000000..7c8740c861 --- /dev/null +++ b/apps/backend/internal/executors/reachability/save_observer.go @@ -0,0 +1,126 @@ +package reachability + +import ( + "context" + "errors" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/task/models" +) + +// connectionConfigKeys names the exact SSH connection-configuration fields — +// host, port, user, identity source, optional ProxyJump, and pinned host +// fingerprint — that a save-detection comparison must diff. Any other Config +// key (or a non-config field) changing is not connection-relevant and must +// not trigger a reset. +var connectionConfigKeys = []string{ + "ssh_host", + "ssh_host_alias", + "ssh_port", + "ssh_user", + "ssh_identity_source", + "ssh_identity_file", + "ssh_proxy_jump", + "ssh_host_fingerprint", +} + +// SaveObserver implements service.ExecutorSaveObserver: it resets the stored +// reachability record and dispatches an off-cycle probe whenever an active +// SSH executor's connection configuration changes. Declared in +// internal/task/service (this package structurally satisfies it there +// without an import back to it). +type SaveObserver struct { + poller *Poller +} + +// NewSaveObserver builds a SaveObserver over poller. poller supplies the +// repository (for the reset and the pre-reset read), the publisher (for the +// change event), and ProbeNow (for the post-reset off-cycle probe) — the same +// primitives task 03/04 already exercise elsewhere in this package. +func NewSaveObserver(poller *Poller) *SaveObserver { + return &SaveObserver{poller: poller} +} + +// OnExecutorSaved resets and re-probes after when it is an active SSH +// executor whose connection configuration changed. before is nil on create, +// which is always treated as a change — there is no prior state to compare. +func (o *SaveObserver) OnExecutorSaved(ctx context.Context, before, after *models.Executor) { + if !eligibleForReachability(after) { + return + } + if !connectionConfigChanged(before, after) { + return + } + o.resetAndProbe(ctx, after) +} + +func eligibleForReachability(executor *models.Executor) bool { + return executor != nil && + executor.Type == models.ExecutorTypeSSH && + executor.Status == models.ExecutorStatusActive +} + +func connectionConfigChanged(before, after *models.Executor) bool { + if before == nil { + return true + } + for _, key := range connectionConfigKeys { + if before.Config[key] != after.Config[key] { + return true + } + } + return false +} + +// resetAndProbe resets the stored record to the new host, publishes a change +// event when doing so actually changes what a client would see, and dispatches +// an off-cycle probe. The pre-reset record is read first because after the +// reset the "was it worth publishing" comparison is no longer possible — the +// reset itself always leaves the unknown/empty-reason placeholder behind. +func (o *SaveObserver) resetAndProbe(ctx context.Context, executor *models.Executor) { + repo := o.poller.store.repo + previous, readErr := repo.GetExecutorReachability(ctx, executor.ID) + if readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound) { + o.poller.log.Warn("executor ssh reachability: read before reset failed", + zap.String("executor_id", executor.ID), zap.Error(readErr)) + } + + newHost := executor.Config["ssh_host"] + if err := repo.ResetExecutorReachability(ctx, executor.ID, newHost, executor.UpdatedAt); err != nil { + o.poller.log.Warn("executor ssh reachability: reset failed", + zap.String("executor_id", executor.ID), zap.Error(err)) + return + } + + if recordWorthPublishing(previous) || (readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound)) { + reset, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + o.poller.log.Warn("executor ssh reachability: read after reset failed", + zap.String("executor_id", executor.ID), zap.Error(err)) + reset = &models.ExecutorReachability{ + ExecutorID: executor.ID, + State: models.ExecutorReachabilityStateUnknown, + Host: newHost, + UpdatedAt: time.Now().UTC(), + } + } + if o.poller.publisher != nil { + o.poller.publisher.PublishChanged(ctx, reset, o.poller.EffectiveIntervalSeconds()) + } + } + + o.poller.ProbeNow(executor) +} + +// recordWorthPublishing reports whether record differs from the synthesized +// unknown/empty-reason placeholder a client already sees for "no record yet" +// — publishing a reset that lands on that same placeholder would be a no-op +// event. +func recordWorthPublishing(record *models.ExecutorReachability) bool { + if record == nil { + return false + } + return record.State != models.ExecutorReachabilityStateUnknown || record.Reason != "" +} diff --git a/apps/backend/internal/executors/reachability/save_observer_test.go b/apps/backend/internal/executors/reachability/save_observer_test.go new file mode 100644 index 0000000000..a50c3a1c68 --- /dev/null +++ b/apps/backend/internal/executors/reachability/save_observer_test.go @@ -0,0 +1,249 @@ +package reachability + +import ( + "context" + "testing" + "time" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" +) + +// newTestSaveObserver builds a SaveObserver over a started Poller wired to +// repo, with probe replaced by a deterministic success so ProbeNow's +// dispatched probe is observable through repo's upsert count instead of real +// network I/O. Callers must defer poller.Stop(). +func newTestSaveObserver(t *testing.T, repo *fakeRepository) (*SaveObserver, *Poller) { + t.Helper() + poller := New(repo, 0, logger.Default()) + poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]} + } + poller.Start(context.Background()) + return NewSaveObserver(poller), poller +} + +func sshExecutorWithConfig(id string, status models.ExecutorStatus, config map[string]string) *models.Executor { + return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: status, Config: config} +} + +// waitForUpsertCount blocks until repo has observed at least want upserts, or +// fails the test after a short deadline — a happens-before proof that the +// observer's fire-and-forget ProbeNow dispatch actually ran, without a fixed +// sleep. +func waitForUpsertCount(t *testing.T, repo *fakeRepository, want int32) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if repo.upsertCount() >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("upsertCount = %d, want >= %d", repo.upsertCount(), want) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2 +// +// A host change on an active SSH executor must reset the stored record to +// the new host and dispatch an off-cycle probe — the reset's whole purpose is +// that a stale unreachable/reachable record about the OLD host must never be +// shown against the new one, even for the single tick before the next probe +// lands. +func TestSaveObserverResetsAndProbesOnHostChange(t *testing.T) { + repo := newFakeRepository() + observer, poller := newTestSaveObserver(t, repo) + defer poller.Stop() + + before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"}) + after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"}) + + observer.OnExecutorSaved(context.Background(), before, after) + + rec, err := repo.GetExecutorReachability(context.Background(), "exec-1") + if err != nil { + t.Fatalf("GetExecutorReachability: %v", err) + } + if rec.Host != "10.0.0.2" { + t.Fatalf("record.Host = %q, want the new host 10.0.0.2 (reset must run before the probe)", rec.Host) + } + + waitForUpsertCount(t, repo, 1) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// Creating an executor has no prior state to compare against, so it must +// always be treated as a change — before=nil is not "nothing changed". +func TestSaveObserverTreatsNilBeforeAsAlwaysChanged(t *testing.T) { + repo := newFakeRepository() + observer, poller := newTestSaveObserver(t, repo) + defer poller.Stop() + + after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"}) + observer.OnExecutorSaved(context.Background(), nil, after) + + if _, err := repo.GetExecutorReachability(context.Background(), "exec-created"); err != nil { + t.Fatalf("GetExecutorReachability: %v, want a reset record from the create", err) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// A non-SSH executor save must never reset or probe — there is no +// reachability record for it to begin with. +func TestSaveObserverSkipsNonSSHExecutor(t *testing.T) { + repo := newFakeRepository() + observer, poller := newTestSaveObserver(t, repo) + defer poller.Stop() + + after := &models.Executor{ID: "exec-local", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive} + observer.OnExecutorSaved(context.Background(), nil, after) + + if _, err := repo.GetExecutorReachability(context.Background(), "exec-local"); err == nil { + t.Fatalf("expected no record for a non-SSH executor") + } + if repo.upsertCount() != 0 { + t.Fatalf("upsertCount = %d, want 0 (no probe should have been dispatched)", repo.upsertCount()) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// A disabled SSH executor must not be reset or probed — probing an executor +// no launch path can use wastes a dial and would misleadingly imply it is +// monitored. +func TestSaveObserverSkipsInactiveSSHExecutor(t *testing.T) { + repo := newFakeRepository() + observer, poller := newTestSaveObserver(t, repo) + defer poller.Stop() + + after := sshExecutorWithConfig("exec-disabled", models.ExecutorStatusDisabled, map[string]string{"ssh_host": "10.0.0.5"}) + observer.OnExecutorSaved(context.Background(), nil, after) + + if _, err := repo.GetExecutorReachability(context.Background(), "exec-disabled"); err == nil { + t.Fatalf("expected no record for a disabled executor") + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// Only the named connection-configuration keys matter. A save that changes +// something else entirely (here: nothing in Config at all differs) must not +// reset an existing record — a reset would wipe legitimate history (state, +// consecutive_failures, last_success_at) for no reason tied to the host's +// actual reachability. +func TestSaveObserverSkipsWhenConnectionConfigUnchanged(t *testing.T) { + repo := newFakeRepository() + observer, poller := newTestSaveObserver(t, repo) + defer poller.Stop() + + config := map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"} + before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config) + after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config) + + observer.OnExecutorSaved(context.Background(), before, after) + + if _, err := repo.GetExecutorReachability(context.Background(), "exec-1"); err == nil { + t.Fatalf("expected no record written when connection config is unchanged") + } + if repo.upsertCount() != 0 { + t.Fatalf("upsertCount = %d, want 0", repo.upsertCount()) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2, AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// A reset that actually changes what a client would see (the previously +// stored record was not already the unknown/empty-reason placeholder) must +// publish a change event, so a UI watching this executor doesn't have to poll +// to notice the reset. +func TestSaveObserverPublishesChangeEventWhenPriorRecordWasObserved(t *testing.T) { + repo := newFakeRepository() + eventBus := bus.NewMemoryEventBus(logger.Default()) + published := make(chan RecordDTO, 4) + if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error { + published <- ev.Data.(RecordDTO) + return nil + }); err != nil { + t.Fatalf("subscribe: %v", err) + } + + poller := New(repo, 0, logger.Default()) + poller.SetPublisher(NewPublisher(eventBus)) + poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome { + return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]} + } + poller.Start(context.Background()) + defer poller.Stop() + observer := NewSaveObserver(poller) + + checkedAt := time.Now().UTC() + repo.records["exec-1"] = &models.ExecutorReachability{ + ExecutorID: "exec-1", State: models.ExecutorReachabilityStateReachable, + Host: "10.0.0.1", CheckedAt: &checkedAt, UpdatedAt: checkedAt, + } + + before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"}) + after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"}) + observer.OnExecutorSaved(context.Background(), before, after) + + select { + case dto := <-published: + if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Host != "10.0.0.2" { + t.Fatalf("dto = %+v, want the reset unknown/10.0.0.2 shape", dto) + } + case <-time.After(time.Second): + t.Fatal("no change event published for a reset over a previously observed record") + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// Resetting an executor that was never probed before (no stored record at +// all) is not itself a client-visible change — the client already sees the +// synthesized unknown placeholder for "no record yet" — so the reset step +// must not publish. The dispatched off-cycle probe that follows is a +// separate, legitimate publish opportunity (a real unknown -> reachable +// transition) and is deliberately excluded here via a blocking probe gate: +// OnExecutorSaved's synchronous portion (the reset and its publish decision) +// completes fully before the probe goroutine can even call probe, so +// asserting silence right after OnExecutorSaved returns isolates the reset +// step's own behavior without racing the probe. +func TestSaveObserverDoesNotPublishFromResetWhenNoPriorRecord(t *testing.T) { + repo := newFakeRepository() + eventBus := bus.NewMemoryEventBus(logger.Default()) + published := make(chan RecordDTO, 4) + if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error { + published <- ev.Data.(RecordDTO) + return nil + }); err != nil { + t.Fatalf("subscribe: %v", err) + } + + probeGate := make(chan struct{}) + poller := New(repo, 0, logger.Default()) + poller.SetPublisher(NewPublisher(eventBus)) + poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome { + <-probeGate + return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]} + } + poller.Start(context.Background()) + defer poller.Stop() + observer := NewSaveObserver(poller) + + after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"}) + observer.OnExecutorSaved(context.Background(), nil, after) + + select { + case dto := <-published: + t.Fatalf("reset step itself must not publish when there was no prior record: %+v", dto) + default: + } + + close(probeGate) + waitForUpsertCount(t, repo, 1) +} diff --git a/apps/backend/internal/executors/reachability/store.go b/apps/backend/internal/executors/reachability/store.go new file mode 100644 index 0000000000..7a70d6fe0d --- /dev/null +++ b/apps/backend/internal/executors/reachability/store.go @@ -0,0 +1,222 @@ +package reachability + +import ( + "context" + "errors" + "time" + + "go.uber.org/zap" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" +) + +// Repository is the narrow slice of repository.ExecutorRepository this +// package depends on. The poller and the off-cycle-probe primitive share it, +// so a fake test double satisfies both without importing the full +// task/repository interface. +type Repository interface { + ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) + GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) + UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error + ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error +} + +// observeResult reports what Observe actually did, for the poller's logging +// and metrics and as the change signal a future consumer (the HTTP/event +// surface) reads off the write path rather than re-deriving. Changed is +// broader than StateChanged: the destination state can hold steady +// (unreachable stays unreachable) while the stored reason moves between two +// different failure causes, and that is still worth telling a client about. +type observeResult struct { + Applied bool + StateChanged bool + Changed bool + Previous models.ExecutorReachabilityState + Current models.ExecutorReachabilityState + // After is the full record as written, for a caller (the HTTP immediate- + // probe route, the change-event publisher) that needs more than the bare + // state enum without a second read-back. + After *models.ExecutorReachability + // Observed is the outcome projected locally from the probe, even when the + // repository rejects the write or a read-back fails. It is never published + // as durable state, but lets a manual probe show the actual result. + Observed *models.ExecutorReachability +} + +// store owns the write path's hysteresis. The consecutive-failure counter +// and the destination state for an existing row are derived entirely inside +// UpsertExecutorReachability's own SQL (see the Persistence section of the +// design) — store's job is narrower: pick InitialState/InitialFailures for a +// never-before-seen row (the insert-only path), and determine whether the +// write actually landed and whether the resulting state changed by reading +// the record before and after the call. +type store struct { + repo Repository + log *logger.Logger +} + +// Observe records one probe outcome for executor. checkedAt is the probe's +// own completion timestamp, supplied by the caller so a test can control it +// precisely (see the design's last-write-wins contract). +func (s *store) Observe(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) observeResult { + before, err := s.repo.GetExecutorReachability(ctx, executor.ID) + if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) { + s.log.Warn("executor ssh reachability: read before write failed", + zap.String("executor_id", executor.ID), zap.Error(err)) + return observeResult{} + } + + checkedAt = checkedAt.UTC().Truncate(time.Microsecond) + obs := buildObservation(executor, outcome, checkedAt) + observed := projectObservation(before, obs) + recordProbeOutcome(outcome.Success, string(obs.Reason)) + if err := s.repo.UpsertExecutorReachability(ctx, obs); err != nil { + s.log.Warn("executor ssh reachability: write failed", + zap.String("executor_id", executor.ID), zap.Error(err)) + return observeResult{Current: observed.State, Observed: observed} + } + + after, err := s.repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + s.log.Warn("executor ssh reachability: read-back after write failed", + zap.String("executor_id", executor.ID), zap.Error(err)) + readBackFailedTotal.Add(1) + return observeResult{Current: observed.State, Observed: observed} + } + + // The upsert's own WHERE (eligibility, last-write-wins on checked_at) may + // have silently dropped this write. Comparing the stored checked_at back + // against the one this observation carried is how we find out — the SQL + // itself never reports it. + if after.CheckedAt == nil || !after.CheckedAt.Equal(checkedAt) { + writeRefusedTotal.Add(1) + s.log.Debug("executor ssh reachability: write refused", + zap.String("executor_id", executor.ID)) + return observeResult{Current: after.State, Observed: observed} + } + + previous := models.ExecutorReachabilityStateUnknown + var previousReason models.ExecutorReachabilityReason + if before != nil { + previous = before.State + previousReason = before.Reason + } + stateChanged := previous != after.State + if stateChanged { + s.logTransition(executor, previous, after.State, obs.Reason, after.ConsecutiveFailures) + recordStateTransition(string(after.State)) + } + changed := stateChanged || previousReason != after.Reason + return observeResult{ + Applied: true, StateChanged: stateChanged, Changed: changed, + Previous: previous, Current: after.State, After: after, + Observed: after, + } +} + +func projectObservation(before *models.ExecutorReachability, obs models.ExecutorReachabilityObservation) *models.ExecutorReachability { + record := &models.ExecutorReachability{ + ExecutorID: obs.ExecutorID, + State: obs.InitialState, + Reason: obs.Reason, + Message: obs.Message, + ConsecutiveFailures: obs.InitialFailures, + Host: obs.Host, + CheckedAt: checkedAtPtr(obs.CheckedAt), + UpdatedAt: time.Now().UTC(), + } + if before == nil { + if obs.Reason == "" { + record.State = models.ExecutorReachabilityStateReachable + record.ConsecutiveFailures = 0 + record.LastSuccessAt = checkedAtPtr(obs.CheckedAt) + } + return record + } + + record.State = before.State + record.ConsecutiveFailures = before.ConsecutiveFailures + record.LastSuccessAt = before.LastSuccessAt + if obs.Reason == "" { + record.State = models.ExecutorReachabilityStateReachable + record.ConsecutiveFailures = 0 + record.LastSuccessAt = checkedAtPtr(obs.CheckedAt) + return record + } + record.ConsecutiveFailures++ + if isStickyReason(obs.Reason) || record.ConsecutiveFailures >= obs.FailureThreshold { + record.State = models.ExecutorReachabilityStateUnreachable + } + return record +} + +func checkedAtPtr(value time.Time) *time.Time { + copy := value + return © +} + +// logTransition logs at Warn going into unreachable and Info coming out of +// it, per the design's Observability section. Unchanged results never reach +// here — Observe only calls this when the state actually changed. +func (s *store) logTransition(executor *models.Executor, previous, current models.ExecutorReachabilityState, reason models.ExecutorReachabilityReason, failures int) { + fields := []zap.Field{ + zap.String("executor_id", executor.ID), + zap.String("host", executor.Config["ssh_host"]), + zap.String("from", string(previous)), + zap.String("to", string(current)), + zap.String("reason", string(reason)), + zap.Int("consecutive_failures", failures), + } + if current == models.ExecutorReachabilityStateUnreachable { + s.log.Warn("executor ssh reachability state transition", fields...) + return + } + s.log.Info("executor ssh reachability state transition", fields...) +} + +// buildObservation projects a probe outcome into the upsert's input. An +// empty Reason means success. InitialState/InitialFailures matter only on +// the insert path (no prior row); every later write derives both in SQL +// from the stored row instead: +// - success: reachable, zero counter. +// - a sticky reason (config, host_key): unreachable immediately, whatever +// the counter — both are deterministic and would repeat, so waiting out +// the threshold buys no confidence. +// - any other failure: the counter starts at 1, which is below +// failureThreshold, so the state stays unknown — a record that doesn't +// exist yet is never promoted by a single below-threshold failure, the +// same rule the SQL enforces for an existing row. +func buildObservation(executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) models.ExecutorReachabilityObservation { + reason := models.ExecutorReachabilityReason(outcome.Reason) + initialState := models.ExecutorReachabilityStateUnknown + initialFailures := 0 + switch { + case outcome.Success: + initialState = models.ExecutorReachabilityStateReachable + case isStickyReason(reason): + initialState = models.ExecutorReachabilityStateUnreachable + initialFailures = 1 + default: + initialFailures = 1 + if initialFailures >= failureThreshold { + initialState = models.ExecutorReachabilityStateUnreachable + } + } + return models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, + SeenUpdatedAt: executor.UpdatedAt, + InitialState: initialState, + InitialFailures: initialFailures, + Reason: reason, + Message: outcome.Message, + Host: outcome.Host, + CheckedAt: checkedAt, + FailureThreshold: failureThreshold, + } +} + +func isStickyReason(reason models.ExecutorReachabilityReason) bool { + return reason == models.ExecutorReachabilityReasonConfig || reason == models.ExecutorReachabilityReasonHostKey +} diff --git a/apps/backend/internal/executors/reachability/store_test.go b/apps/backend/internal/executors/reachability/store_test.go new file mode 100644 index 0000000000..5961bfc48a --- /dev/null +++ b/apps/backend/internal/executors/reachability/store_test.go @@ -0,0 +1,322 @@ +package reachability + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +func newStoreTestRepo(t *testing.T) *sqlite.Repository { + t.Helper() + path := filepath.Join(t.TempDir(), "reachability-store.db") + dbConn, err := db.OpenSQLite(path) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqlxDB := sqlx.NewDb(dbConn, "sqlite3") + repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + _ = sqlxDB.Close() + t.Fatalf("new repository: %v", err) + } + t.Cleanup(func() { _ = sqlxDB.Close() }) + return repo +} + +func newStoreTestExecutor(t *testing.T, repo *sqlite.Repository) *models.Executor { + t.Helper() + executor := &models.Executor{ + ID: "exec-" + t.Name(), + Name: "ssh-host", + Type: models.ExecutorTypeSSH, + Status: models.ExecutorStatusActive, + Config: map[string]string{"ssh_host": "10.0.0.1"}, + } + if err := repo.CreateExecutor(context.Background(), executor); err != nil { + t.Fatalf("create executor: %v", err) + } + return executor +} + +type microsecondReachabilityRepository struct { + *sqlite.Repository +} + +func (r *microsecondReachabilityRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error { + obs.CheckedAt = obs.CheckedAt.UTC().Truncate(time.Microsecond) + return r.Repository.UpsertExecutorReachability(ctx, obs) +} + +type failingGetReachabilityRepository struct { + *sqlite.Repository + err error + failAfter int + calls int +} + +func (r *failingGetReachabilityRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) { + r.calls++ + if r.calls >= r.failAfter { + return nil, r.err + } + return r.Repository.GetExecutorReachability(ctx, executorID) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9, AC-EXECUTORS-SSH-REACHABILITY-001.10 +// +// Drives a success/fail/fail/success sequence and asserts the exact probe +// index on which each direction flips. A naive flip-on-first-result +// implementation would report unreachable after the very first failure +// (index 2) instead of the second (index 3), and would report a below- +// threshold failure as a state change even though the state stands still. +func TestStoreObserveHysteresisSequence(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + s := &store{repo: repo, log: logger.Default()} + ctx := context.Background() + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + type step struct { + label string + outcome agentruntime.SSHProbeOutcome + wantState models.ExecutorReachabilityState + wantChanged bool + wantFailures int + } + steps := []step{ + { + label: "1: first success establishes reachable", + outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, + wantState: models.ExecutorReachabilityStateReachable, + wantChanged: true, + wantFailures: 0, + }, + { + label: "2: below-threshold failure leaves state reachable", + outcome: agentruntime.SSHProbeOutcome{ + Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused", + }, + wantState: models.ExecutorReachabilityStateReachable, + wantChanged: false, + wantFailures: 1, + }, + { + label: "3: failure reaching failureThreshold flips to unreachable", + outcome: agentruntime.SSHProbeOutcome{ + Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused", + }, + wantState: models.ExecutorReachabilityStateUnreachable, + wantChanged: true, + wantFailures: 2, + }, + { + label: "4: a single success flips back to reachable with a zero counter", + outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, + wantState: models.ExecutorReachabilityStateReachable, + wantChanged: true, + wantFailures: 0, + }, + } + + for i, st := range steps { + checkedAt := base.Add(time.Duration(i) * time.Second) + executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID) + result := s.Observe(ctx, executor, st.outcome, checkedAt) + if !result.Applied { + t.Fatalf("%s: write was refused, want applied", st.label) + } + if result.Current != st.wantState { + t.Fatalf("%s: state = %q, want %q", st.label, result.Current, st.wantState) + } + if result.StateChanged != st.wantChanged { + t.Fatalf("%s: changed = %v, want %v", st.label, result.StateChanged, st.wantChanged) + } + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("%s: get: %v", st.label, err) + } + if got.ConsecutiveFailures != st.wantFailures { + t.Fatalf("%s: consecutive_failures = %d, want %d", st.label, got.ConsecutiveFailures, st.wantFailures) + } + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9 +// +// A single config or host_key failure writes unreachable on that first +// probe, with the counter still below failureThreshold — sticky reasons +// bypass the threshold entirely. +func TestStoreObserveStickyFailureIsUnreachableOnFirstProbe(t *testing.T) { + tests := []struct { + name string + reason agentruntime.SSHReachabilityReason + }{ + {"config", agentruntime.SSHReachabilityReasonConfig}, + {"host_key", agentruntime.SSHReachabilityReasonHostKey}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + s := &store{repo: repo, log: logger.Default()} + ctx := context.Background() + + result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{ + Host: "10.0.0.1", Reason: tt.reason, Message: "boom", + }, time.Now().UTC()) + + if !result.Applied || result.Current != models.ExecutorReachabilityStateUnreachable { + t.Fatalf("result = %+v, want applied unreachable on the first probe", result) + } + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.ConsecutiveFailures >= failureThreshold { + t.Fatalf("consecutive_failures = %d, want below failureThreshold (%d) — sticky must not need it", got.ConsecutiveFailures, failureThreshold) + } + }) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// The change-publish trigger is broader than StateChanged: a steady +// unreachable executor can still swap failure reasons (network, then +// timeout) between probes without ever leaving the unreachable state. A +// publisher that only watched StateChanged would miss this and never tell a +// client the diagnostic reason on screen just went stale. +func TestStoreObserveReportsChangedOnReasonChangeWithoutStateChange(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + s := &store{repo: repo, log: logger.Default()} + ctx := context.Background() + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + // Two failures reach failureThreshold and flip to unreachable. + for i := 0; i < 2; i++ { + executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID) + s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{ + Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused", + }, base.Add(time.Duration(i)*time.Second)) + } + + // A third failure with a different transient reason keeps the state + // unreachable (already above threshold) but changes the stored reason. + executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID) + result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{ + Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonTimeout, Message: "handshake timed out", + }, base.Add(2*time.Second)) + + if result.StateChanged { + t.Fatalf("StateChanged = true, want false — state was already unreachable") + } + if !result.Changed { + t.Fatalf("Changed = false, want true — reason moved from network to timeout") + } + if result.After == nil || result.After.Reason != models.ExecutorReachabilityReasonTimeout { + t.Fatalf("After = %+v, want a record carrying reason %q", result.After, models.ExecutorReachabilityReasonTimeout) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// A probe outcome that reproduces the same state and reason as the stored +// record must not be reported as a change — this is what keeps a steady host +// silent on the event bus. +func TestStoreObserveReportsUnchangedWhenStateAndReasonBothRepeat(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + s := &store{repo: repo, log: logger.Default()} + ctx := context.Background() + + executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID) + first := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC()) + if !first.Changed { + t.Fatalf("first observation: Changed = false, want true (unknown -> reachable)") + } + + executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID) + second := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC().Add(time.Second)) + if second.Changed { + t.Fatalf("second observation: Changed = true, want false — same state (reachable) and reason (success)") + } + if second.After == nil { + t.Fatalf("After = nil, want the current stored record") + } +} + +func TestStoreObserveNormalizesDatabaseTimestampPrecision(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + s := &store{repo: µsecondReachabilityRepository{Repository: repo}, log: logger.Default()} + checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 123456789, time.UTC) + + result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{ + Success: true, Host: "10.0.0.1", + }, checkedAt) + if !result.Applied { + t.Fatalf("result = %+v, want applied despite microsecond storage precision", result) + } + if result.After == nil || result.After.CheckedAt == nil { + t.Fatalf("After = %+v, want a persisted record", result.After) + } + if want := checkedAt.Truncate(time.Microsecond); !result.After.CheckedAt.Equal(want) { + t.Fatalf("CheckedAt = %v, want %v", result.After.CheckedAt, want) + } +} + +func TestStoreObserveStopsOnUnexpectedInitialReadError(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + readErr := errors.New("database unavailable") + wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 1} + s := &store{repo: wrapped, log: logger.Default()} + + result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{ + Success: true, Host: "10.0.0.1", + }, time.Now().UTC()) + if result.Applied || result.Observed != nil { + t.Fatalf("result = %+v, want no observation when the initial read fails", result) + } + if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); !errors.Is(err, models.ErrExecutorReachabilityNotFound) { + t.Fatalf("stored record error = %v, want no write", err) + } +} + +func TestStoreObserveReturnsUnsavedObservationOnReadBackError(t *testing.T) { + repo := newStoreTestRepo(t) + executor := newStoreTestExecutor(t, repo) + readErr := errors.New("read-back unavailable") + wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 2} + s := &store{repo: wrapped, log: logger.Default()} + + result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{ + Success: true, Host: "10.0.0.1", + }, time.Now().UTC()) + if result.Applied || result.Observed == nil || result.Observed.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("result = %+v, want an unsaved reachable observation", result) + } + if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); err != nil { + t.Fatalf("stored record lookup: %v, want the write to have landed", err) + } +} + +func mustGetUpdatedAt(t *testing.T, repo *sqlite.Repository, id string) time.Time { + t.Helper() + executor, err := repo.GetExecutor(context.Background(), id) + if err != nil { + t.Fatalf("get executor: %v", err) + } + return executor.UpdatedAt +} diff --git a/apps/backend/internal/gateway/websocket/hub.go b/apps/backend/internal/gateway/websocket/hub.go index defa6c9271..9bce7a6ae5 100644 --- a/apps/backend/internal/gateway/websocket/hub.go +++ b/apps/backend/internal/gateway/websocket/hub.go @@ -59,6 +59,9 @@ type Hub struct { pluginConversationService *plugins.Service conversationSourceReader ConversationSourceReader conversationEpoch string + // sessionLaunchWarnings keeps the latest launch warning long enough for a + // client that subscribes after the event was published to receive it. + sessionLaunchWarnings map[string][]byte // clientDisconnectListener releases connection-bound resources after a // client is removed from the hub. It runs asynchronously so durable cleanup @@ -100,6 +103,7 @@ func NewHub(dispatcher *ws.Dispatcher, log *logger.Logger) *Hub { broadcast: make(chan *ws.Message, 256), dispatcher: dispatcher, sessionMode: newSessionModeTracker(), + sessionLaunchWarnings: make(map[string][]byte), logger: log.WithFields(zap.String("component", "ws_hub")), conversationEpoch: uuid.NewString(), } @@ -185,6 +189,7 @@ func (h *Hub) closeAllClients() { listener := h.clientDisconnectListener h.taskSubscribers = make(map[string]map[*Client]bool) h.sessionSubscribers = make(map[string]map[*Client]bool) + h.sessionLaunchWarnings = make(map[string][]byte) h.runSubscribers = make(map[string]map[*Client]bool) h.systemMetricsSubscribers = make(map[*Client]bool) h.sessionMode.focusByClient = make(map[string]map[*Client]bool) @@ -684,6 +689,9 @@ func (h *Hub) SubscribeToSession(client *Client, sessionID string) bool { zap.String("session_id", sessionID)) h.recomputeSessionMode(sessionID) + if !wasSubscribed { + h.replaySessionLaunchWarning(client, sessionID) + } return !wasSubscribed } diff --git a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go index c66d34fdf1..730e10088e 100644 --- a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go +++ b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go @@ -1,6 +1,7 @@ package websocket import ( + "encoding/json" "testing" "github.com/kandev/kandev/internal/auth/authn" @@ -47,6 +48,45 @@ func TestBroadcastToSessionReachesFocusedAndSubscribedClients(t *testing.T) { } } +func TestSessionLaunchWarningReplaysToLateSubscriber(t *testing.T) { + h := newTestHub(t) + warning, err := ws.NewNotification(ws.ActionSessionLaunchWarning, map[string]any{ + "session_id": "sess-1", + "host": "10.0.0.5", + }) + if err != nil { + t.Fatalf("notification: %v", err) + } + h.rememberAndBroadcastSessionLaunchWarning("sess-1", warning) + + late := newTestClient("late") + registerTestClient(h, late) + if !h.SubscribeToSession(late, "sess-1") { + t.Fatal("late subscriber should create a new membership") + } + + select { + case data := <-late.send: + var got ws.Message + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("decode replay: %v", err) + } + if got.Action != ws.ActionSessionLaunchWarning { + t.Fatalf("replayed action = %q, want %q", got.Action, ws.ActionSessionLaunchWarning) + } + default: + t.Fatal("late subscriber did not receive the cached launch warning") + } + + h.clearSessionLaunchWarning("sess-1") + newClient := newTestClient("new") + registerTestClient(h, newClient) + h.SubscribeToSession(newClient, "sess-1") + if clientReceived(newClient) { + t.Fatal("cleared launch warning was replayed") + } +} + func TestBroadcastToSessionDeduplicatesFocusedSubscription(t *testing.T) { h := newTestHub(t) c := newTestClient("c1") diff --git a/apps/backend/internal/gateway/websocket/launch_warning_cache.go b/apps/backend/internal/gateway/websocket/launch_warning_cache.go new file mode 100644 index 0000000000..92cf51723c --- /dev/null +++ b/apps/backend/internal/gateway/websocket/launch_warning_cache.go @@ -0,0 +1,50 @@ +package websocket + +import ( + "encoding/json" + + ws "github.com/kandev/kandev/pkg/websocket" +) + +// rememberAndBroadcastSessionLaunchWarning stores the latest warning before +// the live fan-out. This ordering closes the subscribe-versus-publish race: +// a client either receives the live notification or replays the cached frame +// when it joins later. +func (h *Hub) rememberAndBroadcastSessionLaunchWarning(sessionID string, message *ws.Message) { + if sessionID == "" || message == nil { + return + } + data, err := json.Marshal(message) + if err != nil { + return + } + h.mu.Lock() + if h.sessionLaunchWarnings == nil { + h.sessionLaunchWarnings = make(map[string][]byte) + } + h.sessionLaunchWarnings[sessionID] = append([]byte(nil), data...) + h.mu.Unlock() + h.BroadcastToSession(sessionID, message) +} + +func (h *Hub) clearSessionLaunchWarning(sessionID string) { + if sessionID == "" { + return + } + h.mu.Lock() + delete(h.sessionLaunchWarnings, sessionID) + h.mu.Unlock() +} + +func (h *Hub) replaySessionLaunchWarning(client *Client, sessionID string) { + if client == nil || sessionID == "" { + return + } + h.mu.RLock() + data := append([]byte(nil), h.sessionLaunchWarnings[sessionID]...) + h.mu.RUnlock() + if len(data) == 0 { + return + } + client.sendNotificationFrame(newOutboundNotification(data, ws.ActionSessionLaunchWarning)) +} diff --git a/apps/backend/internal/gateway/websocket/session_notifications.go b/apps/backend/internal/gateway/websocket/session_notifications.go index 34de44503d..463a356e4b 100644 --- a/apps/backend/internal/gateway/websocket/session_notifications.go +++ b/apps/backend/internal/gateway/websocket/session_notifications.go @@ -53,6 +53,7 @@ func RegisterSessionStreamNotifications(ctx context.Context, eventBus bus.EventB b.subscribe(eventBus, events.BuildSessionModelsWildcardSubject(), ws.ActionSessionModelsUpdated) b.subscribe(eventBus, events.BuildSessionModelFallbackWildcardSubject(), ws.ActionSessionModelFallback) b.subscribe(eventBus, events.BuildSessionModelSelectionWarningWildcardSubject(), ws.ActionSessionModelSelectionWarning) + b.subscribe(eventBus, events.BuildSessionLaunchWarningWildcardSubject(), ws.ActionSessionLaunchWarning) b.subscribe(eventBus, events.BuildSessionMCPStatusWildcardSubject(), ws.ActionSessionMCPStatusUpdated) b.subscribe(eventBus, events.BuildSessionInfoWildcardSubject(), ws.ActionSessionInfoUpdated) b.subscribe(eventBus, events.BuildSessionTodosWildcardSubject(), ws.ActionSessionTodosUpdated) @@ -95,6 +96,10 @@ func (b *SessionStreamBroadcaster) subscribe(eventBus bus.EventBus, subject, act b.logger.Error("failed to build websocket notification", zap.String("action", action), zap.Error(err)) return nil } + if action == ws.ActionSessionLaunchWarning { + b.hub.rememberAndBroadcastSessionLaunchWarning(sessionID, msg) + return nil + } b.hub.BroadcastToSession(sessionID, msg) return nil }) diff --git a/apps/backend/internal/gateway/websocket/session_notifications_test.go b/apps/backend/internal/gateway/websocket/session_notifications_test.go index daabb02523..5b3a2ece05 100644 --- a/apps/backend/internal/gateway/websocket/session_notifications_test.go +++ b/apps/backend/internal/gateway/websocket/session_notifications_test.go @@ -236,3 +236,78 @@ func TestSessionStreamBroadcaster_BroadcastsFallbackNotification(t *testing.T) { } } } + +// TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject verifies the +// session stream broadcaster subscribes the session.launch.warning wildcard +// subject task 05 publishes on. Without this the event never leaves the bus. +func TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject(t *testing.T) { + log := testLogger() + eventBus := &subscriptionRecordingEventBus{MemoryEventBus: bus.NewMemoryEventBus(log)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hub := NewHub(nil, log) + b := RegisterSessionStreamNotifications(ctx, eventBus, hub, log) + _ = b + + for _, subject := range eventBus.subjects { + if subject == events.BuildSessionLaunchWarningWildcardSubject() { + return + } + } + t.Fatalf("session stream broadcaster must subscribe the launch warning wildcard subject %q", + events.BuildSessionLaunchWarningWildcardSubject()) +} + +// TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification verifies a +// published session.launch.warning event is routed to the session's +// subscribers as the session.launch.warning action. +func TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification(t *testing.T) { + hub := newAccessTestHub(t) + eventBus := bus.NewMemoryEventBus(testLogger()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _ = RegisterSessionStreamNotifications(ctx, eventBus, hub, testLogger()) + + client := registerAccessClient(t, hub, "client-launch-warning", authn.Identity{UserID: "user-1", Role: authn.RoleMember}) + raw, err := json.Marshal(map[string]interface{}{"session_id": "session-warn-1"}) + if err != nil { + t.Fatalf("marshal subscribe payload: %v", err) + } + client.handleSessionSubscribe(&ws.Message{ + ID: "1", Type: ws.MessageTypeRequest, Action: ws.ActionSessionSubscribe, Payload: raw, + }) + + sessionID := "session-warn-1" + _ = eventBus.Publish(ctx, events.BuildSessionLaunchWarningSubject(sessionID), bus.NewEvent( + events.SessionLaunchWarning, + "test", + &lifecycle.LaunchWarningEventPayload{ + TaskID: "task-1", + SessionID: sessionID, + ExecutorID: "executor-1", + Host: "10.0.0.5", + State: "unreachable", + Reason: "timeout", + Timestamp: time.Now().UTC().Format(time.RFC3339), + }, + )) + + timeout := time.After(2 * time.Second) + for { + select { + case raw := <-client.send: + var msg ws.Message + if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning { + return + } + case raw := <-client.controlSend: + var msg ws.Message + if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning { + return + } + case <-timeout: + t.Fatalf("client did not receive %s", ws.ActionSessionLaunchWarning) + } + } +} diff --git a/apps/backend/internal/gateway/websocket/task_notifications.go b/apps/backend/internal/gateway/websocket/task_notifications.go index 3c99609679..22b53a60a3 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications.go +++ b/apps/backend/internal/gateway/websocket/task_notifications.go @@ -75,6 +75,7 @@ func RegisterTaskNotifications(ctx context.Context, eventBus bus.EventBus, hub * b.subscribe(eventBus, events.ExecutorCreated, ws.ActionExecutorCreated) b.subscribe(eventBus, events.ExecutorUpdated, ws.ActionExecutorUpdated) b.subscribe(eventBus, events.ExecutorDeleted, ws.ActionExecutorDeleted) + b.subscribe(eventBus, events.ExecutorReachabilityChanged, ws.ActionExecutorReachabilityChanged) b.subscribe(eventBus, events.ExecutorProfileCreated, ws.ActionExecutorProfileCreated) b.subscribe(eventBus, events.ExecutorProfileUpdated, ws.ActionExecutorProfileUpdated) b.subscribe(eventBus, events.ExecutorProfileDeleted, ws.ActionExecutorProfileDeleted) @@ -290,6 +291,9 @@ func (b *TaskEventBroadcaster) routeBroadcast( return nil } case ws.ActionSessionStateChanged: + if sessionID != "" && extractStringField(data, "new_state") == string(models.TaskSessionStateRunning) { + b.hub.clearSessionLaunchWarning(sessionID) + } // Broadcast beyond the session subscribers so the sidebar task // switcher can track state changes for all tasks — but scoped to // the owning workspace's user when auth is enabled. @@ -304,6 +308,7 @@ func (b *TaskEventBroadcaster) routeBroadcast( } case ws.ActionSessionRemoved: if sessionID != "" { + b.hub.clearSessionLaunchWarning(sessionID) if _, hasReceipt := conversationReceiptFromData(data); hasReceipt { b.hub.BroadcastConversationMutation(data) } diff --git a/apps/backend/internal/gateway/websocket/task_notifications_executor_reachability_test.go b/apps/backend/internal/gateway/websocket/task_notifications_executor_reachability_test.go new file mode 100644 index 0000000000..c52a7b0f3f --- /dev/null +++ b/apps/backend/internal/gateway/websocket/task_notifications_executor_reachability_test.go @@ -0,0 +1,36 @@ +package websocket + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + ws "github.com/kandev/kandev/pkg/websocket" +) + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3 +// +// events.ExecutorReachabilityChanged must reach WS clients as +// ws.ActionExecutorReachabilityChanged. Executor events carry no +// workspace_id, so — like ExecutorUpdated before it — this rides the +// broadcaster's trailing default case (global broadcast) rather than a +// dedicated routeBroadcast switch entry. +func TestTaskEventBroadcaster_BroadcastsExecutorReachabilityChanged(t *testing.T) { + log := testLogger() + eventBus := bus.NewMemoryEventBus(log) + hub := NewHub(nil, log) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _ = RegisterTaskNotifications(ctx, eventBus, hub, log) + + payload := map[string]interface{}{"executor_id": "exec-1", "state": "unreachable", "reason": "network"} + _ = eventBus.Publish(ctx, events.ExecutorReachabilityChanged, bus.NewEvent( + events.ExecutorReachabilityChanged, "test", payload, + )) + + msg := <-hub.broadcast + if msg.Action != ws.ActionExecutorReachabilityChanged { + t.Fatalf("broadcast action = %q, want %q", msg.Action, ws.ActionExecutorReachabilityChanged) + } +} diff --git a/apps/backend/internal/gateway/websocket/task_notifications_test.go b/apps/backend/internal/gateway/websocket/task_notifications_test.go index 5dbb0a739c..d81cb748a0 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications_test.go +++ b/apps/backend/internal/gateway/websocket/task_notifications_test.go @@ -226,7 +226,7 @@ func TestTaskEventBroadcaster_NoDuplicateSubscriptions(t *testing.T) { // // Update this number when adding or removing event subscriptions in // RegisterTaskNotifications — it is intentionally exact. - const wantSubscriptions = 75 + const wantSubscriptions = 76 if got := len(b.subscriptions); got != wantSubscriptions { t.Errorf("RegisterTaskNotifications created %d subscriptions, want %d — "+ "did an event get subscribed twice?", got, wantSubscriptions) diff --git a/apps/backend/internal/office/retention/sweep.go b/apps/backend/internal/office/retention/sweep.go index 9c972d19cf..c8e69a8c34 100644 --- a/apps/backend/internal/office/retention/sweep.go +++ b/apps/backend/internal/office/retention/sweep.go @@ -299,13 +299,13 @@ func (s *Sweeper) sweepRuns(ctx context.Context, q queryer, cfg TableSettings, n }}, satelliteResults{} } return SweptTableResult{TableSweepResult: TableSweepResult{ - Deleted: result.RunsDeleted, - Backlog: eligible > int64(batchLimit), - }}, satelliteResults{ - RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted}, - RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted}, - RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted}, - } + Deleted: result.RunsDeleted, + Backlog: eligible > int64(batchLimit), + }}, satelliteResults{ + RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted}, + RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted}, + RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted}, + } } // retentionCutoff turns a table's configured window into the instant a diff --git a/apps/backend/internal/ssh/handlers.go b/apps/backend/internal/ssh/handlers.go index ce9e572905..4c3cdfc6e4 100644 --- a/apps/backend/internal/ssh/handlers.go +++ b/apps/backend/internal/ssh/handlers.go @@ -25,7 +25,10 @@ import ( ws "github.com/kandev/kandev/pkg/websocket" ) -const errorJSONKey = "error" +const ( + errorJSONKey = "error" + errExecutorIDRequired = "executor id required" +) // ExecutorRunningLister is the narrow repository slice we need to surface // active SSH sessions. We list all running executors and resolve each row's @@ -54,11 +57,13 @@ type AgentLister interface { // Handler exposes /api/v1/ssh routes used by the settings UI. type Handler struct { - repo ExecutorRunningLister - executorFetcher ExecutorFetcher - agents AgentLister - resolver *lifecycle.AgentctlResolver - logger *logger.Logger + repo ExecutorRunningLister + executorFetcher ExecutorFetcher + agents AgentLister + resolver *lifecycle.AgentctlResolver + logger *logger.Logger + reachabilityRepo ReachabilityLister + reachabilityPoller ReachabilityProber } // NewHandler builds an SSH HTTP/WS handler. @@ -68,13 +73,17 @@ func NewHandler( agents AgentLister, resolver *lifecycle.AgentctlResolver, log *logger.Logger, + reachabilityRepo ReachabilityLister, + reachabilityPoller ReachabilityProber, ) *Handler { return &Handler{ - repo: repo, - executorFetcher: executorFetcher, - agents: agents, - resolver: resolver, - logger: log.WithFields(zap.String("component", "ssh-handler")), + repo: repo, + executorFetcher: executorFetcher, + agents: agents, + resolver: resolver, + logger: log.WithFields(zap.String("component", "ssh-handler")), + reachabilityRepo: reachabilityRepo, + reachabilityPoller: reachabilityPoller, } } @@ -87,12 +96,14 @@ func RegisterRoutes( registry *registry.Registry, resolver *lifecycle.AgentctlResolver, log *logger.Logger, + reachabilityRepo ReachabilityLister, + reachabilityPoller ReachabilityProber, ) { var agentLister AgentLister if registry != nil { agentLister = registry } - h := NewHandler(repo, executorFetcher, agentLister, resolver, log) + h := NewHandler(repo, executorFetcher, agentLister, resolver, log, reachabilityRepo, reachabilityPoller) h.registerHTTP(router) h.registerWS(dispatcher) } @@ -103,6 +114,9 @@ func (h *Handler) registerHTTP(router *gin.Engine) { api.GET("/executors/:id/sessions", h.httpListSessions) api.POST("/executors/:id/probe-agents", h.httpProbeAgents) api.POST("/executors/:id/probe-shells", h.httpProbeShells) + api.GET("/reachability", h.httpListReachability) + api.GET("/executors/:id/reachability", h.httpGetReachability) + api.POST("/executors/:id/reachability/probe", h.httpProbeReachability) } func (h *Handler) registerWS(dispatcher *ws.Dispatcher) { @@ -179,7 +193,7 @@ func (h *Handler) httpTest(c *gin.Context) { func (h *Handler) httpListSessions(c *gin.Context) { id := strings.TrimSpace(c.Param("id")) if id == "" { - c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"}) + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired}) return } rows, err := h.listSessions(c.Request.Context(), id) @@ -232,7 +246,7 @@ type ProbeAgentsRequest struct { func (h *Handler) httpProbeAgents(c *gin.Context) { id := strings.TrimSpace(c.Param("id")) if id == "" { - c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"}) + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired}) return } if h.executorFetcher == nil || h.agents == nil { @@ -332,7 +346,7 @@ type ProbeShellsResponse struct { func (h *Handler) httpProbeShells(c *gin.Context) { id := strings.TrimSpace(c.Param("id")) if id == "" { - c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"}) + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired}) return } if h.executorFetcher == nil { @@ -421,40 +435,13 @@ func (h *Handler) resolveSSHTarget(ctx context.Context, executorID string) (*lif if executor == nil || executor.Type != models.ExecutorTypeSSH { return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID) } - target, err := sshTargetFromExecutorConfig(executor.Config) + target, err := lifecycle.SSHTargetFromExecutorConfig(executor.Config) if err != nil { return nil, http.StatusBadRequest, err } return target, http.StatusOK, nil } -// sshTargetFromExecutorConfig projects an executor.Config into the -// SSHConnConfig the dialer expects. Keeps the handler decoupled from -// lifecycle's internal metadata-vs-config representation. -func sshTargetFromExecutorConfig(cfg map[string]string) (*lifecycle.SSHTarget, error) { - if cfg == nil { - return nil, fmt.Errorf("ssh executor has no config") - } - port := 0 - if p := strings.TrimSpace(cfg["ssh_port"]); p != "" { - n, err := strconv.Atoi(p) - if err != nil || n < 1 || n > 65535 { - return nil, fmt.Errorf("invalid ssh_port %q", p) - } - port = n - } - return lifecycle.ResolveSSHTarget(lifecycle.SSHConnConfig{ - HostAlias: cfg["ssh_host_alias"], - Host: cfg["ssh_host"], - Port: port, - User: cfg["ssh_user"], - IdentitySource: lifecycle.SSHIdentitySource(cfg["ssh_identity_source"]), - IdentityFile: cfg["ssh_identity_file"], - ProxyJump: cfg["ssh_proxy_jump"], - PinnedFingerprint: cfg["ssh_host_fingerprint"], - }) -} - // --- WS handlers --- func (h *Handler) wsTest(ctx context.Context, msg *ws.Message) (*ws.Message, error) { diff --git a/apps/backend/internal/ssh/reachability_handlers.go b/apps/backend/internal/ssh/reachability_handlers.go new file mode 100644 index 0000000000..29e2679906 --- /dev/null +++ b/apps/backend/internal/ssh/reachability_handlers.go @@ -0,0 +1,228 @@ +package ssh + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" + "github.com/kandev/kandev/internal/task/models" +) + +// reachabilityConfigValidator is the production seam over the runtime's SSH +// reachability prober, used only to check whether an executor's config +// resolves to a dialable target — no probe is actually run here. +var reachabilityConfigValidator = agentruntime.NewSSHReachabilityProber() + +// ReachabilityLister is the narrow repository slice the reachability GET +// routes need: every stored record in one query (the list route) or a +// single record (the single-executor route). Executor lookup itself reuses +// ExecutorFetcher, already wired on Handler for the existing SSH routes. +type ReachabilityLister interface { + ListExecutors(ctx context.Context) ([]*models.Executor, error) + ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) + GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) +} + +// ReachabilityProber is the narrow slice of *reachability.Poller the +// immediate-probe route needs. Declared here (not imported as the concrete +// type) so a handler test can substitute a fake without spinning up a real +// poller when it only needs to exercise coalescing or 404/400/409 +// short-circuits. +type ReachabilityProber interface { + ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool) + EffectiveIntervalSeconds() int +} + +// --- HTTP handlers --- + +func (h *Handler) httpListReachability(c *gin.Context) { + dtos, status, err := h.listReachability(c.Request.Context()) + if err != nil { + h.writeReachabilityError(c, status, err) + return + } + c.JSON(status, dtos) +} + +func (h *Handler) httpGetReachability(c *gin.Context) { + id := strings.TrimSpace(c.Param("id")) + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired}) + return + } + dto, status, err := h.getReachability(c.Request.Context(), id) + if err != nil { + h.writeReachabilityError(c, status, err) + return + } + c.JSON(status, dto) +} + +func (h *Handler) httpProbeReachability(c *gin.Context) { + id := strings.TrimSpace(c.Param("id")) + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired}) + return + } + dto, status, err := h.probeReachability(c.Request.Context(), id) + if err != nil { + h.writeReachabilityError(c, status, err) + return + } + c.JSON(status, dto) +} + +func (h *Handler) writeReachabilityError(c *gin.Context, status int, err error) { + message := "reachability request failed" + if status >= http.StatusBadRequest && status < http.StatusInternalServerError { + message = err.Error() + } else if h.logger != nil { + h.logger.Error("reachability request failed", zap.Int("status", status), zap.Error(err)) + } + c.JSON(status, gin.H{errorJSONKey: message}) +} + +// --- Testable cores --- + +// listReachability returns one DTO per SSH executor that is not +// soft-deleted, ordered by executor_id ascending — the same total order the +// poller's own pass uses. An executor whose status is not active is still +// included, carrying its retained record. +func (h *Handler) listReachability(ctx context.Context) ([]reachabilitypkg.RecordDTO, int, error) { + if h.reachabilityRepo == nil { + return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired") + } + executors, err := h.reachabilityRepo.ListExecutors(ctx) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("list executors: %w", err) + } + records, err := h.reachabilityRepo.ListExecutorReachability(ctx) + if err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("list executor reachability: %w", err) + } + byExecutor := make(map[string]*models.ExecutorReachability, len(records)) + for _, record := range records { + byExecutor[record.ExecutorID] = record + } + + sshExecutors := filterSSHExecutors(executors) + sort.Slice(sshExecutors, func(i, j int) bool { return sshExecutors[i].ID < sshExecutors[j].ID }) + + interval := h.reachabilityIntervalSeconds() + dtos := make([]reachabilitypkg.RecordDTO, 0, len(sshExecutors)) + for _, executor := range sshExecutors { + dtos = append(dtos, buildReachabilityDTOFromRecord(executor, byExecutor[executor.ID], interval)) + } + return dtos, http.StatusOK, nil +} + +// getReachability returns the single record for executorID. Never 404s for +// a missing record — only for a missing or soft-deleted executor — per +// AC-EXECUTORS-SSH-REACHABILITY-002.5. +func (h *Handler) getReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) { + executor, status, err := h.loadSSHExecutor(ctx, executorID) + if err != nil { + return nil, status, err + } + if h.reachabilityRepo == nil { + return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired") + } + record, err := h.reachabilityRepo.GetExecutorReachability(ctx, executorID) + if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) { + return nil, http.StatusInternalServerError, fmt.Errorf("get executor reachability: %w", err) + } + if errors.Is(err, models.ErrExecutorReachabilityNotFound) { + record = nil + } + dto := buildReachabilityDTOFromRecord(executor, record, h.reachabilityIntervalSeconds()) + return &dto, http.StatusOK, nil +} + +// probeReachability runs (or joins an in-flight) synchronous probe for +// executorID and returns its persisted record. Coalesces concurrent callers +// through reachabilityPoller.ProbeAndWait's own singleflight group. +func (h *Handler) probeReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) { + executor, status, err := h.loadSSHExecutor(ctx, executorID) + if err != nil { + return nil, status, err + } + if executor.Status != models.ExecutorStatusActive { + return nil, http.StatusConflict, fmt.Errorf("executor %q is not active", executorID) + } + if h.reachabilityPoller == nil { + return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability probing not wired") + } + result, ok := h.reachabilityPoller.ProbeAndWait(executor) + if !ok { + return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability poller is not accepting probes") + } + dto := reachabilitypkg.BuildRecordDTO(executorID, result.Record, h.reachabilityPoller.EffectiveIntervalSeconds(), result.Persisted) + return &dto, http.StatusOK, nil +} + +// loadSSHExecutor enforces the 404/400 split the reachability routes need: +// 404 for a nonexistent or soft-deleted id, 400 only when the executor +// exists but isn't type ssh. Deliberately does not attempt config +// resolution (unlike resolveSSHTarget) — an unresolvable ssh config is a 200 +// with reason config, decided by the caller, never a 400 here. +func (h *Handler) loadSSHExecutor(ctx context.Context, executorID string) (*models.Executor, int, error) { + executor, err := h.executorFetcher.GetExecutor(ctx, executorID) + if err != nil { + if errors.Is(err, models.ErrExecutorNotFound) { + return nil, http.StatusNotFound, fmt.Errorf("executor %q not found: %w", executorID, err) + } + return nil, http.StatusInternalServerError, fmt.Errorf("look up executor %q: %w", executorID, err) + } + if executor.Type != models.ExecutorTypeSSH { + return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID) + } + return executor, http.StatusOK, nil +} + +func (h *Handler) reachabilityIntervalSeconds() int { + if h.reachabilityPoller == nil { + return 0 + } + return h.reachabilityPoller.EffectiveIntervalSeconds() +} + +func filterSSHExecutors(executors []*models.Executor) []*models.Executor { + out := make([]*models.Executor, 0, len(executors)) + for _, executor := range executors { + if executor != nil && executor.Type == models.ExecutorTypeSSH { + out = append(out, executor) + } + } + return out +} + +// buildReachabilityDTOFromRecord projects executor/record into the wire +// shape without any I/O. record nil means "no record yet" — that synthesizes +// the unknown placeholder unless the executor's own config can't resolve a +// target, in which case AC-EXECUTORS-SSH-REACHABILITY-002.5's sibling rule +// applies: 200 with reason config, the same outcome a real probe would reach +// on its very first attempt, rather than a misleading "not probed yet". +func buildReachabilityDTOFromRecord(executor *models.Executor, record *models.ExecutorReachability, intervalSeconds int) reachabilitypkg.RecordDTO { + if record != nil { + return reachabilitypkg.BuildRecordDTO(executor.ID, record, intervalSeconds, true) + } + if err := reachabilityConfigValidator.ValidateConfig(executor.Config); err != nil { + synthetic := &models.ExecutorReachability{ + ExecutorID: executor.ID, + State: models.ExecutorReachabilityStateUnreachable, + Reason: models.ExecutorReachabilityReasonConfig, + Message: agentruntime.SanitizeSSHReachabilityMessage(err), + Host: executor.Config["ssh_host"], + } + return reachabilitypkg.BuildRecordDTO(executor.ID, synthetic, intervalSeconds, false) + } + return reachabilitypkg.BuildRecordDTO(executor.ID, nil, intervalSeconds, false) +} diff --git a/apps/backend/internal/ssh/reachability_handlers_test.go b/apps/backend/internal/ssh/reachability_handlers_test.go new file mode 100644 index 0000000000..0242ccc4e3 --- /dev/null +++ b/apps/backend/internal/ssh/reachability_handlers_test.go @@ -0,0 +1,362 @@ +package ssh + +import ( + "context" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sync/singleflight" + + "github.com/kandev/kandev/internal/common/logger" + reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability" + "github.com/kandev/kandev/internal/task/models" +) + +func testHandlerLogger() *logger.Logger { + return logger.Default() +} + +type fakeReachabilityExecutorFetcher struct { + executors map[string]*models.Executor +} + +func (f *fakeReachabilityExecutorFetcher) GetExecutor(_ context.Context, id string) (*models.Executor, error) { + executor, ok := f.executors[id] + if !ok { + return nil, models.ErrExecutorNotFound + } + return executor, nil +} + +type fakeReachabilityLister struct { + executors []*models.Executor + records []*models.ExecutorReachability + byID map[string]*models.ExecutorReachability +} + +func (f *fakeReachabilityLister) ListExecutors(_ context.Context) ([]*models.Executor, error) { + return f.executors, nil +} + +func (f *fakeReachabilityLister) ListExecutorReachability(_ context.Context) ([]*models.ExecutorReachability, error) { + return f.records, nil +} + +func (f *fakeReachabilityLister) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) { + if record, ok := f.byID[executorID]; ok { + return record, nil + } + return nil, models.ErrExecutorReachabilityNotFound +} + +func sshExecutor(id string, status models.ExecutorStatus, config map[string]string) *models.Executor { + return &models.Executor{ID: id, Type: models.ExecutorTypeSSH, Status: status, Config: config} +} + +func resolvableSSHConfig() map[string]string { + return map[string]string{ + "ssh_host": "10.0.0.1", + "ssh_user": "deploy", + "ssh_host_fingerprint": "SHA256:test", + } +} + +// fakeReachabilityProber models the exact guarantee *reachability.Poller +// makes for concurrent callers of the same executor id: it coalesces +// through the same golang.org/x/sync/singleflight primitive the production +// Poller uses, so this test proves the handler's own plumbing (always +// routing through the one shared prober instance) rather than re-deriving +// singleflight's own coalescing guarantee, which is already covered by +// TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor. +type fakeReachabilityProber struct { + group singleflight.Group + dialCount int32 + callsEntered int32 + gate chan struct{} + result reachabilitypkg.ProbeResult + interval int +} + +func (f *fakeReachabilityProber) ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool) { + atomic.AddInt32(&f.callsEntered, 1) + v, _, _ := f.group.Do(executor.ID, func() (any, error) { + atomic.AddInt32(&f.dialCount, 1) + <-f.gate + return f.result, nil + }) + return v.(reachabilitypkg.ProbeResult), true +} + +func (f *fakeReachabilityProber) EffectiveIntervalSeconds() int { return f.interval } + +func waitForCallsEntered(t *testing.T, prober *fakeReachabilityProber, want int32) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if atomic.LoadInt32(&prober.callsEntered) >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("callsEntered did not reach %d within 1s (got %d)", want, atomic.LoadInt32(&prober.callsEntered)) +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002 (coalescing) +// +// Two overlapping probe requests for the same executor must share one dial +// and receive one identical record. Written first as a RED test against a +// handler that has not yet been implemented, per the task-04 work order. +func TestProbeReachability_CoalescesConcurrentProbes(t *testing.T) { + executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive} + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + updatedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + prober := &fakeReachabilityProber{ + gate: make(chan struct{}), + result: reachabilitypkg.ProbeResult{ + Record: &models.ExecutorReachability{ + ExecutorID: "exec-1", + State: models.ExecutorReachabilityStateReachable, + UpdatedAt: updatedAt, + }, + Persisted: true, + }, + interval: 30, + } + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober) + + const callers = 2 + dtos := make([]*reachabilitypkg.RecordDTO, callers) + statuses := make([]int, callers) + errs := make([]error, callers) + + var wg sync.WaitGroup + for i := 0; i < callers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + dto, status, err := h.probeReachability(context.Background(), "exec-1") + dtos[i] = dto + statuses[i] = status + errs[i] = err + }(i) + } + + waitForCallsEntered(t, prober, callers) + close(prober.gate) + wg.Wait() + + if got := atomic.LoadInt32(&prober.dialCount); got != 1 { + t.Fatalf("dialCount = %d, want 1 (concurrent probes must coalesce into one dial)", got) + } + for i := 0; i < callers; i++ { + if errs[i] != nil { + t.Fatalf("caller %d: unexpected error: %v", i, errs[i]) + } + if statuses[i] != 200 { + t.Fatalf("caller %d: status = %d, want 200", i, statuses[i]) + } + } + if !reflect.DeepEqual(dtos[0], dtos[1]) { + t.Fatalf("responses differ: %+v vs %+v", dtos[0], dtos[1]) + } + if dtos[0].State != string(models.ExecutorReachabilityStateReachable) { + t.Fatalf("state = %q, want %q", dtos[0].State, models.ExecutorReachabilityStateReachable) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5 +func TestGetReachability_NoRecordAndResolvableConfigReturnsUnknown(t *testing.T) { + executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig()) + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + lister := &fakeReachabilityLister{} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil) + + dto, status, err := h.getReachability(context.Background(), "exec-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != 200 { + t.Fatalf("status = %d, want 200", status) + } + if dto.State != string(models.ExecutorReachabilityStateUnknown) { + t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateUnknown) + } + if dto.Reason != "" { + t.Fatalf("reason = %q, want empty", dto.Reason) + } + if dto.Persisted { + t.Fatal("persisted = true for a synthesized never-probed record") + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5 +// +// An ssh executor whose config cannot resolve a target (no ssh_host here) +// must answer 200 with reason "config", never 400 — resolveSSHTarget's own +// 400-on-unresolvable-config mapping does not apply to the reachability +// routes. +func TestGetReachability_NoRecordAndUnresolvableConfigReturnsReasonConfig(t *testing.T) { + executor := sshExecutor("exec-1", models.ExecutorStatusActive, map[string]string{}) + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + lister := &fakeReachabilityLister{} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil) + + dto, status, err := h.getReachability(context.Background(), "exec-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != 200 { + t.Fatalf("status = %d, want 200", status) + } + if dto.Reason != string(models.ExecutorReachabilityReasonConfig) { + t.Fatalf("reason = %q, want %q", dto.Reason, models.ExecutorReachabilityReasonConfig) + } + if dto.Persisted { + t.Fatal("persisted = true for a synthesized record that was never written") + } +} + +func TestGetReachability_ReturnsStoredRecordVerbatim(t *testing.T) { + executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig()) + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + stored := &models.ExecutorReachability{ + ExecutorID: "exec-1", + State: models.ExecutorReachabilityStateReachable, + Host: "10.0.0.1", + } + lister := &fakeReachabilityLister{byID: map[string]*models.ExecutorReachability{"exec-1": stored}} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil) + + dto, status, err := h.getReachability(context.Background(), "exec-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != 200 { + t.Fatalf("status = %d, want 200", status) + } + if !dto.Persisted { + t.Fatal("persisted = false for an actually-stored record") + } + if dto.State != string(models.ExecutorReachabilityStateReachable) { + t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateReachable) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5 +func TestGetReachability_NonexistentExecutorReturns404(t *testing.T) { + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil) + + _, status, err := h.getReachability(context.Background(), "missing") + if err == nil { + t.Fatal("expected an error for a nonexistent executor") + } + if status != 404 { + t.Fatalf("status = %d, want 404", status) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5 +// +// A wrong-type executor id is a 400, distinct from the config-unresolvable +// case (which is a 200 with reason config). +func TestGetReachability_WrongTypeExecutorReturns400(t *testing.T) { + executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive} + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil) + + _, status, err := h.getReachability(context.Background(), "exec-1") + if err == nil { + t.Fatal("expected an error for a non-ssh executor") + } + if status != 400 { + t.Fatalf("status = %d, want 400", status) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.4 +// +// The list route orders by executor_id ascending, includes a deactivated +// ssh executor's retained record, and omits non-ssh executors entirely. +func TestListReachability_OrdersFiltersAndIncludesDeactivated(t *testing.T) { + sshA := sshExecutor("ssh-b", models.ExecutorStatusActive, resolvableSSHConfig()) + sshB := sshExecutor("ssh-a", models.ExecutorStatusDisabled, resolvableSSHConfig()) + nonSSH := &models.Executor{ID: "local-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive} + fetcher := &fakeReachabilityExecutorFetcher{} + lister := &fakeReachabilityLister{ + executors: []*models.Executor{sshA, sshB, nonSSH}, + records: []*models.ExecutorReachability{ + {ExecutorID: "ssh-a", State: models.ExecutorReachabilityStateUnreachable, Reason: models.ExecutorReachabilityReasonTimeout}, + }, + } + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil) + + dtos, status, err := h.listReachability(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if status != 200 { + t.Fatalf("status = %d, want 200", status) + } + if len(dtos) != 2 { + t.Fatalf("len(dtos) = %d, want 2 (non-ssh executor must be omitted)", len(dtos)) + } + if dtos[0].ExecutorID != "ssh-a" || dtos[1].ExecutorID != "ssh-b" { + t.Fatalf("order = [%s, %s], want [ssh-a, ssh-b] (executor_id ascending)", dtos[0].ExecutorID, dtos[1].ExecutorID) + } + if dtos[0].State != string(models.ExecutorReachabilityStateUnreachable) { + t.Fatalf("deactivated executor's record state = %q, want %q (retained record)", dtos[0].State, models.ExecutorReachabilityStateUnreachable) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-002.6 +// +// A POST against a deactivated ssh executor answers 409 without probing. +func TestProbeReachability_InactiveExecutorReturns409WithoutProbing(t *testing.T) { + executor := sshExecutor("exec-1", models.ExecutorStatusDisabled, resolvableSSHConfig()) + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + prober := &fakeReachabilityProber{gate: make(chan struct{})} + close(prober.gate) // would return immediately if ever called + + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober) + _, status, err := h.probeReachability(context.Background(), "exec-1") + if err == nil { + t.Fatal("expected an error for a probe against an inactive executor") + } + if status != 409 { + t.Fatalf("status = %d, want 409", status) + } + if got := atomic.LoadInt32(&prober.callsEntered); got != 0 { + t.Fatalf("callsEntered = %d, want 0 (must not probe an inactive executor)", got) + } +} + +func TestProbeReachability_NonexistentExecutorReturns404(t *testing.T) { + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{}) + + _, status, err := h.probeReachability(context.Background(), "missing") + if err == nil { + t.Fatal("expected an error for a nonexistent executor") + } + if status != 404 { + t.Fatalf("status = %d, want 404", status) + } +} + +func TestProbeReachability_WrongTypeExecutorReturns400(t *testing.T) { + executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive} + fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}} + h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{}) + + _, status, err := h.probeReachability(context.Background(), "exec-1") + if err == nil { + t.Fatal("expected an error for a non-ssh executor") + } + if status != 400 { + t.Fatalf("status = %d, want 400", status) + } +} diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index 1461399a80..5ff24a84b0 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -607,6 +607,24 @@ func (m *mockRepository) UpdateExecutorRunningStatus(ctx context.Context, sessio func (m *mockRepository) RepairExecutorRunningDead(ctx context.Context, sessionID string) error { return nil } +func (m *mockRepository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) { + return nil, nil +} +func (m *mockRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) { + return nil, models.ErrExecutorReachabilityNotFound +} +func (m *mockRepository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) { + return nil, nil +} +func (m *mockRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error { + return nil +} +func (m *mockRepository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error { + return nil +} +func (m *mockRepository) DeleteExecutorReachability(ctx context.Context, executorID string) error { + return nil +} func (m *mockRepository) CreateEnvironment(ctx context.Context, environment *models.Environment) error { return nil } diff --git a/apps/backend/internal/task/models/executor_reachability.go b/apps/backend/internal/task/models/executor_reachability.go new file mode 100644 index 0000000000..c60d81fd33 --- /dev/null +++ b/apps/backend/internal/task/models/executor_reachability.go @@ -0,0 +1,69 @@ +package models + +import ( + "errors" + "time" +) + +// ErrExecutorReachabilityNotFound is returned when no reachability record +// exists for the requested executor. +var ErrExecutorReachabilityNotFound = errors.New("executor reachability record not found") + +// ExecutorReachabilityState is the coarse status shown to operators. +type ExecutorReachabilityState string + +const ( + ExecutorReachabilityStateUnknown ExecutorReachabilityState = "unknown" + ExecutorReachabilityStateReachable ExecutorReachabilityState = "reachable" + ExecutorReachabilityStateUnreachable ExecutorReachabilityState = "unreachable" +) + +// ExecutorReachabilityReason mirrors lifecycle.SSHReachabilityReason's closed +// set as plain strings — this package sits below the agent runtime tier and +// must not import it. An empty reason means the observation was a success. +type ExecutorReachabilityReason string + +const ( + ExecutorReachabilityReasonConfig ExecutorReachabilityReason = "config" + ExecutorReachabilityReasonTimeout ExecutorReachabilityReason = "timeout" + ExecutorReachabilityReasonHostKey ExecutorReachabilityReason = "host_key" + ExecutorReachabilityReasonAuth ExecutorReachabilityReason = "auth" + ExecutorReachabilityReasonNetwork ExecutorReachabilityReason = "network" + ExecutorReachabilityReasonUnknown ExecutorReachabilityReason = "unknown" +) + +// ExecutorReachability is the single stored row for one SSH executor's +// observed reachability. It lives in its own table, never on Executor +// itself — Executor.Config is user-authored input and Executor.Status is a +// user-controlled switch, and neither should share a row with observed +// state. +type ExecutorReachability struct { + ExecutorID string `json:"executor_id"` + State ExecutorReachabilityState `json:"state"` + Reason ExecutorReachabilityReason `json:"reason"` + Message string `json:"message"` + ConsecutiveFailures int `json:"consecutive_failures"` + Host string `json:"host"` + CheckedAt *time.Time `json:"checked_at,omitempty"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ExecutorReachabilityObservation is what a single probe (or a launch's own +// dial attempt) reports to UpsertExecutorReachability. The derived counter +// and state for an existing row live entirely in the upsert's own SQL — this +// struct carries only the observation and the parameters the SQL needs to +// derive from it. InitialState and InitialFailures are used only when no row +// exists yet for the executor; every later write derives both from the +// stored row instead. +type ExecutorReachabilityObservation struct { + ExecutorID string + SeenUpdatedAt time.Time + InitialState ExecutorReachabilityState + InitialFailures int + Reason ExecutorReachabilityReason + Message string + Host string + CheckedAt time.Time + FailureThreshold int +} diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index da109949aa..e02f76d19f 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -706,6 +706,33 @@ type ExecutorRepository interface { // the resume-safety invariant instead of deleting a resumable row. // Returns models.ErrExecutorRunningNotFound if no row exists for the session. RepairExecutorRunningDead(ctx context.Context, sessionID string) error + + // ListSSHExecutorsForReachability returns every eligible SSH executor + // (type=ssh, not soft-deleted, status=active) ordered ascending by id — + // the poller's per-pass work list. + ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) + // GetExecutorReachability returns the stored record for one executor. + // Returns models.ErrExecutorReachabilityNotFound if none exists. + GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) + // ListExecutorReachability returns every stored reachability record. + ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) + // UpsertExecutorReachability records a single probe (or launch dial) + // observation. The consecutive-failure counter and derived state are + // computed by the statement itself from the row's own prior values, and + // a write is discarded when obs.CheckedAt is not strictly later than the + // stored checked_at — see the system design's Persistence section. + UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error + // ResetExecutorReachability invalidates the stored record after a + // connection-configuration save when seenUpdatedAt still matches the + // executor row. The version guard prevents a delayed save callback from + // resetting a newer configuration. State becomes unknown, the counter and + // reason/message clear, host is set to the newly saved value, and both + // timestamps become NULL. A no-op (zero rows affected) when the executor + // is not an active SSH executor or the version is stale. + ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error + // DeleteExecutorReachability removes the stored record. DeleteExecutor + // calls this in the same transaction as the soft delete. + DeleteExecutorReachability(ctx context.Context, executorID string) error } // EnvironmentRepository handles environment CRUD. diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 9f3a97a089..9e1f3e065f 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -423,6 +423,24 @@ func (r *Repository) runMigrations(ctx context.Context) error { // allocated. _ = r.migrate.Apply("workflow_step_entries.marker_positions", `ALTER TABLE workflow_step_entries ADD COLUMN marker_positions TEXT NOT NULL DEFAULT ''`) + // One row per SSH executor holding observed reachability — deliberately + // not columns on executors, which holds user-authored config and a + // user-controlled status switch. No foreign key: deletion is explicit + // (DeleteExecutor deletes the row in the same transaction as the soft + // delete), not a cascade. + _ = r.migrate.Apply("executor_reachability.table", ` + CREATE TABLE IF NOT EXISTS executor_reachability ( + executor_id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'unknown', + reason TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + consecutive_failures INTEGER NOT NULL DEFAULT 0, + host TEXT NOT NULL DEFAULT '', + checked_at TIMESTAMP, + last_success_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL + )`) + // Checked last so a failure on any required migration above -- // including this file's own marker_positions column -- fails startup // instead of leaving a schema that allocateStepEntryIfPending can't write to. diff --git a/apps/backend/internal/task/repository/sqlite/executor.go b/apps/backend/internal/task/repository/sqlite/executor.go index 9d37afde1e..a692ca078d 100644 --- a/apps/backend/internal/task/repository/sqlite/executor.go +++ b/apps/backend/internal/task/repository/sqlite/executor.go @@ -94,7 +94,13 @@ func (r *Repository) UpdateExecutor(ctx context.Context, executor *models.Execut func (r *Repository) DeleteExecutor(ctx context.Context, id string) error { now := time.Now().UTC() - result, err := r.db.ExecContext(ctx, r.db.Rebind(` + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return fmt.Errorf("begin executor delete: %w", err) + } + defer func() { _ = tx.Rollback() }() + + result, err := tx.ExecContext(ctx, tx.Rebind(` UPDATE executors SET deleted_at = ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL `), now, now, id) if err != nil { @@ -104,9 +110,42 @@ func (r *Repository) DeleteExecutor(ctx context.Context, id string) error { if rows == 0 { return fmt.Errorf("executor not found: %s", id) } + if _, err := tx.ExecContext(ctx, tx.Rebind(` + DELETE FROM executor_reachability WHERE executor_id = ? + `), id); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit executor delete: %w", err) + } return nil } +// scanExecutorRow scans one row of the standard executors column list +// (id, name, type, status, is_system, resumable, config, created_at, +// updated_at, deleted_at) into a models.Executor. Shared by every query that +// selects that exact column set. +func scanExecutorRow(scanner interface{ Scan(...any) error }) (*models.Executor, error) { + executor := &models.Executor{} + var configJSON string + var isSystem int + var resumable int + if err := scanner.Scan( + &executor.ID, &executor.Name, &executor.Type, &executor.Status, + &isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt, + ); err != nil { + return nil, err + } + executor.IsSystem = isSystem == 1 + executor.Resumable = resumable == 1 + if configJSON != "" && configJSON != "{}" { + if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil { + return nil, fmt.Errorf("failed to deserialize executor config: %w", err) + } + } + return executor, nil +} + func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, error) { rows, err := r.ro.QueryContext(ctx, ` SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at @@ -119,23 +158,10 @@ func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, err var result []*models.Executor for rows.Next() { - executor := &models.Executor{} - var configJSON string - var isSystem int - var resumable int - if err := rows.Scan( - &executor.ID, &executor.Name, &executor.Type, &executor.Status, - &isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt, - ); err != nil { + executor, err := scanExecutorRow(rows) + if err != nil { return nil, err } - executor.IsSystem = isSystem == 1 - executor.Resumable = resumable == 1 - if configJSON != "" && configJSON != "{}" { - if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil { - return nil, fmt.Errorf("failed to deserialize executor config: %w", err) - } - } result = append(result, executor) } return result, rows.Err() diff --git a/apps/backend/internal/task/repository/sqlite/executor_reachability.go b/apps/backend/internal/task/repository/sqlite/executor_reachability.go new file mode 100644 index 0000000000..ab8e42942e --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/executor_reachability.go @@ -0,0 +1,183 @@ +package sqlite + +import ( + "context" + "database/sql" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +const executorReachabilityColumns = `executor_id, state, reason, message, consecutive_failures, host, checked_at, last_success_at, updated_at` + +func scanExecutorReachability(scanner interface{ Scan(...any) error }) (*models.ExecutorReachability, error) { + r := &models.ExecutorReachability{} + var state, reason string + var checkedAt, lastSuccessAt sql.NullTime + if err := scanner.Scan( + &r.ExecutorID, &state, &reason, &r.Message, &r.ConsecutiveFailures, &r.Host, + &checkedAt, &lastSuccessAt, &r.UpdatedAt, + ); err != nil { + return nil, err + } + r.State = models.ExecutorReachabilityState(state) + r.Reason = models.ExecutorReachabilityReason(reason) + if checkedAt.Valid { + v := checkedAt.Time + r.CheckedAt = &v + } + if lastSuccessAt.Valid { + v := lastSuccessAt.Time + r.LastSuccessAt = &v + } + return r, nil +} + +// ListSSHExecutorsForReachability returns every eligible SSH executor +// (type=ssh, not soft-deleted, status=active) ordered ascending by id, the +// primary key, never reused, so the order is total and needs no tiebreak. +func (r *Repository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) { + rows, err := r.ro.QueryContext(ctx, ` + SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at + FROM executors + WHERE type = 'ssh' AND deleted_at IS NULL AND status = 'active' + ORDER BY id ASC + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var result []*models.Executor + for rows.Next() { + executor, err := scanExecutorRow(rows) + if err != nil { + return nil, err + } + result = append(result, executor) + } + return result, rows.Err() +} + +// GetExecutorReachability returns the stored record for one executor. +// Returns models.ErrExecutorReachabilityNotFound if none exists. +func (r *Repository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) { + row := r.ro.QueryRowContext(ctx, r.ro.Rebind(` + SELECT `+executorReachabilityColumns+` FROM executor_reachability WHERE executor_id = ? + `), executorID) + rec, err := scanExecutorReachability(row) + if err != nil { + if err == sql.ErrNoRows { + return nil, models.ErrExecutorReachabilityNotFound + } + return nil, err + } + return rec, nil +} + +// ListExecutorReachability returns every stored reachability record. +func (r *Repository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) { + rows, err := r.ro.QueryContext(ctx, ` + SELECT `+executorReachabilityColumns+` FROM executor_reachability ORDER BY executor_id ASC + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var result []*models.ExecutorReachability + for rows.Next() { + rec, err := scanExecutorReachability(rows) + if err != nil { + return nil, err + } + result = append(result, rec) + } + return result, rows.Err() +} + +// UpsertExecutorReachability records a single probe (or launch dial) +// observation. The write is a single statement: the consecutive-failure +// counter and the derived state for an existing row are computed by the +// statement itself from the row's own prior values, so no read-modify-write +// in Go can lose a streak to an interleaving. A write whose CheckedAt is not +// strictly later than the stored checked_at is discarded (last-write-wins), +// and the row is only touched at all when the executor is still the active +// SSH executor obs.SeenUpdatedAt was captured against — see the system +// design's Persistence section for the full contract. +func (r *Repository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error { + now := time.Now().UTC() + checkedAt := obs.CheckedAt.UTC() + var lastSuccessAt any + if obs.Reason == "" { + lastSuccessAt = checkedAt + } + _, err := r.db.ExecContext(ctx, r.db.Rebind(` + INSERT INTO executor_reachability ( + executor_id, state, reason, message, consecutive_failures, host, + checked_at, last_success_at, updated_at + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ? + WHERE EXISTS (SELECT 1 FROM executors e + WHERE e.id = ? AND e.deleted_at IS NULL + AND e.status = 'active' AND e.updated_at = ?) + ON CONFLICT (executor_id) DO UPDATE SET + consecutive_failures = CASE WHEN excluded.reason = '' THEN 0 + ELSE executor_reachability.consecutive_failures + 1 END, + state = CASE WHEN excluded.reason = '' THEN 'reachable' + WHEN excluded.reason IN ('config','host_key') THEN 'unreachable' + WHEN executor_reachability.consecutive_failures + 1 >= ? + THEN 'unreachable' + ELSE executor_reachability.state END, + reason = excluded.reason, message = excluded.message, host = excluded.host, + checked_at = excluded.checked_at, + last_success_at = CASE WHEN excluded.reason = '' THEN excluded.checked_at + ELSE executor_reachability.last_success_at END, + updated_at = excluded.updated_at + WHERE executor_reachability.checked_at IS NULL + OR excluded.checked_at > executor_reachability.checked_at + `), + obs.ExecutorID, string(obs.InitialState), string(obs.Reason), obs.Message, obs.InitialFailures, obs.Host, + checkedAt, lastSuccessAt, now, + obs.ExecutorID, obs.SeenUpdatedAt.UTC(), + obs.FailureThreshold, + ) + return err +} + +// ResetExecutorReachability invalidates the stored record after a +// connection-configuration save. It is its own statement, not a branch of +// the upsert above: the upsert's state CASE has no unknown branch and its +// WHERE admits only a strictly later checked_at, which a reset does not +// carry. The executor updated_at predicate couples the reset to the save +// version, so a delayed callback cannot reset a newer configuration. A +// no-op when the executor isn't an active SSH executor or the version is +// stale: AC-EXECUTORS-SSH-REACHABILITY-001.17 keeps a disabled executor's +// record untouched, so eligibility wins over the reset trigger. +func (r *Repository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error { + now := time.Now().UTC() + _, err := r.db.ExecContext(ctx, r.db.Rebind(` + INSERT INTO executor_reachability ( + executor_id, state, reason, message, consecutive_failures, + checked_at, host, last_success_at, updated_at + ) + SELECT ?, 'unknown', '', '', 0, NULL, ?, NULL, ? + WHERE EXISTS (SELECT 1 FROM executors e + WHERE e.id = ? AND e.type = 'ssh' + AND e.deleted_at IS NULL AND e.status = 'active' + AND e.updated_at = ?) + ON CONFLICT (executor_id) DO UPDATE SET + state = 'unknown', reason = '', message = '', consecutive_failures = 0, + host = excluded.host, checked_at = NULL, last_success_at = NULL, + updated_at = excluded.updated_at + `), executorID, host, now, executorID, seenUpdatedAt.UTC()) + return err +} + +// DeleteExecutorReachability removes the stored record for one executor. +func (r *Repository) DeleteExecutorReachability(ctx context.Context, executorID string) error { + _, err := r.db.ExecContext(ctx, r.db.Rebind(` + DELETE FROM executor_reachability WHERE executor_id = ? + `), executorID) + return err +} diff --git a/apps/backend/internal/task/repository/sqlite/executor_reachability_postgres_test.go b/apps/backend/internal/task/repository/sqlite/executor_reachability_postgres_test.go new file mode 100644 index 0000000000..4bce660aa5 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/executor_reachability_postgres_test.go @@ -0,0 +1,59 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresExecutorReachabilityCheckedAtOrdering guards the SQLite-only +// risk the design calls out: TIMESTAMP takes NUMERIC affinity on SQLite and +// compares the driver's text encoding lexically, so a whole-second value can +// sort after a later sub-second one unless writes are normalized to UTC and +// bind time.Time rather than a pre-formatted string. Postgres compares +// temporally and is unaffected either way, so this test alone would pass +// even with the bug — it exists to prove the fix doesn't regress on +// Postgres, not to catch the bug (that's the SQLite test's job). +func TestPostgresExecutorReachabilityCheckedAtOrdering(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + + executor := &models.Executor{ID: "exec-reachability-pg", Name: "ssh-host", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive} + if err := repo.CreateExecutor(ctx, executor); err != nil { + t.Fatalf("create executor: %v", err) + } + + older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC) + + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert newer: %v", err) + } + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1, + Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused", + Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert older: %v", err) + } + + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("record = %+v, want the newer write to survive", got) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/executor_reachability_test.go b/apps/backend/internal/task/repository/sqlite/executor_reachability_test.go new file mode 100644 index 0000000000..f11d7ea0ac --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/executor_reachability_test.go @@ -0,0 +1,332 @@ +package sqlite + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" +) + +func newReachabilityTestRepo(t *testing.T) (*Repository, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "reachability.db") + dbConn, err := db.OpenSQLite(path) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqlxDB := sqlx.NewDb(dbConn, "sqlite3") + repo, err := NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + _ = sqlxDB.Close() + t.Fatalf("new repository: %v", err) + } + t.Cleanup(func() { _ = sqlxDB.Close() }) + return repo, path +} + +// createSSHExecutor inserts an active SSH executor and returns it, so +// UpsertExecutorReachability's eligibility EXISTS clause is satisfied. +func createSSHExecutor(t *testing.T, repo *Repository) *models.Executor { + t.Helper() + executor := &models.Executor{ + ID: "exec-" + t.Name(), + Name: "ssh-host", + Type: models.ExecutorTypeSSH, + Status: models.ExecutorStatusActive, + } + if err := repo.CreateExecutor(context.Background(), executor); err != nil { + t.Fatalf("create executor: %v", err) + } + return executor +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — last-write-wins on +// checked_at regardless of commit order, including a whole-second value +// racing a sub-second one. +func TestUpsertExecutorReachabilityNewerCheckedAtWinsRegardlessOfOrder(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + + older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC) // same whole second, later fraction + + // Issue the newer write first, the older write second — the newer must survive. + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert newer: %v", err) + } + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateUnreachable, + Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused", + Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert older: %v", err) + } + + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("record = %+v, want the newer (reachable, 10.0.0.2) write to survive", got) + } + if got.CheckedAt == nil || !got.CheckedAt.Equal(newer) { + t.Fatalf("CheckedAt = %v, want %v", got.CheckedAt, newer) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — an equal checked_at leaves +// the row unchanged. +func TestUpsertExecutorReachabilityEqualCheckedAtIsDiscarded(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + + at := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + obs := models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3, + } + if err := repo.UpsertExecutorReachability(ctx, obs); err != nil { + t.Fatalf("first upsert: %v", err) + } + + obs.Host = "10.0.0.9" // a second write at the SAME checked_at, different host + if err := repo.UpsertExecutorReachability(ctx, obs); err != nil { + t.Fatalf("second upsert: %v", err) + } + + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Host != "10.0.0.1" { + t.Fatalf("Host = %q, want the first write's host unchanged (equal checked_at must be discarded)", got.Host) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — a record survives a +// repository re-open against the same database file, byte-identical +// including both timestamps. +func TestExecutorReachabilityRoundTripAcrossReopen(t *testing.T) { + repo, path := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + + at := time.Date(2026, 3, 4, 5, 6, 7, 123_000_000, time.UTC) + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + before, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get before reopen: %v", err) + } + + dbConn, err := db.OpenSQLite(path) + if err != nil { + t.Fatalf("reopen sqlite: %v", err) + } + sqlxDB := sqlx.NewDb(dbConn, "sqlite3") + t.Cleanup(func() { _ = sqlxDB.Close() }) + reopened, err := NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + t.Fatalf("reopen repository: %v", err) + } + + after, err := reopened.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get after reopen: %v", err) + } + if after.Host != before.Host || after.State != before.State { + t.Fatalf("record changed across reopen: before=%+v after=%+v", before, after) + } + if after.CheckedAt == nil || before.CheckedAt == nil || !after.CheckedAt.Equal(*before.CheckedAt) { + t.Fatalf("CheckedAt changed across reopen: before=%v after=%v", before.CheckedAt, after.CheckedAt) + } + if after.LastSuccessAt == nil || before.LastSuccessAt == nil || !after.LastSuccessAt.Equal(*before.LastSuccessAt) { + t.Fatalf("LastSuccessAt changed across reopen: before=%v after=%v", before.LastSuccessAt, after.LastSuccessAt) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.21 +func TestDeleteExecutorRemovesReachabilityRecord(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + if err := repo.DeleteExecutor(ctx, executor.ID); err != nil { + t.Fatalf("delete executor: %v", err) + } + + if _, err := repo.GetExecutorReachability(ctx, executor.ID); err != models.ErrExecutorReachabilityNotFound { + t.Fatalf("err = %v, want ErrExecutorReachabilityNotFound", err) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.22 +func TestResetExecutorReachabilityClearsTheRecordButAdvancesUpdatedAt(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + + at := time.Now().UTC() + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1, + Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused", + Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 1, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + before, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get before reset: %v", err) + } + if before.ConsecutiveFailures == 0 || before.CheckedAt == nil { + t.Fatalf("precondition: expected a populated failure record, got %+v", before) + } + + if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil { + t.Fatalf("reset: %v", err) + } + + after, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get after reset: %v", err) + } + if after.State != models.ExecutorReachabilityStateUnknown { + t.Fatalf("State = %q, want unknown", after.State) + } + if after.ConsecutiveFailures != 0 || after.Reason != "" || after.Message != "" { + t.Fatalf("record not cleared: %+v", after) + } + if after.Host != "10.0.0.9" { + t.Fatalf("Host = %q, want the newly saved host", after.Host) + } + if after.CheckedAt != nil || after.LastSuccessAt != nil { + t.Fatalf("timestamps not cleared: checked_at=%v last_success_at=%v", after.CheckedAt, after.LastSuccessAt) + } + if !after.UpdatedAt.After(before.UpdatedAt) { + t.Fatalf("UpdatedAt did not advance: before=%v after=%v", before.UpdatedAt, after.UpdatedAt) + } +} + +func TestResetExecutorReachabilityIgnoresStaleExecutorVersion(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + staleVersion := executor.UpdatedAt + executor.Config = map[string]string{"ssh_host": "10.0.0.2"} + if err := repo.UpdateExecutor(ctx, executor); err != nil { + t.Fatalf("update executor: %v", err) + } + + if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.2", staleVersion); err != nil { + t.Fatalf("reset: %v", err) + } + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.State != models.ExecutorReachabilityStateReachable || got.Host != "10.0.0.1" { + t.Fatalf("record = %+v, want the newer executor version to remain untouched", got) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-001.17, 001.22 +func TestResetExecutorReachabilityIsANoOpForANonActiveExecutor(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + executor := createSSHExecutor(t, repo) + at := time.Now().UTC() + if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{ + ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt, + InitialState: models.ExecutorReachabilityStateReachable, + Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + executor.Status = models.ExecutorStatusDisabled + if err := repo.UpdateExecutor(ctx, executor); err != nil { + t.Fatalf("disable executor: %v", err) + } + + if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil { + t.Fatalf("reset: %v", err) + } + + got, err := repo.GetExecutorReachability(ctx, executor.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Host != "10.0.0.1" || got.State != models.ExecutorReachabilityStateReachable { + t.Fatalf("record = %+v, want it untouched by a reset on a disabled executor", got) + } +} + +func TestListSSHExecutorsForReachabilityOrdersByID(t *testing.T) { + repo, _ := newReachabilityTestRepo(t) + ctx := context.Background() + + mustCreate := func(id string, execType models.ExecutorType, status models.ExecutorStatus) { + t.Helper() + if err := repo.CreateExecutor(ctx, &models.Executor{ID: id, Name: id, Type: execType, Status: status}); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + mustCreate("b-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive) + mustCreate("a-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive) + mustCreate("c-ssh-disabled", models.ExecutorTypeSSH, models.ExecutorStatusDisabled) + mustCreate("d-docker", models.ExecutorTypeLocalDocker, models.ExecutorStatusActive) + + deleted := &models.Executor{ID: "e-ssh-deleted", Name: "e-ssh-deleted", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive} + if err := repo.CreateExecutor(ctx, deleted); err != nil { + t.Fatalf("create deleted: %v", err) + } + if err := repo.DeleteExecutor(ctx, deleted.ID); err != nil { + t.Fatalf("delete: %v", err) + } + + got, err := repo.ListSSHExecutorsForReachability(ctx) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(got) != 2 || got[0].ID != "a-ssh" || got[1].ID != "b-ssh" { + ids := make([]string, len(got)) + for i, e := range got { + ids[i] = e.ID + } + t.Fatalf("ids = %v, want [a-ssh b-ssh]", ids) + } +} diff --git a/apps/backend/internal/task/service/service.go b/apps/backend/internal/task/service/service.go index c857344299..7c0dd27528 100644 --- a/apps/backend/internal/task/service/service.go +++ b/apps/backend/internal/task/service/service.go @@ -280,6 +280,17 @@ type WorkflowStepCreator interface { CreateStepsFromTemplate(ctx context.Context, workflowID, templateID string) error } +// ExecutorSaveObserver is notified after an executor create or update +// commits, with before the pre-update snapshot (nil on create) and after the +// saved executor. CreateExecutor and UpdateExecutor are the only call sites +// that can form this before/after comparison — before must be captured prior +// to any in-place mutation of the loaded executor. Implementations decide for +// themselves whether the save is worth acting on (e.g. only SSH executors +// whose connection configuration changed). +type ExecutorSaveObserver interface { + OnExecutorSaved(ctx context.Context, before, after *models.Executor) +} + // WorkspaceBootstrapper owns the atomic persistence of a standard Kanban // workspace and its initial workflow state. type WorkspaceBootstrapper interface { @@ -501,6 +512,7 @@ type Service struct { providerProber ProviderDefaultBranchProber gitArchiveCapture GitArchiveCapture workflowStepCreator WorkflowStepCreator + executorSaveObserver ExecutorSaveObserver workspaceBootstrapper WorkspaceBootstrapper workflowStepGetter WorkflowStepGetter workflowMovePreflight WorkflowMovePreflight @@ -930,6 +942,13 @@ func (s *Service) SetWorkflowStepCreator(creator WorkflowStepCreator) { s.workflowStepCreator = creator } +// SetExecutorSaveObserver wires the observer notified after every executor +// create/update commits. Optional — a Service with no observer wired saves +// executors exactly as before. +func (s *Service) SetExecutorSaveObserver(observer ExecutorSaveObserver) { + s.executorSaveObserver = observer +} + func (s *Service) SetWorkspaceBootstrapper(bootstrapper WorkspaceBootstrapper) { s.workspaceBootstrapper = bootstrapper } diff --git a/apps/backend/internal/task/service/service_resources.go b/apps/backend/internal/task/service/service_resources.go index 0391d197a1..737acb980d 100644 --- a/apps/backend/internal/task/service/service_resources.go +++ b/apps/backend/internal/task/service/service_resources.go @@ -1806,9 +1806,19 @@ func (s *Service) CreateExecutor(ctx context.Context, req *CreateExecutorRequest return nil, err } s.publishExecutorEvent(ctx, events.ExecutorCreated, executor) + s.notifyExecutorSaved(ctx, nil, executor) return executor, nil } +// notifyExecutorSaved forwards a committed executor create/update to the +// wired ExecutorSaveObserver, if any. before is nil on create. +func (s *Service) notifyExecutorSaved(ctx context.Context, before, after *models.Executor) { + if s.executorSaveObserver == nil { + return + } + s.executorSaveObserver.OnExecutorSaved(ctx, before, after) +} + func (s *Service) GetExecutor(ctx context.Context, id string) (*models.Executor, error) { return s.executors.GetExecutor(ctx, id) } @@ -1842,12 +1852,14 @@ func (s *Service) UpdateExecutor(ctx context.Context, id string, req *UpdateExec return nil, ErrActiveTaskSessions } } + before := *executor applyExecutorUpdates(executor, req) executor.UpdatedAt = time.Now().UTC() if err := s.executors.UpdateExecutor(ctx, executor); err != nil { return nil, err } s.publishExecutorEvent(ctx, events.ExecutorUpdated, executor) + s.notifyExecutorSaved(ctx, &before, executor) return executor, nil } diff --git a/apps/backend/internal/task/service/service_resources_executors_reachability_test.go b/apps/backend/internal/task/service/service_resources_executors_reachability_test.go new file mode 100644 index 0000000000..ac11220cba --- /dev/null +++ b/apps/backend/internal/task/service/service_resources_executors_reachability_test.go @@ -0,0 +1,190 @@ +package service + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// fakeExecutorSaveObserver records every OnExecutorSaved call it receives, so +// a test can assert the exact before/after pair the service handed it without +// depending on the reachability package's own behavior. +type fakeExecutorSaveObserver struct { + calls []executorSaveCall +} + +type executorSaveCall struct { + before *models.Executor + after *models.Executor +} + +func (f *fakeExecutorSaveObserver) OnExecutorSaved(_ context.Context, before, after *models.Executor) { + f.calls = append(f.calls, executorSaveCall{before: before, after: after}) +} + +func sshExecutorCreateRequest(name string, config map[string]string) *CreateExecutorRequest { + return &CreateExecutorRequest{ + Name: name, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive, + Resumable: true, Config: config, + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// Creating an SSH executor always counts as a connection-configuration +// change — there is no prior state to compare against — so the observer must +// see before=nil paired with the newly created executor. +func TestCreateExecutorNotifiesSaveObserverWithNilBefore(t *testing.T) { + svc, _, _ := createTestService(t) + observer := &fakeExecutorSaveObserver{} + svc.SetExecutorSaveObserver(observer) + ctx := context.Background() + + executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{ + "ssh_host": "10.0.0.5", "ssh_user": "deploy", + })) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + + if len(observer.calls) != 1 { + t.Fatalf("observer calls = %d, want 1", len(observer.calls)) + } + call := observer.calls[0] + if call.before != nil { + t.Fatalf("before = %+v, want nil on create", call.before) + } + if call.after == nil || call.after.ID != executor.ID { + t.Fatalf("after = %+v, want the created executor", call.after) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// The service notifies the observer on every executor create, regardless of +// type — deciding whether a save is SSH-reachability-relevant is the +// observer implementation's job (internal/executors/reachability.SaveObserver), +// not the service's. This keeps the service layer decoupled from SSH-specific +// eligibility rules, the same way publishExecutorEvent fires unconditionally +// and lets subscribers filter. +func TestCreateExecutorNotifiesSaveObserverForNonSSHType(t *testing.T) { + svc, _, _ := createTestService(t) + observer := &fakeExecutorSaveObserver{} + svc.SetExecutorSaveObserver(observer) + ctx := context.Background() + + executor, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{ + Name: "Local runner", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive, Resumable: true, + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + + if len(observer.calls) != 1 { + t.Fatalf("observer calls = %d, want 1 (gating is the observer's job, not the service's)", len(observer.calls)) + } + if observer.calls[0].after == nil || observer.calls[0].after.ID != executor.ID { + t.Fatalf("after = %+v, want the created executor", observer.calls[0].after) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1, AC-EXECUTORS-SSH-REACHABILITY-003.2 +// +// Updating an SSH executor's host must notify the observer with the literal +// old-host/new-host pair — captured strictly before applyExecutorUpdates +// mutates the executor in place, per the work order's own risk callout. A +// self-comparison (before and after pointing at the same, already-mutated +// struct) would silently make this test pass with either host string, so it +// asserts both values explicitly. +func TestUpdateExecutorNotifiesSaveObserverOnHostChange(t *testing.T) { + svc, _, _ := createTestService(t) + executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH) + ctx := context.Background() + if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"}, + }); err != nil { + t.Fatalf("seed UpdateExecutor: %v", err) + } + + observer := &fakeExecutorSaveObserver{} + svc.SetExecutorSaveObserver(observer) + + if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Config: map[string]string{"ssh_host": "10.0.0.2", "ssh_user": "deploy"}, + }); err != nil { + t.Fatalf("UpdateExecutor: %v", err) + } + + if len(observer.calls) != 1 { + t.Fatalf("observer calls = %d, want 1", len(observer.calls)) + } + call := observer.calls[0] + if call.before == nil || call.before.Config["ssh_host"] != "10.0.0.1" { + t.Fatalf("before host = %+v, want 10.0.0.1", call.before) + } + if call.after == nil || call.after.Config["ssh_host"] != "10.0.0.2" { + t.Fatalf("after host = %+v, want 10.0.0.2", call.after) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// A name-only update still notifies the observer (the service does not +// pre-filter by which fields changed) — the observer's own +// connectionConfigChanged comparison is what decides a name-only save isn't +// reachability-relevant, and that decision is covered in the reachability +// package's own save_observer_test.go, not here. This test only pins that the +// service keeps calling with the correct before/after regardless of which +// fields the caller supplied. +func TestUpdateExecutorNotifiesSaveObserverForNameOnlyChange(t *testing.T) { + svc, _, _ := createTestService(t) + executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH) + ctx := context.Background() + if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"}, + }); err != nil { + t.Fatalf("seed UpdateExecutor: %v", err) + } + + observer := &fakeExecutorSaveObserver{} + svc.SetExecutorSaveObserver(observer) + + newName := "Renamed SSH box" + if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{Name: &newName}); err != nil { + t.Fatalf("UpdateExecutor: %v", err) + } + + if len(observer.calls) != 1 { + t.Fatalf("observer calls = %d, want 1", len(observer.calls)) + } + if observer.calls[0].before.Config["ssh_host"] != "10.0.0.1" || observer.calls[0].after.Config["ssh_host"] != "10.0.0.1" { + t.Fatalf("host should be unchanged across a name-only update: %+v", observer.calls[0]) + } + if observer.calls[0].after.Name != newName { + t.Fatalf("after.Name = %q, want %q", observer.calls[0].after.Name, newName) + } +} + +// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1 +// +// A nil observer (the default in every test not exercising reachability) must +// never be dereferenced by CreateExecutor/UpdateExecutor. +func TestExecutorSaveObserverNilSafe(t *testing.T) { + svc, _, _ := createTestService(t) + ctx := context.Background() + + executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{ + "ssh_host": "10.0.0.5", + })) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + + newHost := "10.0.0.9" + if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Config: map[string]string{"ssh_host": newHost}, + }); err != nil { + t.Fatalf("UpdateExecutor: %v", err) + } +} diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index 54da120c6f..8908c79d84 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -271,6 +271,7 @@ const ( ActionSessionModelsUpdated = "session.models_updated" ActionSessionModelFallback = "session.model_fallback" ActionSessionModelSelectionWarning = "session.model_selection_warning" + ActionSessionLaunchWarning = "session.launch.warning" ActionSessionMCPStatusUpdated = "session.mcp_status_updated" ActionSessionInfoUpdated = "session.info_updated" ActionSessionSetMode = "session.set_mode" @@ -298,6 +299,7 @@ const ( ActionExecutorCreated = "executor.created" ActionExecutorUpdated = "executor.updated" ActionExecutorDeleted = "executor.deleted" + ActionExecutorReachabilityChanged = "executor.reachability.changed" ActionEnvironmentCreated = "environment.created" ActionEnvironmentUpdated = "environment.updated" ActionEnvironmentDeleted = "environment.deleted" diff --git a/apps/web/app/settings/executors/ssh/[executorId]/page.tsx b/apps/web/app/settings/executors/ssh/[executorId]/page.tsx index 5471522f64..159af4413a 100644 --- a/apps/web/app/settings/executors/ssh/[executorId]/page.tsx +++ b/apps/web/app/settings/executors/ssh/[executorId]/page.tsx @@ -13,6 +13,7 @@ import { fetchExecutor, listExecutors, updateExecutor } from "@/lib/api/domains/ import { SSHConnectionCard } from "@/components/settings/ssh-connection-card"; import type { SSHExecutorConfig } from "@/components/settings/ssh-connection-card"; import { SSHSessionsCard } from "@/components/settings/ssh-sessions-card"; +import { SSHReachabilityCard } from "@/components/settings/ssh-reachability-card"; import { listSSHSessions } from "@/lib/api/domains/ssh-api"; import { getExecutorLabel } from "@/lib/executor-icons"; import { @@ -135,6 +136,7 @@ function SSHExecutorView({ coordinatedSaveId={`ssh-executor:${executor.id}`} runningSessionCount={sessionCount} /> + ); diff --git a/apps/web/components/settings/ssh-reachability-card.test.tsx b/apps/web/components/settings/ssh-reachability-card.test.tsx new file mode 100644 index 0000000000..f4139d9ad5 --- /dev/null +++ b/apps/web/components/settings/ssh-reachability-card.test.tsx @@ -0,0 +1,213 @@ +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { StateProvider, useAppStoreApi } from "@/components/state-provider"; +import type { SSHReachabilityRecord } from "@/lib/types/http-ssh"; + +const getSSHExecutorReachability = vi.fn(); +const probeSSHExecutorReachability = vi.fn(); + +vi.mock("@/lib/api/domains/ssh-api", () => ({ + getSSHExecutorReachability: (...args: unknown[]) => getSSHExecutorReachability(...args), + probeSSHExecutorReachability: (...args: unknown[]) => probeSSHExecutorReachability(...args), +})); + +import { SSHReachabilityCard, isReachabilityStale } from "./ssh-reachability-card"; + +const EXECUTOR_ID = "executor-1"; +const BASE_TIMESTAMP = "2026-09-17T00:00:00.000Z"; +const STATE_TESTID = "ssh-reachability-state"; + +function record(overrides: Partial = {}): SSHReachabilityRecord { + return { + executor_id: EXECUTOR_ID, + state: "reachable", + reason: "", + consecutive_failures: 0, + host: "10.0.0.5", + checked_at: BASE_TIMESTAMP, + last_success_at: BASE_TIMESTAMP, + updated_at: BASE_TIMESTAMP, + probing_enabled: true, + probe_interval_seconds: 60, + persisted: true, + ...overrides, + }; +} + +function renderCard(executorId = EXECUTOR_ID) { + return render( + + + , + ); +} + +function PushReachability({ record: pushed }: { record: SSHReachabilityRecord }) { + const storeApi = useAppStoreApi(); + return ( + + ); +} + +function renderCardWithPush(pushed: SSHReachabilityRecord) { + return render( + + + + , + ); +} + +async function waitForState(text: string) { + await waitFor(() => expect(screen.getByTestId(STATE_TESTID).textContent).toBe(text)); +} + +beforeEach(() => { + getSSHExecutorReachability.mockReset(); + probeSSHExecutorReachability.mockReset(); +}); + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +describe("isReachabilityStale", () => { + it("is false when there is no probing cadence or no completed probe", () => { + expect(isReachabilityStale(null, 60, true, Date.now())).toBe(false); + expect(isReachabilityStale(BASE_TIMESTAMP, 60, false, Date.now())).toBe(false); + }); + + it("is false within three intervals and true just past it", () => { + const checkedMs = Date.parse(BASE_TIMESTAMP); + expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000)).toBe(false); + expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000 + 1)).toBe(true); + }); +}); + +describe("SSHReachabilityCard rendering", () => { + it("renders state, host, failure count and age for a reachable record", async () => { + getSSHExecutorReachability.mockResolvedValue(record()); + renderCard(); + await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy()); + expect(screen.getByTestId("ssh-reachability-host").textContent).toContain("10.0.0.5"); + expect(screen.queryByTestId("ssh-reachability-reason")).toBeNull(); + }); + + it("renders reason and message for an unreachable record", async () => { + getSSHExecutorReachability.mockResolvedValue( + record({ + state: "unreachable", + reason: "timeout", + message: "dial tcp: i/o timeout", + consecutive_failures: 3, + }), + ); + renderCard(); + await waitFor(() => expect(screen.getByTestId("ssh-reachability-reason")).toBeTruthy()); + expect(screen.getByTestId("ssh-reachability-message").textContent).toBe( + "dial tcp: i/o timeout", + ); + expect(screen.getByTestId("ssh-reachability-failures").textContent).toContain("3"); + }); + + it("marks a record older than three intervals as stale", async () => { + const old = new Date(Date.now() - 4 * 60_000).toISOString(); + getSSHExecutorReachability.mockResolvedValue( + record({ probe_interval_seconds: 60, checked_at: old }), + ); + renderCard(); + await waitFor(() => expect(screen.getByTestId("ssh-reachability-stale")).toBeTruthy()); + }); + + it("does not mark a fresh record as stale", async () => { + const fresh = new Date().toISOString(); + getSSHExecutorReachability.mockResolvedValue( + record({ + probe_interval_seconds: 60, + checked_at: fresh, + last_success_at: fresh, + updated_at: fresh, + }), + ); + renderCard(); + await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy()); + expect(screen.queryByTestId("ssh-reachability-stale")).toBeNull(); + }); +}); + +describe("SSHReachabilityCard live updates", () => { + it("applies a pushed executor.reachability.changed record live, without a reload", async () => { + getSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" })); + renderCardWithPush(record({ state: "unreachable", reason: "auth", message: "boom" })); + await waitForState("Reachable"); + fireEvent.click(screen.getByRole("button", { name: "push" })); + await waitForState("Unreachable"); + expect(getSSHExecutorReachability).toHaveBeenCalledTimes(1); + }); + + it("replaces a stored record with a reset (null checked_at, newer updated_at)", async () => { + getSSHExecutorReachability.mockResolvedValue( + record({ state: "unreachable", reason: "timeout", updated_at: BASE_TIMESTAMP }), + ); + renderCardWithPush( + record({ + state: "unknown", + reason: "", + checked_at: null, + last_success_at: null, + updated_at: "2026-09-17T00:01:00.000Z", + }), + ); + await waitForState("Unreachable"); + fireEvent.click(screen.getByRole("button", { name: "push" })); + await waitForState("Unknown"); + }); +}); + +describe("SSHReachabilityCard refresh cadence", () => { + it("says periodic probing is off and starts no refresh timer when probing_enabled is false", async () => { + const setIntervalSpy = vi.spyOn(window, "setInterval"); + getSSHExecutorReachability.mockResolvedValue( + record({ probing_enabled: false, probe_interval_seconds: 0 }), + ); + renderCard(); + await waitFor(() => expect(screen.getByTestId("ssh-reachability-probing-off")).toBeTruthy()); + // Exclude testing-library's own internal waitFor polling interval (50ms); + // this asserts the card itself never armed a refresh timer. + const ownIntervalCalls = setIntervalSpy.mock.calls.filter((call) => call[1] !== 50); + expect(ownIntervalCalls).toHaveLength(0); + expect(screen.queryByTestId(STATE_TESTID)?.textContent).not.toBe("Unreachable"); + }); + + it("starts a refresh timer at the configured probe interval when probing is enabled", async () => { + const setIntervalSpy = vi.spyOn(window, "setInterval"); + getSSHExecutorReachability.mockResolvedValue(record({ probe_interval_seconds: 45 })); + renderCard(); + await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy()); + expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 45_000); + }); +}); + +describe("SSHReachabilityCard errors and probe-now", () => { + it("reports reachability as not known, never unreachable, when the initial load fails", async () => { + getSSHExecutorReachability.mockRejectedValue(new Error("network error")); + renderCard(); + await waitFor(() => expect(screen.getByTestId("ssh-reachability-not-known")).toBeTruthy()); + expect(screen.queryByTestId(STATE_TESTID)).toBeNull(); + }); + + it("runs an immediate probe and applies the result", async () => { + getSSHExecutorReachability.mockResolvedValue( + record({ state: "unreachable", reason: "network" }), + ); + probeSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" })); + renderCard(); + await waitForState("Unreachable"); + fireEvent.click(screen.getByTestId("ssh-reachability-probe-now")); + await waitForState("Reachable"); + expect(probeSSHExecutorReachability).toHaveBeenCalledWith(EXECUTOR_ID); + }); +}); diff --git a/apps/web/components/settings/ssh-reachability-card.tsx b/apps/web/components/settings/ssh-reachability-card.tsx new file mode 100644 index 0000000000..802d655f88 --- /dev/null +++ b/apps/web/components/settings/ssh-reachability-card.tsx @@ -0,0 +1,198 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { Badge } from "@kandev/ui/badge"; +import { Button } from "@kandev/ui/button"; +import { CardContent } from "@kandev/ui/card"; +import { IconLoader2 } from "@tabler/icons-react"; +import type { SSHReachabilityRecord, SSHReachabilityReason } from "@/lib/types/http-ssh"; +import { formatRelative } from "@/lib/i18n/formats"; +import { useSSHReachability } from "@/hooks/domains/settings/use-ssh-reachability"; +import { SettingsCard } from "@/components/settings/settings-card"; +import { SettingsCardHeader } from "@/components/settings/settings-card-header"; +import { settingsActionClassName } from "@/components/settings/settings-control"; + +export interface SSHReachabilityCardProps { + executorId: string; +} + +const STALE_INTERVAL_MULTIPLIER = 3; + +/** + * A record older than three probe intervals is stale rather than current. + * Disabled probing has no cadence to measure against, and a never-probed + * record has nothing to be stale relative to, so both read as not-stale. + */ +export function isReachabilityStale( + checkedAt: string | null, + probeIntervalSeconds: number, + probingEnabled: boolean, + now: number = Date.now(), +): boolean { + if (!probingEnabled || !checkedAt || probeIntervalSeconds <= 0) return false; + const checkedMs = Date.parse(checkedAt); + if (Number.isNaN(checkedMs)) return false; + return now - checkedMs > probeIntervalSeconds * 1000 * STALE_INTERVAL_MULTIPLIER; +} + +export function reachabilityReasonKey(reason: SSHReachabilityReason): string | null { + switch (reason) { + case "config": + return "executors:sshReachabilityReasonConfig"; + case "timeout": + return "executors:sshReachabilityReasonTimeout"; + case "host_key": + return "executors:sshReachabilityReasonHostKey"; + case "auth": + return "executors:sshReachabilityReasonAuth"; + case "network": + return "executors:sshReachabilityReasonNetwork"; + case "unknown": + return "executors:sshReachabilityReasonUnknown"; + default: + return null; + } +} + +export function reachabilityStateKey(state: SSHReachabilityRecord["state"]): string { + switch (state) { + case "reachable": + return "executors:sshReachabilityStateReachable"; + case "unreachable": + return "executors:sshReachabilityStateUnreachable"; + default: + return "executors:sshReachabilityStateUnknown"; + } +} + +/** + * Renders the most recent SSH reachability record for one executor: state, + * probed host, failure reason/message, consecutive-failure count, and the + * age of the last completed and last successful probe. Never fetches + * directly into local-only state — every fetch result is written through the + * store so a live executor.reachability.changed push and this card's own + * refresh share one reconciliation path. + */ +export function SSHReachabilityCard({ executorId }: SSHReachabilityCardProps) { + const { t } = useTranslation(); + const { record, loadError, probing, probeNow, now } = useSSHReachability(executorId); + + return ( + + void probeNow()} + disabled={probing} + data-testid="ssh-reachability-probe-now" + className={settingsActionClassName("cursor-pointer")} + > + {probing ? : null} + {t("executors:sshReachabilityProbeNow")} + + } + /> + + + + + ); +} + +function ReachabilityBody({ + record, + loadError, + now, +}: { + record: SSHReachabilityRecord | undefined; + loadError: boolean; + now: number; +}) { + const { t } = useTranslation(); + if (!record) { + if (loadError) { + return ( +

+ {t("executors:sshReachabilityLoadFailed")} +

+ ); + } + return null; + } + const stale = isReachabilityStale( + record.checked_at, + record.probe_interval_seconds, + record.probing_enabled, + now, + ); + const reasonKey = record.reason ? reachabilityReasonKey(record.reason) : null; + return ( +
+
+ + {stale ? ( + + {t("executors:sshReachabilityStale")} + + ) : null} +
+ {record.host ? ( +

+ {t("executors:sshReachabilityHost", { host: record.host })} +

+ ) : null} + {record.state === "unreachable" && reasonKey ? ( +

{t(reasonKey)}

+ ) : null} + {record.state === "unreachable" && record.message ? ( +

+ {record.message} +

+ ) : null} +

+ {t("executors:sshReachabilityConsecutiveFailures", { count: record.consecutive_failures })} +

+

+ {record.checked_at + ? t("executors:sshReachabilityAgeLastProbe", { age: formatRelative(record.checked_at) }) + : t("executors:sshReachabilityNeverProbed")} +

+

+ {record.last_success_at + ? t("executors:sshReachabilityAgeLastSuccess", { + age: formatRelative(record.last_success_at), + }) + : t("executors:sshReachabilityNoSuccess")} +

+ {!record.probing_enabled ? ( +

+ {t("executors:sshReachabilityProbingOff")} +

+ ) : null} +
+ ); +} + +const STATE_BADGE_CLASSNAMES: Record = { + reachable: "border-green-500/30 bg-green-500/10 text-green-700", + unreachable: "border-red-500/30 bg-red-500/10 text-red-700", + unknown: "border-muted-foreground/30 bg-muted text-muted-foreground", +}; + +function StateBadge({ state }: { state: SSHReachabilityRecord["state"] }) { + const { t } = useTranslation(); + const variantClass = STATE_BADGE_CLASSNAMES[state]; + return ( + + {t(reachabilityStateKey(state))} + + ); +} diff --git a/apps/web/components/settings/ssh-settings.tsx b/apps/web/components/settings/ssh-settings.tsx index 534683dbb3..59735babed 100644 --- a/apps/web/components/settings/ssh-settings.tsx +++ b/apps/web/components/settings/ssh-settings.tsx @@ -7,3 +7,5 @@ export { SSHConnectionCard } from "./ssh-connection-card"; export type { SSHConnectionCardProps, SSHExecutorConfig } from "./ssh-connection-card"; export { SSHSessionsCard } from "./ssh-sessions-card"; export type { SSHSessionsCardProps } from "./ssh-sessions-card"; +export { SSHReachabilityCard } from "./ssh-reachability-card"; +export type { SSHReachabilityCardProps } from "./ssh-reachability-card"; diff --git a/apps/web/components/task/launch-warning.test.tsx b/apps/web/components/task/launch-warning.test.tsx new file mode 100644 index 0000000000..d568fce1d0 --- /dev/null +++ b/apps/web/components/task/launch-warning.test.tsx @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { StateProvider } from "@/components/state-provider"; +import type { LaunchWarningEntry } from "@/lib/state/slices/session-runtime/types"; +import { LaunchWarning } from "./launch-warning"; + +const SESSION_ID = "session-1"; + +function makeEntry(overrides: Partial = {}): LaunchWarningEntry { + return { + executorId: "executor-1", + host: "10.0.0.5", + state: "unreachable", + reason: "timeout", + ...overrides, + }; +} + +afterEach(() => { + cleanup(); + vi.useRealTimers(); +}); + +function renderWithEntry(entry?: LaunchWarningEntry) { + return render( + + + , + ); +} + +describe("LaunchWarning", () => { + it("renders nothing when no launch warning was published for this session", () => { + const { container } = renderWithEntry(undefined); + expect(container.textContent).toBe(""); + }); + + it("names the probed host from the event payload", () => { + renderWithEntry(makeEntry()); + expect(screen.getByTestId("launch-warning").textContent).toContain("10.0.0.5"); + }); + + it("shows the age of the last successful probe", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-06-11T00:05:00.000Z")); + try { + renderWithEntry(makeEntry({ lastSuccessAt: "2026-06-11T00:00:00.000Z" })); + expect(screen.getByTestId("launch-warning-last-success").textContent).toMatch(/5m/); + } finally { + vi.useRealTimers(); + } + }); + + it("states plainly that no successful probe was ever recorded, not a zero age", () => { + renderWithEntry(makeEntry({ lastSuccessAt: undefined })); + const text = screen.getByTestId("launch-warning-last-success").textContent ?? ""; + expect(text).not.toMatch(/0[ms]/); + expect(text.length).toBeGreaterThan(0); + }); + + it.each(["config", "timeout", "host_key", "auth", "network", "unknown"])( + "renders a reason sentence for %s without throwing", + (reason) => { + renderWithEntry(makeEntry({ reason })); + expect(screen.getByTestId("launch-warning-reason").textContent?.length).toBeGreaterThan(0); + }, + ); + + it("never renders any confirmation control — the launch is never blocked", () => { + renderWithEntry(makeEntry()); + expect(screen.queryByRole("button")).toBeNull(); + }); +}); diff --git a/apps/web/components/task/launch-warning.tsx b/apps/web/components/task/launch-warning.tsx new file mode 100644 index 0000000000..05c1bb78d2 --- /dev/null +++ b/apps/web/components/task/launch-warning.tsx @@ -0,0 +1,67 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { IconAlertTriangle } from "@tabler/icons-react"; +import { useAppStore } from "@/components/state-provider"; +import { formatRelative } from "@/lib/i18n/formats"; + +type LaunchWarningProps = { + sessionId: string; +}; + +function reasonKey(reason: string): string { + switch (reason) { + case "config": + return "task:launchWarningReasonConfig"; + case "timeout": + return "task:launchWarningReasonTimeout"; + case "host_key": + return "task:launchWarningReasonHostKey"; + case "auth": + return "task:launchWarningReasonAuth"; + case "network": + return "task:launchWarningReasonNetwork"; + default: + return "task:launchWarningReasonUnknown"; + } +} + +/** + * Renders the session.launch.warning event verbatim at the point the user + * initiated the launch — never re-derives reachability from the settings + * store slice, and never blocks or confirms the launch it describes. + */ +export function LaunchWarning({ sessionId }: LaunchWarningProps) { + const { t } = useTranslation(); + const entry = useAppStore((state) => state.launchWarning.bySessionId[sessionId]); + + if (!entry) return null; + + return ( +
+
+
+
+

{t("task:launchWarningTitle")}

+ {entry.host ? ( +

+ {t("task:launchWarningHost", { host: entry.host })} +

+ ) : null} +

+ {entry.lastSuccessAt + ? t("task:launchWarningLastSuccess", { age: formatRelative(entry.lastSuccessAt) }) + : t("task:launchWarningNeverSucceeded")} +

+

+ {t(reasonKey(entry.reason))} +

+
+
+ ); +} diff --git a/apps/web/components/task/task-chat-panel.launch-error.test.tsx b/apps/web/components/task/task-chat-panel.launch-error.test.tsx index ae4e46fb72..4e812dced3 100644 --- a/apps/web/components/task/task-chat-panel.launch-error.test.tsx +++ b/apps/web/components/task/task-chat-panel.launch-error.test.tsx @@ -44,6 +44,7 @@ const appStoreState = { }, agentProfiles: { items: [] }, messages: { bySession: { [SESSION_ID]: [priorTranscriptMessage] } }, + launchWarning: { bySessionId: {} }, }; const panelState = { diff --git a/apps/web/components/task/task-chat-panel.tsx b/apps/web/components/task/task-chat-panel.tsx index ae9eb7c268..694fedabd5 100644 --- a/apps/web/components/task/task-chat-panel.tsx +++ b/apps/web/components/task/task-chat-panel.tsx @@ -54,6 +54,7 @@ import { routePanelMouseDown } from "./chat/route-panel-mouse-down"; import { useTranslation } from "react-i18next"; import { loadMessageWindowAround } from "@/hooks/domains/session/load-message-window"; +import { LaunchWarning } from "./launch-warning"; import { useTaskLaunchErrorContext } from "./task-launch-error-context"; import { useTaskStatusSummary } from "@/hooks/domains/task/use-task-status-summary"; import { TaskMarkdownFileLinkProvider } from "@/components/shared/task-markdown-file-link-provider"; @@ -1253,54 +1254,67 @@ export const TaskChatPanel = memo(function TaskChatPanel({ {!hideLaunchQueueStatus && } {!hideWipQueueStatus && } - - - - ) : undefined - } - /> - + +
+ {resolvedSessionId ? ( +
+ +
+ ) : null} +
+ + + ) : undefined + } + /> + +
+
{isJumpLoading && (
{ + await this.reachabilityProbeNowButton.click(); + await expect(this.reachabilityProbeNowButton).toBeEnabled({ timeout: 30_000 }); + } } diff --git a/apps/web/e2e/tests/ssh/reachability.spec.ts b/apps/web/e2e/tests/ssh/reachability.spec.ts new file mode 100644 index 0000000000..423fef25b7 --- /dev/null +++ b/apps/web/e2e/tests/ssh/reachability.spec.ts @@ -0,0 +1,100 @@ +import { test, expect } from "../../fixtures/ssh-test-base"; +import { dropTrafficToPort22, restoreTraffic } from "../../helpers/ssh"; +import { SSHSettingsPage } from "../../pages/SSHSettingsPage"; + +/** + * SSH reachability: settings-panel state transitions and non-gating launch + * behavior against a real sshd target. + * + * Drives the failure threshold through repeated "Probe now" clicks rather + * than the scheduled poller's wall-clock interval — `httpProbeReachability` + * runs the exact same hysteresis path (`store.Observe`) an automatic pass + * would, so this proves the same threshold logic without making the spec's + * runtime depend on a configured interval. + * + * Covers AC-EXECUTORS-SSH-REACHABILITY-001.9, 001.11, 002.1, 003.1, 003.3. + */ +test.describe("ssh executor — reachability", () => { + test("reflects reachable, then unreachable after the failure threshold, then clears on a single success", async ({ + testPage, + seedData, + }) => { + test.setTimeout(150_000); + const page = new SSHSettingsPage(testPage); + await page.gotoExisting(seedData.sshExecutorId); + + // A fresh probe against the live target starts reachable. + await page.probeNow(); + await expect(page.reachabilityState).toHaveText(/^reachable$/i); + await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host); + + dropTrafficToPort22(seedData.sshTarget); + try { + // First failed probe: below the two-probe failure threshold, so the + // panel must not have flipped yet. + await page.probeNow(); + await expect(page.reachabilityState).toHaveText(/^reachable$/i); + + // Second consecutive failure crosses the threshold. + await page.probeNow(); + await expect(page.reachabilityState).toHaveText(/^unreachable$/i); + await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host); + await expect(page.reachabilityReason).toBeVisible(); + } finally { + restoreTraffic(seedData.sshTarget); + } + + // A single successful probe clears an unreachable state immediately; + // no second confirming probe is required. + await page.probeNow(); + await expect(page.reachabilityState).toHaveText(/^reachable$/i); + }); + + test("a launch started while the host is unreachable is attempted and fails naming the host, not refused", async ({ + apiClient, + seedData, + }) => { + test.setTimeout(240_000); + dropTrafficToPort22(seedData.sshTarget); + try { + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Reachability: launch while unreachable", + seedData.agentProfileId, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + executor_profile_id: seedData.sshExecutorProfileId, + }, + ); + + // The task and its session are created rather than refused — the + // launch is attempted and only fails once the real connection attempt + // gives up. `dropTrafficToPort22` drops inbound packets on the + // container's own port 22, but Docker's own port-publishing plumbing + // (docker-proxy on some hosts) can still complete the client's TCP + // handshake before its own upstream dial to the container stalls, so + // the failure surfaces only once that stalls out — well past the + // client-side `sshDialTimeout` (30s) alone. Give this real, variable + // network path a wide budget rather than one tuned to a fast local + // failure (e.g. a missing remote binary). + let errMsg = ""; + await expect + .poll( + async () => { + const { sessions } = await apiClient.listTaskSessions(task.id); + const session = sessions[0]; + errMsg = session?.error_message ?? ""; + return session?.state ?? ""; + }, + { timeout: 200_000, message: "session reaches FAILED against the unreachable host" }, + ) + .toBe("FAILED"); + expect(errMsg).toContain(seedData.sshTarget.host); + } finally { + restoreTraffic(seedData.sshTarget); + } + }); +}); diff --git a/apps/web/hooks/domains/settings/use-ssh-reachability.ts b/apps/web/hooks/domains/settings/use-ssh-reachability.ts new file mode 100644 index 0000000000..96befd84b3 --- /dev/null +++ b/apps/web/hooks/domains/settings/use-ssh-reachability.ts @@ -0,0 +1,82 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { useAppStore, useAppStoreApi } from "@/components/state-provider"; +import { + getSSHExecutorReachability, + probeSSHExecutorReachability, +} from "@/lib/api/domains/ssh-api"; + +/** + * Owns the settings-domain fetch, refresh, and immediate-probe lifecycle for + * one SSH executor. The store remains the source of truth for the record, so + * WebSocket updates and HTTP responses use the same reconciliation path. + */ +export function useSSHReachability(executorId: string) { + const record = useAppStore((state) => state.sshReachability.byExecutorId[executorId]); + const storeApi = useAppStoreApi(); + const [loadError, setLoadError] = useState(false); + const [probing, setProbing] = useState(false); + const [now, setNow] = useState(() => Date.now()); + const seqRef = useRef(0); + + const load = useCallback(async () => { + const seq = ++seqRef.current; + try { + const response = await getSSHExecutorReachability(executorId); + if (seq !== seqRef.current) return; + setLoadError(false); + storeApi.getState().setSSHReachability(response); + } catch { + if (seq !== seqRef.current) return; + setLoadError(true); + } + }, [executorId, storeApi]); + + useEffect(() => { + seqRef.current = 0; + setLoadError(false); + setNow(Date.now()); + void load(); + return () => { + seqRef.current = -1; + }; + // The executor identity is the lifecycle boundary. `load` is stable for + // that identity and must not restart the request on every store update. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [executorId]); + + useEffect(() => { + if (!record || !record.probing_enabled || record.probe_interval_seconds <= 0) return; + const interval = window.setInterval(() => void load(), record.probe_interval_seconds * 1000); + return () => window.clearInterval(interval); + }, [record, load]); + + // A failed refresh can leave the record object unchanged. Keep the stale + // badge live in that case instead of waiting for a successful fetch to + // trigger another render. + useEffect(() => { + if (!record?.probing_enabled || !record.checked_at || record.probe_interval_seconds <= 0) { + return; + } + const checkedAt = Date.parse(record.checked_at); + if (Number.isNaN(checkedAt)) return; + const staleAt = checkedAt + record.probe_interval_seconds * 1000 * 3; + const delay = Math.max(0, staleAt - Date.now() + 1); + const timer = window.setTimeout(() => setNow(Date.now()), delay); + return () => window.clearTimeout(timer); + }, [record]); + + const probeNow = useCallback(async () => { + setProbing(true); + try { + const response = await probeSSHExecutorReachability(executorId); + setLoadError(false); + storeApi.getState().setSSHReachability(response); + } catch { + setLoadError(true); + } finally { + setProbing(false); + } + }, [executorId, storeApi]); + + return { record, loadError, probing, probeNow, now }; +} diff --git a/apps/web/lib/api/domains/ssh-api.test.ts b/apps/web/lib/api/domains/ssh-api.test.ts new file mode 100644 index 0000000000..0d2c0b61bb --- /dev/null +++ b/apps/web/lib/api/domains/ssh-api.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("@/lib/config", () => ({ + getBackendConfig: () => ({ apiBaseUrl: "http://api.test" }), +})); + +import { + getSSHReachability, + getSSHExecutorReachability, + probeSSHExecutorReachability, +} from "./ssh-api"; +import type { SSHReachabilityRecord } from "@/lib/types/http-ssh"; + +const BASE = "http://api.test/api/v1/ssh"; + +type FetchInput = Parameters[0]; +type FetchInit = Parameters[1]; + +const fetchSpy = vi.fn<(...args: [FetchInput, FetchInit?]) => Promise>(); + +beforeEach(() => { + fetchSpy.mockReset(); + vi.stubGlobal("fetch", fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function lastCall(): { url: string; init: FetchInit | undefined } { + const call = fetchSpy.mock.calls.at(-1); + if (!call) throw new Error("expected fetch to have been called"); + return { url: String(call[0]), init: call[1] }; +} + +function record(overrides: Partial = {}): SSHReachabilityRecord { + return { + executor_id: "exec-1", + state: "reachable", + reason: "", + consecutive_failures: 0, + host: "build.example", + checked_at: "2026-09-17T00:00:00Z", + last_success_at: "2026-09-17T00:00:00Z", + updated_at: "2026-09-17T00:00:00Z", + probing_enabled: true, + probe_interval_seconds: 60, + persisted: true, + ...overrides, + }; +} + +describe("getSSHReachability", () => { + it("GETs the list route", async () => { + fetchSpy.mockResolvedValueOnce(jsonResponse([record()])); + const result = await getSSHReachability(); + expect(result).toEqual([record()]); + const { url, init } = lastCall(); + expect(url).toBe(`${BASE}/reachability`); + expect(init?.method ?? "GET").toBe("GET"); + }); +}); + +describe("getSSHExecutorReachability", () => { + it("GETs the single-executor route", async () => { + fetchSpy.mockResolvedValueOnce(jsonResponse(record())); + const result = await getSSHExecutorReachability("exec-1"); + expect(result).toEqual(record()); + const { url } = lastCall(); + expect(url).toBe(`${BASE}/executors/exec-1/reachability`); + }); + + it("encodes the executor id", async () => { + fetchSpy.mockResolvedValueOnce(jsonResponse(record({ executor_id: "exec/1" }))); + await getSSHExecutorReachability("exec/1"); + const { url } = lastCall(); + expect(url).toBe(`${BASE}/executors/exec%2F1/reachability`); + }); +}); + +describe("probeSSHExecutorReachability", () => { + it("POSTs the probe-now route", async () => { + fetchSpy.mockResolvedValueOnce(jsonResponse(record())); + const result = await probeSSHExecutorReachability("exec-1"); + expect(result).toEqual(record()); + const { url, init } = lastCall(); + expect(url).toBe(`${BASE}/executors/exec-1/reachability/probe`); + expect(init?.method).toBe("POST"); + }); +}); diff --git a/apps/web/lib/api/domains/ssh-api.ts b/apps/web/lib/api/domains/ssh-api.ts index 6e070aeb44..98936f20d6 100644 --- a/apps/web/lib/api/domains/ssh-api.ts +++ b/apps/web/lib/api/domains/ssh-api.ts @@ -5,6 +5,7 @@ import type { SSHSession, SSHAgentReadinessResponse, SSHProbeShellsResponse, + SSHReachabilityRecord, } from "@/lib/types/http-ssh"; export async function testSSHConnection( @@ -72,3 +73,36 @@ export async function probeSSHShells( }, ); } + +export async function getSSHReachability( + options?: ApiRequestOptions, +): Promise { + return fetchJson("/api/v1/ssh/reachability", options); +} + +export async function getSSHExecutorReachability( + executorId: string, + options?: ApiRequestOptions, +): Promise { + return fetchJson( + `/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability`, + options, + ); +} + +export async function probeSSHExecutorReachability( + executorId: string, + options?: ApiRequestOptions, +): Promise { + return fetchJson( + `/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability/probe`, + { + ...options, + init: { + ...(options?.init ?? {}), + method: "POST", + headers: { ...(options?.init?.headers ?? {}), "Content-Type": "application/json" }, + }, + }, + ); +} diff --git a/apps/web/lib/state/app-state-types.ts b/apps/web/lib/state/app-state-types.ts index 302ed0d9f5..567bfc54d8 100644 --- a/apps/web/lib/state/app-state-types.ts +++ b/apps/web/lib/state/app-state-types.ts @@ -19,6 +19,7 @@ import type { import type { SystemHealthResponse } from "@/lib/types/health"; import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime"; import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use"; +import type { SSHReachabilityRecord } from "@/lib/types/http-ssh"; import type { UISliceActions as UIA } from "./slices/ui/types"; import type * as UISliceTypes from "./slices/ui/types"; import type { @@ -121,6 +122,7 @@ export type AppState = KanbanSlice & { sleepInhibition: (typeof defaultSettingsState)["sleepInhibition"]; userSettings: (typeof defaultSettingsState)["userSettings"]; agentProfileRecentUse: (typeof defaultSettingsState)["agentProfileRecentUse"]; + sshReachability: (typeof defaultSettingsState)["sshReachability"]; // Session slice messages: (typeof defaultSessionState)["messages"]; @@ -152,6 +154,7 @@ export type AppState = KanbanSlice & { sessionMode: (typeof defaultSessionRuntimeState)["sessionMode"]; userShells: (typeof defaultSessionRuntimeState)["userShells"]; prepareProgress: (typeof defaultSessionRuntimeState)["prepareProgress"]; + launchWarning: (typeof defaultSessionRuntimeState)["launchWarning"]; sessionTodos: (typeof defaultSessionRuntimeState)["sessionTodos"]; agentCapabilities: (typeof defaultSessionRuntimeState)["agentCapabilities"]; sessionModels: (typeof defaultSessionRuntimeState)["sessionModels"]; @@ -282,6 +285,7 @@ export type AppState = KanbanSlice & { upsertAgentUpdateJob: (job: AgentUpdateJob) => void; appendAgentUpdateOutput: (agentName: string, jobId: string, chunk: string) => void; clearAgentUpdateJob: (agentName: string) => void; + setSSHReachability: (record: SSHReachabilityRecord) => void; setRepositories: (workspaceId: string, repositories: Repository[]) => void; upsertRepository: (workspaceId: string, repository: Repository) => void; setRepositoriesLoading: (workspaceId: string, loading: boolean) => void; @@ -599,6 +603,11 @@ export type AppState = KanbanSlice & { ) => void; setSessionPollMode: (sessionId: string, mode: SessionPollMode) => void; setEmbeddedVscodeSupport: (sessionId: string, supported: boolean) => void; + setLaunchWarning: ( + sessionId: string, + entry: import("./slices/session-runtime/types").LaunchWarningEntry, + ) => void; + clearLaunchWarning: (sessionId: string) => void; /* prettier-ignore */ setSidebarActiveView: UIA["setSidebarActiveView"]; createSidebarView: UIA["createSidebarView"]; updateSidebarDraft: UIA["updateSidebarDraft"]; diff --git a/apps/web/lib/state/default-state.ts b/apps/web/lib/state/default-state.ts index 1dccbf08f6..f2ae95ef99 100644 --- a/apps/web/lib/state/default-state.ts +++ b/apps/web/lib/state/default-state.ts @@ -101,6 +101,7 @@ export const defaultState = { sessionMode: defaultSessionRuntimeState.sessionMode, userShells: defaultSessionRuntimeState.userShells, prepareProgress: defaultSessionRuntimeState.prepareProgress, + launchWarning: defaultSessionRuntimeState.launchWarning, sessionTodos: defaultSessionRuntimeState.sessionTodos, agentCapabilities: defaultSessionRuntimeState.agentCapabilities, sessionModels: defaultSessionRuntimeState.sessionModels, @@ -515,6 +516,7 @@ export function mergeInitialState(initialState?: HydrationState): DefaultState { sessionMode: { ...defaultState.sessionMode, ...initialState.sessionMode }, userShells: { ...defaultState.userShells, ...initialState.userShells }, prepareProgress: { ...defaultState.prepareProgress, ...initialState.prepareProgress }, + launchWarning: { ...defaultState.launchWarning, ...initialState.launchWarning }, sessionTodos: { ...defaultState.sessionTodos, ...initialState.sessionTodos }, agentCapabilities: { ...defaultState.agentCapabilities, ...initialState.agentCapabilities }, sessionModels: { ...defaultState.sessionModels, ...initialState.sessionModels }, diff --git a/apps/web/lib/state/slices/session-runtime/purge-session.test.ts b/apps/web/lib/state/slices/session-runtime/purge-session.test.ts index e9c31b0706..fa17eff9b7 100644 --- a/apps/web/lib/state/slices/session-runtime/purge-session.test.ts +++ b/apps/web/lib/state/slices/session-runtime/purge-session.test.ts @@ -18,6 +18,20 @@ describe("purgeSessionRuntimeState", () => { store = makeStore(); }); + it("clears a launch warning when a relaunch reaches RUNNING", () => { + const s = store.getState(); + s.setLaunchWarning(SESSION_ID, { + executorId: "executor-1", + host: "10.0.0.5", + state: "unreachable", + reason: "timeout", + }); + + s.clearLaunchWarning(SESSION_ID); + + expect(store.getState().launchWarning.bySessionId[SESSION_ID]).toBeUndefined(); + }); + it("drops per-session maps, process output, and env-scoped buffers", () => { const s = store.getState(); s.registerSessionEnvironment(SESSION_ID, "env-1"); @@ -39,6 +53,12 @@ describe("purgeSessionRuntimeState", () => { status: "running", }); s.appendProcessOutput("proc-1", "process noise"); + s.setLaunchWarning("session-1", { + executorId: "executor-1", + host: "10.0.0.5", + state: "unreachable", + reason: "timeout", + }); store.setState((draft) => { purgeSessionRuntimeState(draft, SESSION_ID); @@ -52,6 +72,7 @@ describe("purgeSessionRuntimeState", () => { expect(after.processes.devProcessBySessionId[SESSION_ID]).toBeUndefined(); expect(after.processes.processesById["proc-1"]).toBeUndefined(); expect(after.processes.outputsByProcessId["proc-1"]).toBeUndefined(); + expect(after.launchWarning.bySessionId["session-1"]).toBeUndefined(); // env-scoped buffers gone because no other session references env-1. expect(after.shell.outputs["env-1"]).toBeUndefined(); expect(after.shell.statuses["env-1"]).toBeUndefined(); diff --git a/apps/web/lib/state/slices/session-runtime/session-runtime-slice.ts b/apps/web/lib/state/slices/session-runtime/session-runtime-slice.ts index d464e3a85d..1f116f060e 100644 --- a/apps/web/lib/state/slices/session-runtime/session-runtime-slice.ts +++ b/apps/web/lib/state/slices/session-runtime/session-runtime-slice.ts @@ -50,6 +50,7 @@ function purgePerSessionRuntime(state: SessionRuntimeSliceState, sessionId: stri delete state.promptUsage.bySessionId[sessionId]; delete state.sessionTodos.bySessionId[sessionId]; delete state.prepareProgress.bySessionId[sessionId]; + delete state.launchWarning.bySessionId[sessionId]; delete state.sessionPollMode.bySessionId[sessionId]; delete state.embeddedVscodeSupport.bySessionId[sessionId]; } @@ -121,6 +122,7 @@ export const defaultSessionRuntimeState: SessionRuntimeSliceState = { sessionTodos: { bySessionId: {} }, userShells: { byEnvironmentId: {}, dismissedByEnvironmentId: {}, loading: {}, loaded: {} }, prepareProgress: { bySessionId: {} }, + launchWarning: { bySessionId: {} }, sessionPollMode: { bySessionId: {} }, embeddedVscodeSupport: { bySessionId: {} }, workspaceFilesRefresh: { bySessionId: {} }, @@ -534,5 +536,13 @@ export const createSessionRuntimeSlice: StateCreator< set((draft) => { draft.sessionTodos.bySessionId[sessionId] = entries; }), + setLaunchWarning: (sessionId, entry) => + set((draft) => { + draft.launchWarning.bySessionId[sessionId] = entry; + }), + clearLaunchWarning: (sessionId) => + set((draft) => { + delete draft.launchWarning.bySessionId[sessionId]; + }), ...buildUserShellActions(set), }); diff --git a/apps/web/lib/state/slices/session-runtime/types.ts b/apps/web/lib/state/slices/session-runtime/types.ts index 308bdbcd57..108365ef9c 100644 --- a/apps/web/lib/state/slices/session-runtime/types.ts +++ b/apps/web/lib/state/slices/session-runtime/types.ts @@ -445,6 +445,22 @@ export type EmbeddedVscodeSupportState = { bySessionId: Record; }; +/** + * Applied verbatim from a session.launch.warning event — no field is ever + * re-derived from the reachability settings store. See launch-warning.tsx. + */ +export type LaunchWarningEntry = { + executorId: string; + host: string; + state: string; + reason: string; + lastSuccessAt?: string; +}; + +export type LaunchWarningState = { + bySessionId: Record; +}; + export type SessionRuntimeSliceState = { terminal: TerminalState; shell: ShellState; @@ -465,6 +481,7 @@ export type SessionRuntimeSliceState = { sessionTodos: SessionTodosState; userShells: UserShellsState; prepareProgress: PrepareProgressState; + launchWarning: LaunchWarningState; sessionPollMode: SessionPollModeState; embeddedVscodeSupport: EmbeddedVscodeSupportState; workspaceFilesRefresh: { bySessionId: Record }; @@ -558,6 +575,8 @@ export type SessionRuntimeSliceActions = { completeWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean; failWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt, details: string) => boolean; clearWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean; + setLaunchWarning: (sessionId: string, entry: LaunchWarningEntry) => void; + clearLaunchWarning: (sessionId: string) => void; }; export type SessionRuntimeSlice = SessionRuntimeSliceState & SessionRuntimeSliceActions; diff --git a/apps/web/lib/state/slices/settings/settings-slice.test.ts b/apps/web/lib/state/slices/settings/settings-slice.test.ts index d4d66fb8b1..b0bac80fb6 100644 --- a/apps/web/lib/state/slices/settings/settings-slice.test.ts +++ b/apps/web/lib/state/slices/settings/settings-slice.test.ts @@ -366,6 +366,107 @@ describe("setAvailableAgents capability propagation", () => { }); }); +describe("ssh reachability", () => { + const EXEC_ID = "exec-1"; + const BASE_TIMESTAMP = "2026-09-17T00:00:00Z"; + + function reachabilityRecord( + overrides: Partial = {}, + ): import("@/lib/types/http-ssh").SSHReachabilityRecord { + return { + executor_id: EXEC_ID, + state: "reachable", + reason: "", + consecutive_failures: 0, + host: "build.example", + checked_at: BASE_TIMESTAMP, + last_success_at: BASE_TIMESTAMP, + updated_at: BASE_TIMESTAMP, + probing_enabled: true, + probe_interval_seconds: 60, + persisted: true, + ...overrides, + }; + } + + it("stores a record keyed by executor id", () => { + const store = makeStore(); + store.getState().setSSHReachability(reachabilityRecord()); + expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reachabilityRecord()); + }); + + it("keeps the record with the later updated_at and discards an older one", () => { + const store = makeStore(); + const newer = reachabilityRecord({ state: "unreachable", updated_at: "2026-09-17T01:00:00Z" }); + const older = reachabilityRecord({ state: "reachable", updated_at: BASE_TIMESTAMP }); + + store.getState().setSSHReachability(newer); + store.getState().setSSHReachability(older); + + expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(newer); + }); + + it("applies a reset (null checked_at, newer updated_at) over a stored reachable/unreachable record", () => { + // A connection-configuration reset clears checked_at but still advances + // updated_at. Reconciling on checked_at would make this reset compare as + // older than the record it just invalidated and get discarded, leaving a + // stale reachable/unreachable state showing for a host the user just + // re-pointed. + const store = makeStore(); + const stale = reachabilityRecord({ + state: "unreachable", + checked_at: "2026-09-17T02:00:00Z", + updated_at: "2026-09-17T02:00:00Z", + }); + const reset = reachabilityRecord({ + state: "unknown", + checked_at: null, + updated_at: "2026-09-17T03:00:00Z", + persisted: false, + }); + + store.getState().setSSHReachability(stale); + store.getState().setSSHReachability(reset); + + expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reset); + }); + + it("discards a null-updated_at placeholder applied after a record with a real timestamp", () => { + // The synthesized never-probed placeholder always loses. + const store = makeStore(); + const real = reachabilityRecord({ updated_at: BASE_TIMESTAMP }); + const placeholder = reachabilityRecord({ state: "unknown", updated_at: null }); + + store.getState().setSSHReachability(real); + store.getState().setSSHReachability(placeholder); + + expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(real); + }); + + it("applies the first record for an executor even when its updated_at is null", () => { + const store = makeStore(); + const placeholder = reachabilityRecord({ state: "unknown", updated_at: null }); + + store.getState().setSSHReachability(placeholder); + + expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(placeholder); + }); + + it("tracks distinct executors independently", () => { + const store = makeStore(); + const first = reachabilityRecord({ executor_id: EXEC_ID }); + const second = reachabilityRecord({ executor_id: "exec-2", state: "unreachable" }); + + store.getState().setSSHReachability(first); + store.getState().setSSHReachability(second); + + expect(store.getState().sshReachability.byExecutorId).toEqual({ + [EXEC_ID]: first, + "exec-2": second, + }); + }); +}); + describe("folder opener discovery", () => { it("defaults unavailable and preserves capability through editor preference updates", () => { const store = makeStore(); diff --git a/apps/web/lib/state/slices/settings/settings-slice.ts b/apps/web/lib/state/slices/settings/settings-slice.ts index 2dc5810223..b9eb53bc0b 100644 --- a/apps/web/lib/state/slices/settings/settings-slice.ts +++ b/apps/web/lib/state/slices/settings/settings-slice.ts @@ -38,6 +38,7 @@ export const defaultSettingsState: SettingsSliceState = { sleepInhibition: { response: null, loaded: false, loading: false, error: false }, userSettings: createDefaultUserSettings(), agentProfileRecentUse: { records: {}, loaded: false }, + sshReachability: { byExecutorId: {} }, }; type ImmerSet = Parameters< @@ -437,6 +438,29 @@ function createSecretAndSpriteActions( }; } +// reachabilityUpdatedAtMs parses updated_at to epoch ms for reconciliation. +// A null updated_at (the synthesized never-probed placeholder) sorts as +// "always older" so any real record replaces it. +function reachabilityUpdatedAtMs(updatedAt: string | null): number { + return updatedAt ? Date.parse(updatedAt) : -Infinity; +} + +function createSSHReachabilityActions(set: ImmerSet): Pick { + return { + setSSHReachability: (record) => + set((draft) => { + const current = draft.sshReachability.byExecutorId[record.executor_id]; + if ( + current && + reachabilityUpdatedAtMs(current.updated_at) > reachabilityUpdatedAtMs(record.updated_at) + ) { + return; + } + draft.sshReachability.byExecutorId[record.executor_id] = record; + }), + }; +} + export const createSettingsSlice: StateCreator< SettingsSlice, [["zustand/immer", never]], @@ -450,4 +474,5 @@ export const createSettingsSlice: StateCreator< ...createInstallJobActions(set), ...createAgentUpdateJobActions(set), ...createSecretAndSpriteActions(set), + ...createSSHReachabilityActions(set), }); diff --git a/apps/web/lib/state/slices/settings/types.ts b/apps/web/lib/state/slices/settings/types.ts index 733018a2d7..00420bea73 100644 --- a/apps/web/lib/state/slices/settings/types.ts +++ b/apps/web/lib/state/slices/settings/types.ts @@ -34,6 +34,7 @@ import type { } from "@/lib/agent-profile-recent-use"; import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use"; import type { TaskColor } from "@/lib/task-colors"; +import type { SSHReachabilityRecord } from "@/lib/types/http-ssh"; export type { AgentProfileRecentUseRecord, @@ -44,6 +45,15 @@ export type ExecutorsState = { items: Executor[]; }; +/** + * SSH reachability records keyed by executor id. Populated by a per-card + * fetch (task 06's SSHReachabilityCard) and kept live via the + * executor.reachability.changed WS event. + */ +export type SSHReachabilityStoreState = { + byExecutorId: Record; +}; + export type SettingsAgentsState = { items: Agent[]; }; @@ -520,6 +530,7 @@ export type SettingsSliceState = { sleepInhibition: SleepInhibitionStoreState; userSettings: UserSettingsState; agentProfileRecentUse: AgentProfileRecentUseState; + sshReachability: SSHReachabilityStoreState; }; export type SettingsSliceActions = { @@ -568,6 +579,16 @@ export type SettingsSliceActions = { record: AgentProfileRecentUseRecord, ) => void; bumpAgentProfilesVersion: () => void; + /** + * Applies a reachability record (a fetch response or a pushed + * executor.reachability.changed event). Reconciles on updated_at, never + * checked_at: a connection-configuration reset clears checked_at (null) + * while still advancing updated_at, so comparing on checked_at would make + * the reset compare as older than the record it just invalidated and get + * discarded. A null updated_at (the synthesized never-probed placeholder) + * always loses to a record that has one. + */ + setSSHReachability: (record: SSHReachabilityRecord) => void; }; export type SettingsSlice = SettingsSliceState & SettingsSliceActions; diff --git a/apps/web/lib/state/store-overrides.ts b/apps/web/lib/state/store-overrides.ts index 47fb5ce41c..f77b9cd0c7 100644 --- a/apps/web/lib/state/store-overrides.ts +++ b/apps/web/lib/state/store-overrides.ts @@ -53,6 +53,7 @@ export function buildStateOverrides(m: DefaultState) { sessionMode: m.sessionMode, userShells: m.userShells, prepareProgress: m.prepareProgress, + launchWarning: m.launchWarning, sessionTodos: m.sessionTodos, agentCapabilities: m.agentCapabilities, sessionModels: m.sessionModels, diff --git a/apps/web/lib/types/backend.ts b/apps/web/lib/types/backend.ts index 5a3178a93e..11b67aea2e 100644 --- a/apps/web/lib/types/backend.ts +++ b/apps/web/lib/types/backend.ts @@ -44,6 +44,7 @@ import type { } from "@/lib/types/github"; import type { TaskMR, TaskMRDeletedEvent, TaskMRAutomationOptions } from "@/lib/types/gitlab"; import type { TaskStatusSummary } from "@/lib/types/task-status-summary"; +import type { SSHReachabilityRecord } from "@/lib/types/http-ssh"; import type { AgentProfileRecentUseApiRecord } from "@/lib/types/http-agent-profile-recent-use"; import type { SystemMetricsSnapshot, StorageAnalysisUpdatedPayload } from "./system"; import type { AgentRuntimeAvailability } from "./agent-runtime"; @@ -52,6 +53,7 @@ import type { ExecutorProfilePayload, PrepareProgressPayload, PrepareCompletedPayload, + LaunchWarningPayload, EnvironmentPayload, } from "./executor-payloads"; @@ -358,6 +360,7 @@ export { type ExecutorProfilePayload, type PrepareProgressPayload, type PrepareCompletedPayload, + type LaunchWarningPayload, type EnvironmentPayload, } from "./executor-payloads"; @@ -552,6 +555,11 @@ export type BackendMessageMap = SessionBackendMessageMap & "executor.prepare.completed", PrepareCompletedPayload >; + "executor.reachability.changed": BackendMessage< + "executor.reachability.changed", + SSHReachabilityRecord + >; + "session.launch.warning": BackendMessage<"session.launch.warning", LaunchWarningPayload>; "environment.created": BackendMessage<"environment.created", EnvironmentPayload>; "environment.updated": BackendMessage<"environment.updated", EnvironmentPayload>; "environment.deleted": BackendMessage<"environment.deleted", EnvironmentPayload>; diff --git a/apps/web/lib/types/executor-payloads.ts b/apps/web/lib/types/executor-payloads.ts index 0ab8a2e3a7..b2364c499f 100644 --- a/apps/web/lib/types/executor-payloads.ts +++ b/apps/web/lib/types/executor-payloads.ts @@ -65,6 +65,17 @@ export type PrepareCompletedPayload = { timestamp: string; }; +export type LaunchWarningPayload = { + task_id: string; + session_id: string; + executor_id: string; + host: string; + state: string; + reason: string; + last_success_at?: string; + timestamp: string; +}; + export type EnvironmentPayload = { id: string; name: string; diff --git a/apps/web/lib/types/http-ssh.ts b/apps/web/lib/types/http-ssh.ts index 8ee6bc4d3d..1cffac43ef 100644 --- a/apps/web/lib/types/http-ssh.ts +++ b/apps/web/lib/types/http-ssh.ts @@ -70,3 +70,32 @@ export interface SSHProbeShellsResponse { duration_ms: number; available: string[]; } + +export type SSHReachabilityState = "unknown" | "reachable" | "unreachable"; + +export type SSHReachabilityReason = + | "" + | "config" + | "timeout" + | "host_key" + | "auth" + | "network" + | "unknown"; + +/** Mirrors the backend's reachability.RecordDTO wire shape. */ +export interface SSHReachabilityRecord { + executor_id: string; + state: SSHReachabilityState; + reason: SSHReachabilityReason; + message?: string; + consecutive_failures: number; + host?: string; + checked_at: string | null; + last_success_at: string | null; + /** Null on the synthesized "never probed" placeholder. */ + updated_at: string | null; + probing_enabled: boolean; + /** The effective, clamped probe interval in seconds (0 when disabled). */ + probe_interval_seconds: number; + persisted: boolean; +} diff --git a/apps/web/lib/ws/handlers/agent-session.test.ts b/apps/web/lib/ws/handlers/agent-session.test.ts index 33cb96487f..a7d1d4f0f2 100644 --- a/apps/web/lib/ws/handlers/agent-session.test.ts +++ b/apps/web/lib/ws/handlers/agent-session.test.ts @@ -33,6 +33,7 @@ function makeStore(overrides: Record = {}) { setActiveSessionAuto: vi.fn(), setSessionAgentctlStatus: vi.fn(), setResumeSkipped: vi.fn(), + clearLaunchWarning: vi.fn(), setSessionFailureNotification: vi.fn(), setContextWindow: vi.fn(), clearContextWindow: vi.fn(), @@ -2079,6 +2080,7 @@ describe("session.state_changed resume-skipped clearing", () => { it("clears the resume-skipped marker on a RUNNING transition", () => { handler(makeMessage({ task_id: "t-1", session_id: "s-1", new_state: "RUNNING" })); expect(store.getState().setResumeSkipped).toHaveBeenCalledWith("s-1", false); + expect(store.getState().clearLaunchWarning).toHaveBeenCalledWith("s-1"); }); it("does NOT clear the marker on a STARTING transition (failed resumes keep the retry button)", () => { diff --git a/apps/web/lib/ws/handlers/agent-session.ts b/apps/web/lib/ws/handlers/agent-session.ts index 8f2e47fc6b..b5a310620e 100644 --- a/apps/web/lib/ws/handlers/agent-session.ts +++ b/apps/web/lib/ws/handlers/agent-session.ts @@ -853,6 +853,12 @@ function handleQueueStatusChangedMessage( state.setQueueEntries(payload.session_id, entries, meta); } +function clearResumeAndLaunchWarnings(store: StoreApi, sessionId: string) { + const state = store.getState(); + state.setResumeSkipped(sessionId, false); + state.clearLaunchWarning?.(sessionId); +} + /** Registers the task-session WebSocket handlers (state, messages, workspace sources, queue). */ // eslint-disable-next-line max-lines-per-function -- session events remain one ordered registry. export function registerTaskSessionHandlers(store: StoreApi): WsHandlers { @@ -912,7 +918,7 @@ export function registerTaskSessionHandlers(store: StoreApi): WsHandle // it: a failed manual resume emits STARTING before the launch fails, // and clearing there would drop the Start agent retry affordance. if (newState === "RUNNING") { - store.getState().setResumeSkipped(sessionId, false); + clearResumeAndLaunchWarnings(store, sessionId); } maybeAdoptSessionOnTransition( diff --git a/apps/web/lib/ws/handlers/executors.ts b/apps/web/lib/ws/handlers/executors.ts index da84aef80a..7e67252d1c 100644 --- a/apps/web/lib/ws/handlers/executors.ts +++ b/apps/web/lib/ws/handlers/executors.ts @@ -42,5 +42,8 @@ export function registerExecutorsHandlers(store: StoreApi): WsHandlers }, })); }, + "executor.reachability.changed": (message) => { + store.getState().setSSHReachability(message.payload); + }, }; } diff --git a/apps/web/lib/ws/handlers/ssh-launch-warning.test.ts b/apps/web/lib/ws/handlers/ssh-launch-warning.test.ts new file mode 100644 index 0000000000..b533bfb816 --- /dev/null +++ b/apps/web/lib/ws/handlers/ssh-launch-warning.test.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { StoreApi } from "zustand"; +import type { AppState } from "@/lib/state/store"; +import type { BackendMessageMap, LaunchWarningPayload } from "@/lib/types/backend"; +import { registerSSHLaunchWarningHandlers } from "./ssh-launch-warning"; + +const SESSION_ID = "session-1"; +const TASK_ID = "task-1"; + +function makeStore() { + const state = { + launchWarning: { bySessionId: {} as Record }, + setLaunchWarning: vi.fn(), + } as unknown as AppState; + + return { + getState: () => state, + setState: vi.fn(), + subscribe: vi.fn(), + destroy: vi.fn(), + getInitialState: vi.fn(), + } as unknown as StoreApi; +} + +function makePayload(overrides: Partial = {}): LaunchWarningPayload { + return { + task_id: TASK_ID, + session_id: SESSION_ID, + executor_id: "executor-1", + host: "10.0.0.5", + state: "unreachable", + reason: "timeout", + last_success_at: "2026-06-10T00:00:00.000Z", + timestamp: "2026-06-11T00:00:00.000Z", + ...overrides, + }; +} + +function makeMessage(payload: LaunchWarningPayload): BackendMessageMap["session.launch.warning"] { + return { + id: "message-1", + type: "notification", + action: "session.launch.warning", + payload, + timestamp: "2026-06-11T00:00:00.000Z", + }; +} + +describe("session.launch.warning handler", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("stores a normalized entry keyed by session id", () => { + const store = makeStore(); + const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!; + + handler(makeMessage(makePayload())); + + expect(store.getState().setLaunchWarning).toHaveBeenCalledWith(SESSION_ID, { + executorId: "executor-1", + host: "10.0.0.5", + state: "unreachable", + reason: "timeout", + lastSuccessAt: "2026-06-10T00:00:00.000Z", + }); + }); + + it("omits lastSuccessAt when the record has never succeeded", () => { + const store = makeStore(); + const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!; + + handler(makeMessage(makePayload({ last_success_at: undefined }))); + + expect(store.getState().setLaunchWarning).toHaveBeenCalledWith( + SESSION_ID, + expect.objectContaining({ lastSuccessAt: undefined }), + ); + }); + + it("ignores payloads without a session id", () => { + const store = makeStore(); + const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!; + + handler(makeMessage(makePayload({ session_id: "" }))); + + expect(store.getState().setLaunchWarning).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/lib/ws/handlers/ssh-launch-warning.ts b/apps/web/lib/ws/handlers/ssh-launch-warning.ts new file mode 100644 index 0000000000..968cd5c2c5 --- /dev/null +++ b/apps/web/lib/ws/handlers/ssh-launch-warning.ts @@ -0,0 +1,20 @@ +import type { StoreApi } from "zustand"; +import type { AppState } from "@/lib/state/store"; +import type { WsHandlers } from "@/lib/ws/handlers/types"; +import type { LaunchWarningPayload } from "@/lib/types/backend"; + +export function registerSSHLaunchWarningHandlers(store: StoreApi): WsHandlers { + return { + "session.launch.warning": (message) => { + const payload = message.payload as LaunchWarningPayload; + if (!payload.session_id) return; + store.getState().setLaunchWarning(payload.session_id, { + executorId: payload.executor_id, + host: payload.host, + state: payload.state, + reason: payload.reason, + lastSuccessAt: payload.last_success_at, + }); + }, + }; +} diff --git a/apps/web/lib/ws/router.ts b/apps/web/lib/ws/router.ts index 74c5ef613e..66d7177315 100644 --- a/apps/web/lib/ws/router.ts +++ b/apps/web/lib/ws/router.ts @@ -20,6 +20,7 @@ import { registerDiffsHandlers } from "@/lib/ws/handlers/diffs"; import { registerExecutorsHandlers } from "@/lib/ws/handlers/executors"; import { registerExecutorProfileHandlers } from "@/lib/ws/handlers/executor-profiles"; import { registerExecutorPrepareHandlers } from "@/lib/ws/handlers/executor-prepare"; +import { registerSSHLaunchWarningHandlers } from "@/lib/ws/handlers/ssh-launch-warning"; import { registerGitStatusHandlers } from "@/lib/ws/handlers/git-status"; import { registerKanbanHandlers } from "@/lib/ws/handlers/kanban"; import { registerSystemEventsHandlers } from "@/lib/ws/handlers/system-events"; @@ -59,6 +60,7 @@ export function registerWsHandlers(store: StoreApi) { ...registerExecutorsHandlers(store), ...registerExecutorProfileHandlers(store), ...registerExecutorPrepareHandlers(store), + ...registerSSHLaunchWarningHandlers(store), ...registerAgentsHandlers(store), ...registerTaskSessionHandlers(store), ...registerAvailableCommandsHandlers(store), diff --git a/apps/web/src/locales/en/executors.json b/apps/web/src/locales/en/executors.json index 5f0b6ae77d..a5ffbd99ff 100644 --- a/apps/web/src/locales/en/executors.json +++ b/apps/web/src/locales/en/executors.json @@ -386,6 +386,28 @@ "sshProbeHint": "Click \"{{button}}\" to check which agents are installed on the remote.", "sshProxyJumpHint": "Single bastion hop. Chained jumps are not supported.", "sshProxyJumpLabel": "ProxyJump (optional)", + "sshReachabilityAgeLastProbe": "Last probe: {{age}}", + "sshReachabilityAgeLastSuccess": "Last success: {{age}}", + "sshReachabilityConsecutiveFailures_one": "{{count}} consecutive failure", + "sshReachabilityConsecutiveFailures_other": "{{count}} consecutive failures", + "sshReachabilityDescription": "The most recent SSH probe result for this host, checked automatically in the background.", + "sshReachabilityHost": "Probed host: {{host}}", + "sshReachabilityLoadFailed": "Reachability status could not be loaded. It is not known.", + "sshReachabilityNeverProbed": "Not probed yet.", + "sshReachabilityNoSuccess": "No successful probe recorded.", + "sshReachabilityProbeNow": "Probe now", + "sshReachabilityProbingOff": "Periodic probing is off. This shows the last recorded result, not the current state.", + "sshReachabilityReasonAuth": "Authentication failed", + "sshReachabilityReasonConfig": "The connection configuration is invalid", + "sshReachabilityReasonHostKey": "The host key does not match the pinned fingerprint", + "sshReachabilityReasonNetwork": "A network error occurred", + "sshReachabilityReasonTimeout": "The connection attempt timed out", + "sshReachabilityReasonUnknown": "An unknown error occurred", + "sshReachabilityStale": "Stale: no probe within the last three check intervals", + "sshReachabilityStateReachable": "Reachable", + "sshReachabilityStateUnknown": "Unknown", + "sshReachabilityStateUnreachable": "Unreachable", + "sshReachabilityTitle": "Reachability", "sshReclaimTaskDirDescription": "Every task that runs on {{host}} keeps its checkout under {{path}} after the task ends, so a resumed or unarchived task finds its work where it left it.", "sshReclaimTaskDirLabel": "Delete a task's directory when the task is archived or deleted", "sshReclaimTaskDirTitle": "Remote task directories", diff --git a/apps/web/src/locales/en/task.json b/apps/web/src/locales/en/task.json index f78806eee1..538b8df438 100644 --- a/apps/web/src/locales/en/task.json +++ b/apps/web/src/locales/en/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "The task launch needs attention.", "launchNeedsAttention": "Launch needs attention", "launchErrorNoChanges": "No task or branch settings were changed.", + "launchWarningTitle": "SSH host may be unreachable", + "launchWarningHost": "Probed host: {{host}}", + "launchWarningLastSuccess": "Last success: {{age}}", + "launchWarningNeverSucceeded": "No successful probe has ever been recorded for this host.", + "launchWarningReasonConfig": "The last attempt failed because the connection configuration is invalid.", + "launchWarningReasonTimeout": "The last attempt timed out.", + "launchWarningReasonHostKey": "The last attempt failed because the host key did not match the pinned fingerprint.", + "launchWarningReasonAuth": "The last attempt failed authentication.", + "launchWarningReasonNetwork": "The last attempt failed with a network error.", + "launchWarningReasonUnknown": "The last attempt failed for an unknown reason.", "launchPromptPreview": "Launch prompt preview", "retryWithDefaultBranch": "Retry with default branch", "pickBaseBranch": "Pick a base branch", diff --git a/apps/web/src/locales/pseudo/executors.json b/apps/web/src/locales/pseudo/executors.json index d021ffe94c..bf23528b08 100644 --- a/apps/web/src/locales/pseudo/executors.json +++ b/apps/web/src/locales/pseudo/executors.json @@ -386,6 +386,28 @@ "sshProbeHint": "Ćĺĩćķ \"{{button}}\" ţō ćĥēćķ ŵĥĩćĥ àĝēńţś àŕē ĩńśţàĺĺēď ōń ţĥē ŕēḿōţē.", "sshProxyJumpHint": "Śĩńĝĺē ƀàśţĩōń ĥōƥ. Ćĥàĩńēď ĵũḿƥś àŕē ńōţ śũƥƥōŕţēď.", "sshProxyJumpLabel": "ƤŕōxŷĴũḿƥ (ōƥţĩōńàĺ)", + "sshReachabilityAgeLastProbe": "Ĺàśţ ƥŕōƀē: {{age}}", + "sshReachabilityAgeLastSuccess": "Ĺàśţ śũććēśś: {{age}}", + "sshReachabilityConsecutiveFailures_one": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕē", + "sshReachabilityConsecutiveFailures_other": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕēś", + "sshReachabilityDescription": "Ţĥē ḿōśţ ŕēćēńţ ŚŚĤ ƥŕōƀē ŕēśũĺţ ƒōŕ ţĥĩś ĥōśţ, ćĥēćķēď àũţōḿàţĩćàĺĺŷ ĩń ţĥē ƀàćķĝŕōũńď.", + "sshReachabilityHost": "Ƥŕōƀēď ĥōśţ: {{host}}", + "sshReachabilityLoadFailed": "Ŕēàćĥàƀĩĺĩţŷ śţàţũś ćōũĺď ńōţ ƀē ĺōàďēď. Ĩţ ĩś ńōţ ķńōŵń.", + "sshReachabilityNeverProbed": "Ńōţ ƥŕōƀēď ŷēţ.", + "sshReachabilityNoSuccess": "Ńō śũććēśśƒũĺ ƥŕōƀē ŕēćōŕďēď.", + "sshReachabilityProbeNow": "Ƥŕōƀē ńōŵ", + "sshReachabilityProbingOff": "Ƥēŕĩōďĩć ƥŕōƀĩńĝ ĩś ōƒƒ. Ţĥĩś śĥōŵś ţĥē ĺàśţ ŕēćōŕďēď ŕēśũĺţ, ńōţ ţĥē ćũŕŕēńţ śţàţē.", + "sshReachabilityReasonAuth": "Àũţĥēńţĩćàţĩōń ƒàĩĺēď", + "sshReachabilityReasonConfig": "Ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď", + "sshReachabilityReasonHostKey": "Ţĥē ĥōśţ ķēŷ ďōēś ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ", + "sshReachabilityReasonNetwork": "À ńēţŵōŕķ ēŕŕōŕ ōććũŕŕēď", + "sshReachabilityReasonTimeout": "Ţĥē ćōńńēćţĩōń àţţēḿƥţ ţĩḿēď ōũţ", + "sshReachabilityReasonUnknown": "Àń ũńķńōŵń ēŕŕōŕ ōććũŕŕēď", + "sshReachabilityStale": "Śţàĺē: ńō ƥŕōƀē ŵĩţĥĩń ţĥē ĺàśţ ţĥŕēē ćĥēćķ ĩńţēŕvàĺś", + "sshReachabilityStateReachable": "Ŕēàćĥàƀĺē", + "sshReachabilityStateUnknown": "Ũńķńōŵń", + "sshReachabilityStateUnreachable": "Ũńŕēàćĥàƀĺē", + "sshReachabilityTitle": "Ŕēàćĥàƀĩĺĩţŷ", "sshReclaimTaskDirDescription": "Ēvēŕŷ ţàśķ ţĥàţ ŕũńś ōń {{host}} ķēēƥś ĩţś ćĥēćķōũţ ũńďēŕ {{path}} àƒţēŕ ţĥē ţàśķ ēńďś, śō à ŕēśũḿēď ōŕ ũńàŕćĥĩvēď ţàśķ ƒĩńďś ĩţś ŵōŕķ ŵĥēŕē ĩţ ĺēƒţ ĩţ.", "sshReclaimTaskDirLabel": "Ďēĺēţē à ţàśķ'ś ďĩŕēćţōŕŷ ŵĥēń ţĥē ţàśķ ĩś àŕćĥĩvēď ōŕ ďēĺēţēď", "sshReclaimTaskDirTitle": "Ŕēḿōţē ţàśķ ďĩŕēćţōŕĩēś", diff --git a/apps/web/src/locales/pseudo/task.json b/apps/web/src/locales/pseudo/task.json index e0cea045b3..8c2609657b 100644 --- a/apps/web/src/locales/pseudo/task.json +++ b/apps/web/src/locales/pseudo/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "Ţĥē ţàśķ ĺàũńćĥ ńēēďś àţţēńţĩōń.", "launchNeedsAttention": "Ĺàũńćĥ ńēēďś àţţēńţĩōń", "launchErrorNoChanges": "Ńō ţàśķ ōŕ ƀŕàńćĥ śēţţĩńĝś ŵēŕē ćĥàńĝēď.", + "launchWarningTitle": "ŚŚĤ ĥōśţ ḿàŷ ƀē ũńŕēàćĥàƀĺē", + "launchWarningHost": "Ƥŕōƀēď ĥōśţ: {{host}}", + "launchWarningLastSuccess": "Ĺàśţ śũććēśś: {{age}}", + "launchWarningNeverSucceeded": "Ńō śũććēśśƒũĺ ƥŕōƀē ĥàś ēvēŕ ƀēēń ŕēćōŕďēď ƒōŕ ţĥĩś ĥōśţ.", + "launchWarningReasonConfig": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď.", + "launchWarningReasonTimeout": "Ţĥē ĺàśţ àţţēḿƥţ ţĩḿēď ōũţ.", + "launchWarningReasonHostKey": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ĥōśţ ķēŷ ďĩď ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ.", + "launchWarningReasonAuth": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď àũţĥēńţĩćàţĩōń.", + "launchWarningReasonNetwork": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ŵĩţĥ à ńēţŵōŕķ ēŕŕōŕ.", + "launchWarningReasonUnknown": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƒōŕ àń ũńķńōŵń ŕēàśōń.", "launchPromptPreview": "Ĺàũńćĥ ƥŕōḿƥţ ƥŕēvĩēŵ", "retryWithDefaultBranch": "Ŕēţŕŷ ŵĩţĥ ďēƒàũĺţ ƀŕàńćĥ", "pickBaseBranch": "Ƥĩćķ à ƀàśē ƀŕàńćĥ", diff --git a/apps/web/src/locales/pt-pt/executors.json b/apps/web/src/locales/pt-pt/executors.json index 496e96a3f5..3898d1cc07 100644 --- a/apps/web/src/locales/pt-pt/executors.json +++ b/apps/web/src/locales/pt-pt/executors.json @@ -370,6 +370,28 @@ "sshProbeHint": "Clique em \"{{button}}\" para verificar que agentes estão instalados no anfitrião remoto.", "sshProxyJumpHint": "Um único salto por bastião. Saltos encadeados não são suportados.", "sshProxyJumpLabel": "ProxyJump (opcional)", + "sshReachabilityAgeLastProbe": "Última verificação: {{age}}", + "sshReachabilityAgeLastSuccess": "Último sucesso: {{age}}", + "sshReachabilityConsecutiveFailures_one": "{{count}} falha consecutiva", + "sshReachabilityConsecutiveFailures_other": "{{count}} falhas consecutivas", + "sshReachabilityDescription": "O resultado mais recente da verificação SSH deste anfitrião, verificado automaticamente em segundo plano.", + "sshReachabilityHost": "Anfitrião verificado: {{host}}", + "sshReachabilityLoadFailed": "Não foi possível carregar o estado de alcançabilidade. Não é conhecido.", + "sshReachabilityNeverProbed": "Ainda não verificado.", + "sshReachabilityNoSuccess": "Nenhuma verificação bem-sucedida registada.", + "sshReachabilityProbeNow": "Verificar agora", + "sshReachabilityProbingOff": "A verificação periódica está desativada. Isto mostra o último resultado registado, não o estado atual.", + "sshReachabilityReasonAuth": "Falha na autenticação", + "sshReachabilityReasonConfig": "A configuração da ligação é inválida", + "sshReachabilityReasonHostKey": "A chave do anfitrião não corresponde à impressão digital fixada", + "sshReachabilityReasonNetwork": "Ocorreu um erro de rede", + "sshReachabilityReasonTimeout": "A tentativa de ligação expirou", + "sshReachabilityReasonUnknown": "Ocorreu um erro desconhecido", + "sshReachabilityStale": "Desatualizado: sem verificação nos últimos três intervalos", + "sshReachabilityStateReachable": "Alcançável", + "sshReachabilityStateUnknown": "Desconhecido", + "sshReachabilityStateUnreachable": "Inalcançável", + "sshReachabilityTitle": "Alcançabilidade", "sshReclaimTaskDirDescription": "Cada tarefa executada em {{host}} mantém a sua cópia de trabalho em {{path}} depois de terminar, para que uma tarefa retomada ou desarquivada encontre o trabalho onde o deixou.", "sshReclaimTaskDirLabel": "Eliminar o diretório de uma tarefa quando a tarefa for arquivada ou eliminada", "sshReclaimTaskDirTitle": "Diretórios de tarefas remotos", diff --git a/apps/web/src/locales/pt-pt/task.json b/apps/web/src/locales/pt-pt/task.json index 0e41cbf824..1dc9be4a5f 100644 --- a/apps/web/src/locales/pt-pt/task.json +++ b/apps/web/src/locales/pt-pt/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "O início da tarefa requer atenção.", "launchNeedsAttention": "O início requer atenção", "launchErrorNoChanges": "As definições da tarefa e do branch não foram alteradas.", + "launchWarningTitle": "O anfitrião SSH pode estar inacessível", + "launchWarningHost": "Anfitrião testado: {{host}}", + "launchWarningLastSuccess": "Último sucesso: {{age}}", + "launchWarningNeverSucceeded": "Nunca foi registada uma sondagem bem-sucedida para este anfitrião.", + "launchWarningReasonConfig": "A última tentativa falhou porque a configuração da ligação é inválida.", + "launchWarningReasonTimeout": "A última tentativa expirou.", + "launchWarningReasonHostKey": "A última tentativa falhou porque a chave do anfitrião não correspondeu à impressão digital fixada.", + "launchWarningReasonAuth": "A última tentativa falhou na autenticação.", + "launchWarningReasonNetwork": "A última tentativa falhou com um erro de rede.", + "launchWarningReasonUnknown": "A última tentativa falhou por um motivo desconhecido.", "launchPromptPreview": "Pré-visualização do prompt de início", "retryWithDefaultBranch": "Tentar novamente com o branch predefinido", "pickBaseBranch": "Escolher um branch base", diff --git a/apps/web/src/locales/zh-cn/executors.json b/apps/web/src/locales/zh-cn/executors.json index ba649a88db..721577e016 100644 --- a/apps/web/src/locales/zh-cn/executors.json +++ b/apps/web/src/locales/zh-cn/executors.json @@ -370,6 +370,28 @@ "sshProbeHint": "点击“{{button}}”检查远程主机上已安装哪些智能体。", "sshProxyJumpHint": "仅支持单跳堡垒。不支持链式跳转。", "sshProxyJumpLabel": "ProxyJump（可选）", + "sshReachabilityAgeLastProbe": "上次检测：{{age}}", + "sshReachabilityAgeLastSuccess": "上次成功：{{age}}", + "sshReachabilityConsecutiveFailures_one": "连续 {{count}} 次失败", + "sshReachabilityConsecutiveFailures_other": "连续 {{count}} 次失败", + "sshReachabilityDescription": "此主机最近一次 SSH 可达性检测结果，系统会在后台自动检测。", + "sshReachabilityHost": "检测的主机：{{host}}", + "sshReachabilityLoadFailed": "无法加载可达性状态，当前状态未知。", + "sshReachabilityNeverProbed": "尚未检测。", + "sshReachabilityNoSuccess": "没有成功的检测记录。", + "sshReachabilityProbeNow": "立即检测", + "sshReachabilityProbingOff": "定期检测已关闭。这里显示的是上次记录的结果，而非当前状态。", + "sshReachabilityReasonAuth": "身份验证失败", + "sshReachabilityReasonConfig": "连接配置无效", + "sshReachabilityReasonHostKey": "主机密钥与已固定的指纹不匹配", + "sshReachabilityReasonNetwork": "发生网络错误", + "sshReachabilityReasonTimeout": "连接尝试超时", + "sshReachabilityReasonUnknown": "发生未知错误", + "sshReachabilityStale": "已过期：最近三个检测周期内未检测", + "sshReachabilityStateReachable": "可达", + "sshReachabilityStateUnknown": "未知", + "sshReachabilityStateUnreachable": "不可达", + "sshReachabilityTitle": "可达性", "sshReclaimTaskDirDescription": "在 {{host}} 上运行的每个任务在结束后都会把检出保留在 {{path}}，这样恢复或取消归档的任务仍能找到原来的工作内容。", "sshReclaimTaskDirLabel": "任务被归档或删除时删除其目录", "sshReclaimTaskDirTitle": "远程任务目录", diff --git a/apps/web/src/locales/zh-cn/task.json b/apps/web/src/locales/zh-cn/task.json index 455d259c3c..a099a11788 100644 --- a/apps/web/src/locales/zh-cn/task.json +++ b/apps/web/src/locales/zh-cn/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "任务启动需要处理。", "launchNeedsAttention": "启动需要处理", "launchErrorNoChanges": "任务或分支设置未更改。", + "launchWarningTitle": "SSH 主机可能无法访问", + "launchWarningHost": "已探测主机：{{host}}", + "launchWarningLastSuccess": "上次成功：{{age}}", + "launchWarningNeverSucceeded": "从未记录过此主机的成功探测。", + "launchWarningReasonConfig": "上次尝试失败，因为连接配置无效。", + "launchWarningReasonTimeout": "上次尝试超时。", + "launchWarningReasonHostKey": "上次尝试失败，因为主机密钥与固定的指纹不匹配。", + "launchWarningReasonAuth": "上次尝试的身份验证失败。", + "launchWarningReasonNetwork": "上次尝试因网络错误而失败。", + "launchWarningReasonUnknown": "上次尝试因未知原因失败。", "launchPromptPreview": "启动提示词预览", "retryWithDefaultBranch": "使用默认分支重试", "pickBaseBranch": "选择基础分支", diff --git a/apps/web/src/locales/zh-hk/executors.json b/apps/web/src/locales/zh-hk/executors.json index 3321a84e74..d076c7bfbb 100644 --- a/apps/web/src/locales/zh-hk/executors.json +++ b/apps/web/src/locales/zh-hk/executors.json @@ -370,6 +370,28 @@ "sshProbeHint": "點擊「{{button}}」檢查遠端主機上已安裝哪些代理程式。", "sshProxyJumpHint": "僅支援單跳堡壘。不支援鏈式跳轉。", "sshProxyJumpLabel": "ProxyJump（可選）", + "sshReachabilityAgeLastProbe": "上次檢測：{{age}}", + "sshReachabilityAgeLastSuccess": "上次成功：{{age}}", + "sshReachabilityConsecutiveFailures_one": "連續 {{count}} 次失敗", + "sshReachabilityConsecutiveFailures_other": "連續 {{count}} 次失敗", + "sshReachabilityDescription": "此主機最近一次 SSH 可達性檢測結果，系統會在後台自動檢測。", + "sshReachabilityHost": "檢測的主機：{{host}}", + "sshReachabilityLoadFailed": "無法載入可達性狀態，目前狀態未知。", + "sshReachabilityNeverProbed": "尚未檢測。", + "sshReachabilityNoSuccess": "沒有成功的檢測記錄。", + "sshReachabilityProbeNow": "立即檢測", + "sshReachabilityProbingOff": "定期檢測已關閉。這裏顯示的是上次記錄的結果，而非目前狀態。", + "sshReachabilityReasonAuth": "身份驗證失敗", + "sshReachabilityReasonConfig": "連接設定無效", + "sshReachabilityReasonHostKey": "主機密鑰與已固定的指紋不匹配", + "sshReachabilityReasonNetwork": "發生網絡錯誤", + "sshReachabilityReasonTimeout": "連接嘗試超時", + "sshReachabilityReasonUnknown": "發生未知錯誤", + "sshReachabilityStale": "已過期：最近三個檢測週期內未檢測", + "sshReachabilityStateReachable": "可達", + "sshReachabilityStateUnknown": "未知", + "sshReachabilityStateUnreachable": "不可達", + "sshReachabilityTitle": "可達性", "sshReclaimTaskDirDescription": "在 {{host}} 上執行的每個任務在結束後都會把檢出保留在 {{path}}，這樣恢復或取消歸檔的任務仍能找到原來的工作內容。", "sshReclaimTaskDirLabel": "任務被歸檔或刪除時刪除其目錄", "sshReclaimTaskDirTitle": "遠端任務目錄", diff --git a/apps/web/src/locales/zh-hk/task.json b/apps/web/src/locales/zh-hk/task.json index 84d52405bf..3ad8917c52 100644 --- a/apps/web/src/locales/zh-hk/task.json +++ b/apps/web/src/locales/zh-hk/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "任務啓動需要處理。", "launchNeedsAttention": "啓動需要處理", "launchErrorNoChanges": "任務或分支設定未更改。", + "launchWarningTitle": "SSH 主機可能無法存取", + "launchWarningHost": "已探測主機：{{host}}", + "launchWarningLastSuccess": "上次成功：{{age}}", + "launchWarningNeverSucceeded": "從未記錄過此主機的成功探測。", + "launchWarningReasonConfig": "上次嘗試失敗，因為連接設定無效。", + "launchWarningReasonTimeout": "上次嘗試超時。", + "launchWarningReasonHostKey": "上次嘗試失敗，因為主機密鑰與固定的指紋不匹配。", + "launchWarningReasonAuth": "上次嘗試的身份驗證失敗。", + "launchWarningReasonNetwork": "上次嘗試因網絡錯誤而失敗。", + "launchWarningReasonUnknown": "上次嘗試因未知原因失敗。", "launchPromptPreview": "啓動提示詞預覽", "retryWithDefaultBranch": "使用預設分支重試", "pickBaseBranch": "選擇基礎分支", diff --git a/apps/web/src/locales/zh-tw/executors.json b/apps/web/src/locales/zh-tw/executors.json index 8bf05d9bf9..dd860bace2 100644 --- a/apps/web/src/locales/zh-tw/executors.json +++ b/apps/web/src/locales/zh-tw/executors.json @@ -370,6 +370,28 @@ "sshProbeHint": "點選「{{button}}」檢查遠端主機上已安裝哪些代理程式。", "sshProxyJumpHint": "僅支援單跳堡壘。不支援鏈式跳轉。", "sshProxyJumpLabel": "ProxyJump（可選）", + "sshReachabilityAgeLastProbe": "上次檢測：{{age}}", + "sshReachabilityAgeLastSuccess": "上次成功：{{age}}", + "sshReachabilityConsecutiveFailures_one": "連續 {{count}} 次失敗", + "sshReachabilityConsecutiveFailures_other": "連續 {{count}} 次失敗", + "sshReachabilityDescription": "此主機最近一次 SSH 可達性檢測結果，系統會在後台自動檢測。", + "sshReachabilityHost": "檢測的主機：{{host}}", + "sshReachabilityLoadFailed": "無法載入可達性狀態，目前狀態未知。", + "sshReachabilityNeverProbed": "尚未檢測。", + "sshReachabilityNoSuccess": "沒有成功的檢測記錄。", + "sshReachabilityProbeNow": "立即檢測", + "sshReachabilityProbingOff": "定期檢測已關閉。這裡顯示的是上次記錄的結果，而非目前狀態。", + "sshReachabilityReasonAuth": "身分驗證失敗", + "sshReachabilityReasonConfig": "連線設定無效", + "sshReachabilityReasonHostKey": "主機金鑰與已固定的指紋不匹配", + "sshReachabilityReasonNetwork": "發生網路錯誤", + "sshReachabilityReasonTimeout": "連線嘗試超時", + "sshReachabilityReasonUnknown": "發生未知錯誤", + "sshReachabilityStale": "已過期：最近三個檢測週期內未檢測", + "sshReachabilityStateReachable": "可達", + "sshReachabilityStateUnknown": "未知", + "sshReachabilityStateUnreachable": "不可達", + "sshReachabilityTitle": "可達性", "sshReclaimTaskDirDescription": "在 {{host}} 上執行的每個任務在結束後都會把檢出保留在 {{path}}，這樣恢復或取消歸檔的任務仍能找到原來的工作內容。", "sshReclaimTaskDirLabel": "任務被歸檔或刪除時刪除其目錄", "sshReclaimTaskDirTitle": "遠端任務目錄", diff --git a/apps/web/src/locales/zh-tw/task.json b/apps/web/src/locales/zh-tw/task.json index 53fc24ffd0..1c4c4ca996 100644 --- a/apps/web/src/locales/zh-tw/task.json +++ b/apps/web/src/locales/zh-tw/task.json @@ -2245,6 +2245,16 @@ "launchErrorSessionPreview": "任務啟動需要處理。", "launchNeedsAttention": "啟動需要處理", "launchErrorNoChanges": "任務或分支設定未更改。", + "launchWarningTitle": "SSH 主機可能無法存取", + "launchWarningHost": "已探測主機：{{host}}", + "launchWarningLastSuccess": "上次成功：{{age}}", + "launchWarningNeverSucceeded": "從未記錄過此主機的成功探測。", + "launchWarningReasonConfig": "上次嘗試失敗，因為連線設定無效。", + "launchWarningReasonTimeout": "上次嘗試超時。", + "launchWarningReasonHostKey": "上次嘗試失敗，因為主機金鑰與固定的指紋不匹配。", + "launchWarningReasonAuth": "上次嘗試的身分驗證失敗。", + "launchWarningReasonNetwork": "上次嘗試因網路錯誤而失敗。", + "launchWarningReasonUnknown": "上次嘗試因未知原因失敗。", "launchPromptPreview": "啟動提示詞預覽", "retryWithDefaultBranch": "使用預設分支重試", "pickBaseBranch": "選擇基礎分支", diff --git a/docs/plans/ssh-host-reachability/plan.md b/docs/plans/ssh-host-reachability/plan.md new file mode 100644 index 0000000000..32bb3bb1f1 --- /dev/null +++ b/docs/plans/ssh-host-reachability/plan.md @@ -0,0 +1,540 @@ +--- +created: 2026-09-08 +status: draft +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 + - REQ-EXECUTORS-SSH-REACHABILITY-002 + - REQ-EXECUTORS-SSH-REACHABILITY-003 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md + - ../../specs/executors/system-design/ssh-reachability-surfaces.md +legacy_specs: [] +--- + +# Implementation Plan: SSH Host Reachability + +## Overview + +Kandev gains a continuous, cheap answer to one question per configured SSH +host — can the backend still open an authenticated SSH connection to the host +the user trusted — and puts that answer on the SSH settings page. + +The build order is bottom-up so that nothing observable exists until the thing +underneath it is proven. Wave 1 delivers two independent primitives that no +caller can reach: the single-host probe with its classification order in +`lifecycle`, and the `executor_reachability` table with its last-write-wins +upsert in the SQLite repository. Wave 2 is the poller, which is the first code +that can call either one, and which owns the hysteresis rule, the interval +configuration, and the shutdown contract. Wave 3 opens the two producers and +consumers of the record that the poller does not own: the HTTP/event +projection, and the launch path, which must never read a record. Waves 4 and 5 are the two user +surfaces, data layer first. Wave 6 proves the whole thing against a real sshd +container and documents the operator key. + +The order matters in one specific way: the launch path (task 05) is the +sharpest edge in this capability, because REQ-003 requires that a reachability +record never gate a launch. Building it after the poller means the record +already exists and is populated, so task 05's own tests can assert the negative +— that `CreateInstance` reads no record — against a store that genuinely holds +one rather than against an empty stub. + +**Two flows were cut from this package to fit the specification size budget:** +the task-card reachability indicator and the launch path's record *write*. Both +are recorded in the requirement document's `## Out of scope` with enough detail +to seed their own cards, and their acceptance-criterion IDs +(`AC-…-002.8`, `.9` and `AC-…-003.4` through `.7`) are retired, not reused. The +launch path keeps its two surviving obligations: it never gates on a record, and +its failure message names the host and the reason classified from that launch's +own attempt. + +**Reconciled against spec round 5 (2026-09-15, commit `b0481c492`).** Four +rounds of spec review closed all 15 open findings from round 4. The changes +that affect this plan and its work orders: + +- A configuration reset is now a dedicated second SQL statement, not a branch + of the observation upsert, and it carries the newly saved `host`. Ownership + of *detecting* a save lives in `internal/task/service` (`CreateExecutor` / + `UpdateExecutor`), which notifies an `ExecutorSaveObserver` the reachability + package implements — not `internal/ssh`, as the pre-round-5 plan assumed. +- `ProbeSSHHost` must bound the SSH handshake on its context, not only the TCP + dial, or a stalled handshake outlives both `probeTimeout` and `Stop`. +- A new criterion, `AC-EXECUTORS-SSH-REACHABILITY-001.29`, requires a bastion + (`ProxyJump`) host-key mismatch to classify as `host_key`, matching a + target-host mismatch. `host` stays the target in every case; a bastion-hop + failure is disambiguated in `message`, never in `host`. +- The client reconciliation key for a pushed or refetched record is + `updated_at`, not `checked_at` — a reset clears `checked_at`, so ordering on + it would make every reset look older than the record it supersedes and get + discarded. The DTO gained an `updated_at` field. +- The GET routes now have a full edge contract: `404` for a nonexistent or + soft-deleted id, `400` only for a wrong executor type (not for an + unresolvable configuration, which is a `200` carrying reason `config`), + `GET /reachability` ordered by `executor_id` ascending and including + inactive executors. +- The settings page runs no refresh timer at all while probing is disabled, + rather than reading a zero effective interval literally. +- The pre-launch warning collapsed from two mechanisms (an interactive prompt + versus a session-recorded non-interactive note) into one: the backend always + emits a `session.launch.warning` event for the launched session, and the + gateway replays the latest warning to a later subscriber while the backend + is running. A client that initiated that launch also renders the same event + inline. There is no "interactive user" branch on the backend. **The system design + attributes the producer to `internal/task/service`; that package is not in + the launch call path.** Per `apps/backend/AGENTS.md`'s Execution Flow + (Orchestrator → Lifecycle Manager → ExecutorBackend) and the verified call + site, the producer belongs in + `internal/agent/runtime/lifecycle/manager_launch.go`'s + `launchBuildExecutorRequest`, immediately before `rt.CreateInstance` at its + existing call site, using the `Manager`'s existing `eventPublisher` (the + same mechanism behind `PublishPrepareProgress`). Tasks 05 and 07 below use + this verified location; the spec's package reference needs a follow-up + correction. +- The operator configuration key's environment and YAML paths diverge: an + unparsable environment value falls back to the default, but an unparsable + `config.yaml` value refuses boot like every other typed catalog key. A + negative whole number decodes cleanly from either source and needs its own + clamp branch (`below 0` → the default). + +## Scope + +### In scope + +- A single-host SSH reachability probe and a total-ordered classification of + its failure into the closed reason set `config`, `timeout`, `host_key`, + `auth`, `network`, `unknown`. +- Persistence of exactly one reachability record per SSH executor, surviving + restart, converging under concurrent writes on the later observation, and + removed when the executor is soft-deleted. +- A bounded-concurrency poller on a configurable interval, with asymmetric + hysteresis (a threshold of consecutive failures down, one success up), an + overlap-skip rule, and a disable switch. +- HTTP and WebSocket projection of the record, an immediate-probe action that + coalesces concurrent callers, and a change-only change event. +- The launch path as a non-gating consumer: it reads no record, and its + dial-failure message names the host and the classified reason. +- The SSH executor settings surface and the pre-launch warning, localized in + all five supported locales. +- Public documentation for the one new operator configuration key. + +### Out of scope + +- Health probing for any non-SSH executor type. `HealthCheckAll` and every + other runtime's `HealthCheck` are unchanged, including + `SSHExecutor.HealthCheck`, which keeps returning `nil`. +- Repairing what the probe finds: re-resolving a moved host, re-pinning a + changed fingerprint, restarting or migrating a session off an unreachable + host. +- Gating, deferring, queueing, or re-routing a launch on a probe result. +- **Deferred (seeds its own card):** the task-card reachability indicator, and + the launch path as a probe *producer* writing `reachable` on a successful + dial and a classified failure on a failed one. Both are specified in the + requirement document's `## Out of scope`. Until the latter lands the poller + is the only writer, so a host that dies between passes lags the settings page + by up to the failure threshold times the effective interval. +- Alerting outside the product surface (push, email, webhook, chat). +- Probing remote capability on the cadence. Platform detection, `git` + presence, per-agent binary readiness, shell discovery, and the agentctl + cache check stay on the existing manual endpoints. +- Any change to the behavior of the existing `POST /api/v1/ssh/test` + endpoint, which keeps dialing unpinned against a possibly-unsaved form + configuration and keeps writing no record. +- A runtime feature toggle. The interval key's `0` value is the kill switch, + and a toggle would leave a retired identity to carry forever. +- History, trend, uptime percentage, or an incident log. One row per executor. + +## Technical approach + +### Probe primitive (`internal/agent/runtime/lifecycle`) + +A new `executor_ssh_reachability_probe.go` exports the probe and its +classification. The probe takes a resolved `*SSHTarget` and a deadline, calls +the existing `DialSSH` with `PinnedFingerprint` set, closes the client +immediately, and returns a typed outcome. It runs no remote command, so its +cost is one TCP connection plus one SSH handshake. + +Classification walks the total order `config`, `timeout`, `host_key`, `auth`, +`network`, `unknown` and takes the first match. `timeout` precedes `network` +because a deadline expiry surfaces through `dialDirect` as a dial error and +would otherwise read as a refused connection. `host_key` precedes `auth` +because both arrive as handshake failures from `ssh.NewClientConn` and only +`host_key` is a security event; the existing `errHostKeyMismatch` is the +discriminator. A bastion (`ProxyJump`) host-key rejection also classifies as +`host_key` per `AC-EXECUTORS-SSH-REACHABILITY-001.29`, even though it arrives +as `ssh: bastion dial: …` rather than `errHostKeyMismatch`, because only the +target hop is fingerprint-pinned — the reason names what failed, not which hop. +`host` always stays the target executor's host, success or failure; a +bastion-hop failure is disambiguated in `message` alone, which names the +bastion and states the failure arose on the jump hop. + +`ProbeSSHHost` bounds the handshake phase on its context, not only the TCP +dial: `ssh.NewClientConn` takes no context and `ssh.ClientConfig.Timeout` +covers only `DialSSH`'s own TCP connect, so a host that completes TCP and then +stalls the handshake would otherwise hold its goroutine past `probeTimeout` +and past `Stop`. The probe sets a deadline on the connection covering the +handshake and clears it once the transport is up. + +Resolution also validates the pin before any dial: `ResolveSSHTarget` does not +reject an empty `PinnedFingerprint`, and `buildClientConfig`'s host-key +callback accepts any key when the pin is empty, so a probe that skipped this +check would dial unpinned and record a substituted host as `reachable`. The +probe returns reason `config` for an empty pin, exactly as for an +unresolvable configuration, before any network access. + +`sshTargetFromExecutorConfig` moves out of `internal/ssh/handlers.go` and is +exported from `lifecycle` as `SSHTargetFromExecutorConfig`, so the poller and +the handlers share one projection of `executors.config` and cannot drift. The +handler delegates to it. + +### Persistence (`internal/task/models`, `internal/task/repository`) + +One new table, created through the existing idempotent +`r.migrate.Apply("executor_reachability.table", …)` mechanism in +`base_migrations.go`, following the `repository_secret_bindings` precedent, so +it is safe on SQLite, on Postgres, and on an already-migrated database: + +```sql +CREATE TABLE IF NOT EXISTS executor_reachability ( + executor_id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'unknown', + reason TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + consecutive_failures INTEGER NOT NULL DEFAULT 0, + host TEXT NOT NULL DEFAULT '', + checked_at TIMESTAMP, + last_success_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL +); +``` + +`last_success_at` is distinct from `checked_at` because the pre-launch warning +must name the age of the last *successful* probe, and `checked_at` is +overwritten by every failure. It is `NULL` until the first success and is +cleared only by a connection-configuration reset. + +A separate table rather than columns on `executors`: `executors.config` is +user-authored input and `executors.status` is a user-controlled enable switch, +so observed state must not share either row. It also keeps a write path that +fires every interval off the row that every executor read touches. + +`UpsertExecutorReachability` carries +`WHERE excluded.checked_at > executor_reachability.checked_at OR +executor_reachability.checked_at IS NULL`, and reads and writes the +consecutive-failure counter inside that same statement's transaction, so a +scheduled pass and an immediate probe racing on one executor converge on the +later observation and the streak cannot be lost to an interleaving. +`DeleteExecutor` deletes the record alongside the soft delete. + +A **second** repository method, `ResetExecutorReachability(ctx, executorID, +host string) error`, is a distinct statement, not a branch of the upsert +above: the upsert's `state` CASE has no `unknown` branch and its `WHERE` +admits only a strictly later `checked_at`, which a reset does not carry. It +writes `unknown`, a zero counter, cleared reason/message, the newly saved +`host`, and both timestamps `NULL`, guarded only by `type = 'ssh' AND +deleted_at IS NULL AND status = 'active'` — no `checked_at` or `updated_at` +pin, because a reset is ordered by the save that caused it, not by an +observation clock, and must win over whatever is stored. An executor that is +`ssh` but not `active` is *not* reset (eligibility wins per +`AC-…-001.17`), and the statement lands zero rows for it. + +### Poller (`internal/executors/reachability`, new package) + +A ticker loop started from `internal/backendapp/main.go` alongside the +integration pollers and registered with `addRuntimeCleanup`. `Start` and +`Stop` are idempotent, `Stop` drains the in-flight pass, and a `goleak` +`TestMain` guards the package — the conventions `healthpoll` established, +adopted without importing it, because `healthpoll.Prober` is shaped around one +integration with many workspaces rather than many independent hosts with +per-host results and bounded concurrency. + +One pass lists `type = 'ssh' AND deleted_at IS NULL AND status = 'active'`, +sorts ascending by `executors.id` (the primary key, never reused, so the order +is total and no tiebreak is needed), fans out over a semaphore of +`passConcurrency`, and awaits all workers. A mutex-guarded `passRunning` flag +makes an overlapping tick a no-op — dropped, not queued, because a queued tick +would compound the overload that caused the overlap and carries identical +information. + +The write path owns hysteresis: on success, `reachable` with a zero counter +from any prior state; on failure, increment, and become `unreachable` only on +the probe where the counter reaches `failureThreshold`. A sticky failure +(`config` or `host_key`) bypasses the threshold and writes `unreachable` on +that probe, because both repeat until a human changes something: the threshold +would buy no confidence, and a host-key mismatch is a possible interception +that must not wait out an interval. + +One new catalog entry, `executors.sshReachabilityIntervalSeconds` / +`KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS`, default `60`, supported +range 15 to 3600, `0` disables. Adding it requires a matching entry in +`auditedStartupEnvironmentInventory()` in `catalog_test.go`, which is +deliberately independent of `startupCatalog`, plus a new `ExecutorsConfig` +section on `config.Config` and its `SetDefault`. The other three knobs stay +compile-time constants — `probeTimeout` 10s, `passConcurrency` 4, +`failureThreshold` 2 — because no operator scenario needs them independently +and each key is a permanent contract. + +**The environment and YAML paths do not behave alike, and the clamp needs a +negative branch.** `applyBoundedIntEnv` reads the environment only and returns +early when it is unset, so a `config.yaml` value never reaches the +fallback-and-bounds logic; a non-integer YAML value instead hits +`decodeConfig`'s ordinary typed-key path and refuses boot, exactly like any +other catalog key. A negative whole number decodes cleanly from either source, +so the package's own clamp (not the catalog's) needs a `below 0` branch +yielding the default `60`, alongside `0` disables, `1`–`14` → `15`, and above +`3600` → `3600`, logged once at startup. + +Observability follows the `office_stall_*` precedent: `expvar` counters +`executor_ssh_reachability_probe_total` (by outcome), +`executor_ssh_reachability_state_transitions_total` (by destination state), +`executor_ssh_reachability_pass_skipped_total`, and +`executor_ssh_reachability_probe_duration_ms`, each also emitted as a +structured `zap` log. Transitions log at `Warn` downward and `Info` upward; +an unchanged probe does not log, so a steady host is silent. + +### API and events (`internal/ssh`, `internal/events`, `pkg/websocket`) + +Reachability is its own resource, not fields on the executor DTO, so a record +that changes every interval does not invalidate the executor list payload: + +| Route | Purpose | +| --- | --- | +| `GET /api/v1/ssh/reachability` | Every SSH executor's record in one request. | +| `GET /api/v1/ssh/executors/:id/reachability` | One executor's record. | +| `POST /api/v1/ssh/executors/:id/reachability/probe` | Probe now, persist, return the record. | + +The `POST` route coalesces through a `golang.org/x/sync/singleflight` group +keyed by executor id and the SSH connection configuration, so two overlapping +requests for one connection open one connection while a request for a changed +host or pin gets its own probe. It works while the poller is disabled. The +coalesced probe does +**not** run on any caller's request context — a disconnecting caller must +neither cancel it nor fail the other caller waiting on it — but it is also +**not** `context.Background()`: it is registered on the reachability package's +own `WaitGroup` and context, the off-cycle-probe primitive task 03 exposes, so +`Stop` still cancels and awaits it. Status codes are `404` for a nonexistent +or soft-deleted executor id, `400` when the id names a non-`ssh` executor, and +`409` for a probe request against an inactive SSH executor. `resolveSSHTarget`'s existing mapping +must **not** be reused wholesale for the probe route — that helper also maps a +config-resolution failure to `400`, which is exactly the case +`AC-…-002.5` requires to return a `200` outcome carrying reason `config` +instead. `GET` on an executor with no record returns the `unknown` shape with +every timestamp, including `updated_at`, `null`, never `404`. +`GET /reachability` orders its entries by `executor_id` ascending — the same +order the pass uses — and includes executors whose `status` is not `active`, +carrying their retained record. + +`events.ExecutorReachabilityChanged = "executor.reachability.changed"` is +published only when `state` or `reason` differs from the stored record, and is +bridged in `task_notifications.go` alongside `events.ExecutorUpdated`. +Publishing every probe result would put one message per executor per interval +on every connected client forever. The record DTO carries `updated_at` +alongside `checked_at`; it is the field a client reconciles two racing +payloads on (see Frontend below), because a reset clears `checked_at` but +still advances `updated_at`. + +**Save detection lives in `internal/task/service`, not `internal/ssh`.** +`Service.CreateExecutor` and `Service.UpdateExecutor` +(`service_resources.go`) are the only two call sites that can create a +before/after connection-configuration comparison — `internal/ssh` never sees +an executor write. Each notifies an `ExecutorSaveObserver` interface the +service declares and the `internal/executors/reachability` package +implements, wired at construction. `applyExecutorUpdates` mutates the loaded +executor in place, so `UpdateExecutor` must copy the before-values out ahead +of that call; read afterwards they compare the row against itself and the +reset never fires. On any difference in the fields Terminology names (a +create is a difference by definition), the observer calls +`ResetExecutorReachability` with the newly saved host and dispatches an +out-of-band probe through the same off-cycle primitive the immediate-probe +route uses, if the executor is still eligible. + +### Launch interaction (`internal/agent/runtime/lifecycle`) + +`SSHExecutor.CreateInstance` reads no record and no code path consults one to +decide whether to proceed. With the producer write deferred, it writes none +either, so this task adds no recorder and no dependency from `lifecycle` onto +the store. + +The launch-failure message names the target host and the reason classified from +**that launch's own** dial through `ClassifyDialError`, so a user is not +left holding an agent-generated message about a firewall when the cause was a +stale address. Kandev never intercepts or edits agent output; it adds its own +attribution alongside. + +**The pre-launch warning is one mechanism, not two.** There is no branch on +whether an interactive user is present. `Manager.launchBuildExecutorRequest` +(`manager_launch.go`), which already resolves the per-executor-type backend +and calls `rt.CreateInstance` at its single call site for every executor type, +starts a bounded asynchronous read of the target executor's reachability record +through a narrow read-only accessor immediately before that call — never inside +`CreateInstance` itself, so the negative test (`CreateInstance` reads no record) +stays meaningful. When the executor is `ssh`, the record's state is +`unreachable`, and either periodic probing is enabled or the record's `checked_at` +is within three times the *default* interval, it publishes +`session.launch.warning` for the launched session via the +`Manager`'s existing `eventPublisher` (the mechanism behind +`PublishPrepareProgress`), carrying `executor_id`, `host`, `state`, `reason`, +`last_success_at`, and a timestamp. A read failure produces no warning and +cannot delay the launch. Every launch path — WS-initiated, a +dependency chain, a workflow transition, an autostart — converges on this one +call site, so none can diverge from another. A client that initiated the +launch itself renders the same event inline at the point of initiation, in +addition to it appearing in the session-scoped event stream; there is no second +backend code path to keep in step. + +### Frontend (`apps/web`) + +`lib/types/http-ssh.ts` gains the record shape including `probing_enabled`, +which is `false` when the interval key is `0` so a surface can distinguish +"not probed yet" from "probing is off" without inferring it from an absent +timestamp, `probe_interval_seconds`, the effective interval the staleness rule +and the refresh cadence are both computed from, and `updated_at`. +`lib/api/domains/ssh-api.ts` gains the three calls. The settings slice holds +records keyed by executor id and applies the change action, reconciling a +racing refetch against a pushed event by keeping whichever payload carries the +**later `updated_at`**, not `checked_at` — a reset clears `checked_at`, so +ordering on it would discard the reset in favor of the stale record it just +invalidated. + +The settings card renders state, probed host, reason and message when +`unreachable`, the consecutive-failure count, and the age of the last probe; +it marks the result stale past three intervals, states plainly when probing is +off, and offers the probe-now action. Because a steady state is silent on the +wire, the card refetches its own record on mount and on the interval while it +is open — and runs **no timer at all** while `probing_enabled` is `false`, +fetching once on open and again after an immediate probe instead of reading a +zero effective interval as a cadence. + +**The pre-launch warning renders a session event, not a derived client +computation.** The backend always publishes `session.launch.warning` for the +launched session (see Launch interaction above); the gateway replays the latest +warning to a later subscriber while the backend is running. The session view +renders it through the existing session-scoped WS-event handler pattern +(the `executor-prepare.ts` shape), and a client that initiated that launch +also renders the same event inline at the initiation point, naming the host +and the age of the last successful probe. Neither surface derives +"unreachable" from the reachability store slice itself — both render the one +event the backend decided to raise, so there is no second code path that could +diverge from the backend's when-to-warn rule. The launch proceeds without a +confirmation step in either rendering. + +All copy goes through `t()` in `executors.json` and `tasks.json` across `en`, +`pt-pt`, `zh-cn`, `zh-hk`, and `zh-tw`, with the failure reason rendered as +translated copy rather than the raw token, and state exposed to assistive +technology as text rather than by color alone. + +## Tests + +Every acceptance criterion below is traced to the current requirements +revision. A criterion owned by two work orders appears once per side of the +split. + +| Acceptance criterion | Evidence | +| --- | --- | +| `AC-…-001.4`, `.5`, `.6`, `.7`, `.8` | `internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go` — `TestProbeSucceedsAndClosesWithoutRemoteCommand`, `TestProbeReportsConfigWithoutDialing`, `TestProbeRejectsUnpinnedFingerprintWithoutDialing`, `TestProbeRejectsChangedHostKeyWithoutRepinning`, `TestProbeDeadlineReportsTimeout`, `TestProbeAbandonsStalledHandshakeAtDeadline`, `TestFailureCarriesExactlyOneReasonAndMessage` | +| `AC-…-001.27`, `.29` | `executor_ssh_reachability_probe_test.go` — `TestClassifyDialErrorOrdering`, table-driven, one case per reason plus the deadline-before-network and host-key-before-auth orderings; `TestBastionHostKeyMismatchClassifiesAsHostKey`, `TestBastionFailureNamesTargetHostAndBastionInMessage` | +| `AC-…-001.16`, `.21`, `.22` | `internal/task/repository/sqlite/executor_reachability_test.go` — `TestSoftDeleteRemovesReachabilityRecord`, `TestUpsertKeepsLaterObservationRegardlessOfCommitOrder`, `TestEqualTimestampWriteIsDiscarded`, `TestRecordAndBothTimestampsSurviveRepositoryReopen`; Postgres parity in `executor_reachability_postgres_test.go` | +| `AC-…-001.1`, `.2`, `.3`, `.13`, `.14`, `.15`, `.17`, `.20`, `.26` | `internal/executors/reachability/poller_test.go` — `TestPassProbesEligibleExecutorsInIDOrder`, `TestPassSkipsNonSSHAndInactiveExecutors`, `TestInactiveExecutorRetainsItsRecord`, `TestOverlappingTickIsDroppedNotQueued`, `TestPassConcurrencyIsBounded`, `TestEmptyPassIsSilent`, `TestResultForNowIneligibleExecutorIsDiscarded`, `TestStopDrainsInFlightPass`, `TestStopDiscardsCancelledProbeResults`; `goleak_test.go` | +| `AC-…-001.9`, `.10`, `.11`, `.12` | `internal/executors/reachability/hysteresis_test.go` — `TestNonStickyFailureFlipsOnThresholdProbe`, `TestStickyFailureFlipsOnFirstProbe`, `TestBelowThresholdFailureStoresReasonWithoutChangingState`, `TestSingleSuccessRecoversFromUnreachable`, `TestAbsentRecordReportsUnknown`, each table-driven over probe sequences asserting the exact probe on which the state flips | +| `AC-…-001.23`, `.24`, `.25` | `internal/common/config/config_test.go` — `TestSSHReachabilityIntervalClampsOutOfRangeAndLogsOnce`, `TestAbsentNegativeOrMalformedIntervalUsesDefault`, `TestZeroIntervalDisablesPollerButNegativeDoesNot`; `internal/executors/reachability/poller_test.go` — `TestDisabledPollerRunsNoPassIncludingAtStart` | +| `AC-…-001.18`, `.19`, `.28`, `AC-…-002.7` | `internal/task/service/service_resources_executors_reachability_test.go` — `TestSavingConnectionConfigNotifiesObserverWithBeforeAndAfterFields`, `TestUnchangedSaveNotifiesNothing`, `TestCreateExecutorAlwaysNotifies`; `internal/executors/reachability/save_observer_test.go` — `TestObserverResetsRecordAndSchedulesOffCycleProbe`, `TestObserverSkipsIneligibleExecutor`; `internal/ssh/reachability_handlers_test.go` — `TestInFlightProbeResultDiscardedAfterConfigSave`, `TestImmediateProbeRunsWhilePollerDisabled`, `TestImmediateProbeRefusedForInactiveExecutor` | +| `AC-…-002.2`, `.3` | `internal/executors/reachability/publish_test.go` — `TestChangePublishesEvent`, `TestUnchangedProbePublishesNothing`, `TestResetPublishesEvent`; `internal/gateway/websocket/task_notifications_test.go` bridge assertion | +| `AC-…-002.1`, `.4`, `.5`, `.6`, `.10`, `.11`, `.12` | `internal/ssh/reachability_handlers_test.go` — `TestOverlappingImmediateProbesShareOneProbe`, `TestSharedProbeCompletesWhenStarterDisconnects`, `TestSharedProbeNotOnCallerRequestContext`, `TestGETReturnsUnknownShapeWithNullUpdatedAtForUnprobedExecutor`, `TestGETReachabilityOrderedByExecutorIDAscending`, `TestGET404ForSoftDeletedExecutor`; `apps/web/components/settings/ssh-reachability-card.test.tsx` — field coverage per state, stale past three intervals, probing-off copy with no refresh timer, load failure renders "not known", not-persisted probe result; `apps/web/lib/state/slices/settings/settings-slice.test.ts` — `applies executor.reachability.changed`, `keeps the later updated_at`; `pnpm run i18n:check` | +| `AC-…-003.1` | `internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go` — `TestCreateInstanceReadsNoReachabilityRecord`, asserting against a store fake that fails the test on any read, plus `TestLaunchProceedsAgainstUnreachableExecutor` | +| `AC-…-003.2`, `.3` | `internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go` — `TestDialFailureMessageNamesHostAndClassifiedReason`, `TestWarningPublishedForUnreachableExecutorAcrossEveryLaunchPath` (WS-initiated, dependency chain, workflow transition, autostart), `TestNoWarningWhenProbingDisabledAndRecordStale`, `TestWarningStillPublishedWhenProbingDisabledButRecordFresh`; `apps/web/lib/ws/handlers/ssh-launch-warning.test.ts` — session-stream event applied to session-runtime state; `apps/web/components/task/launch-warning.test.tsx` — inline render for the initiating client names host and last-success age, and does not block | + +## E2E tests + +The `containers` Playwright project already gates real-sshd scenarios on +`KANDEV_E2E_CONTAINERS=1` and owns the `kandev-sshd:e2e` image, so all four +flows live in `apps/web/e2e/tests/ssh/reachability.spec.ts` under that project. + +| Flow | Acceptance criteria | +| --- | --- | +| A reachable host renders the settings panel row as reachable. | `AC-…-002.1` | +| The sshd container is stopped mid-test; after the failure threshold the settings panel reports the host unreachable by name. | `AC-…-001.9`, `AC-…-002.1` | +| The container is restarted; the next probe clears the panel on a single success. | `AC-…-001.11`, `AC-…-002.1` | +| A launch started against the stopped host is attempted and fails with a message naming the host, rather than being refused. | `AC-…-003.1`, `AC-…-003.3` | + +## Work orders + +- [x] [Task 01: Single-host SSH probe and failure classification](task-01-probe-and-classification.md) +- [x] [Task 02: Reachability record persistence](task-02-reachability-persistence.md) +- [x] [Task 03: Reachability poller, hysteresis, and configuration](task-03-reachability-poller.md) +- [x] [Task 04: Reachability API, change event, and immediate probe](task-04-reachability-api-and-events.md) +- [x] [Task 05: Launch-path non-gating and failure attribution](task-05-launch-non-gating.md) +- [x] [Task 06: Reachability data layer and settings surface](task-06-settings-surface.md) +- [x] [Task 07: Pre-launch reachability warning](task-07-pre-launch-warning.md) +- [x] [Task 08: Container E2E coverage and operator documentation](task-08-e2e-and-docs.md) + +### Waves + +| Wave | Tasks | Parallel-safe | +| --- | --- | --- | +| 1 | 01, 02 | Yes — disjoint packages, only 02 carries a migration | +| 2 | 03 | No — first caller of both primitives | +| 3 | 04, 05 | Yes — disjoint files, both depend only on 03 | +| 4 | 06 | No — consumes the contract task 04 defines | +| 5 | 07 | No — reads the store slice task 06 creates | +| 6 | 08 | No — proves the assembled surfaces | + +## Verification results + +All 8 work orders are done. Each task's exact verification commands and +results are recorded in its own `## Results` section +(`task-01-probe-and-classification.md` through +`task-08-e2e-and-docs.md`). Task 08's container E2E spec +(`apps/web/e2e/tests/ssh/reachability.spec.ts`) is the plan's only +end-to-end evidence and passes against a real sshd container +(`KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --project=containers +tests/ssh/reachability.spec.ts`, 2 passed). + +## Risks + +- **The probe's classification depends on error shapes from + `golang.org/x/crypto/ssh` that are not part of that package's stable API.** + A library upgrade could turn a `host_key` failure into `unknown`. Mitigated + by keying `host_key` off Kandev's own `errHostKeyMismatch` rather than an + upstream string, and by table-driven tests over constructed errors; the + residual risk is confined to `auth` versus `network` on exotic errors, where + the fallback is `unknown` rather than a wrong security verdict. +- **A per-interval write path against SQLite adds sustained write traffic that + did not exist before.** At the default 60s interval and a realistic executor + count this is small, but an operator with many SSH executors and a short + interval could contend with the busy `task_sessions` write path. The + supported range's 15s floor bounds this; the separate table keeps the + contention off the executor row. +- **The launch path must read the record for its warning while never reading + it to decide.** `AC-…-003.2` needs the last-success age, so a read exists in + the same flow as `AC-…-003.1`'s prohibition. A future change that moves that + read into `CreateInstance` to "avoid a pointless launch" would silently + violate REQ-003 without breaking any existing test. Mitigated by keeping the + read in `Manager.launchBuildExecutorRequest`, the one call site every + executor-type dispatch already passes through, and by task 05's negative + test, which asserts against a store fake that fails on any read. +- **The system design's stated package for the warning producer + (`internal/task/service`) does not match the verified call path.** Every + launch — WS-initiated, a dependency chain, a workflow transition, an + autostart — converges in `internal/agent/runtime/lifecycle` at + `Manager.launchBuildExecutorRequest`'s single `rt.CreateInstance` call site + (confirmed against `apps/backend/AGENTS.md`'s Execution Flow and + `manager_launch.go`); `internal/task/service` is not in that path. This plan + and tasks 05/07 use the verified location. The spec still needs a follow-up + correction so a future reader isn't sent to the wrong package. +- **Deferring the launch-path write leaves the poller as the only writer.** A + host that dies between passes lags the settings page by up to the failure + threshold times the effective interval, and a failed launch — the strongest + available evidence — updates nothing. Bounded and accepted: `AC-…-003.3` + still names the host and the classified reason on the failure itself, which + is what the 2026-09-06 report actually asked for. +- **Adding a catalog entry touches a deliberately duplicated inventory.** + `auditedStartupEnvironmentInventory()` is independent of `startupCatalog` by + design; forgetting it fails `TestConfigurationCatalogMatchesAuditedEnvironmentInventory` + rather than shipping silently, so the risk is a wasted cycle, not a defect. +- **Five-locale copy gates the build.** `check-i18n-keys.mjs` fails on a + missing key, a dropped placeholder, or a value left identical to English, so + tasks 06 and 07 each carry their own translation work rather than deferring + it. +- **The `containers` E2E project needs a real Docker daemon and is slow.** + Task 08's specs cannot run on a host without Docker, so the rest of the work + package must be verifiable without them; every acceptance criterion above has + non-E2E evidence, and the four flows there re-prove end to end what unit + tests already cover against a host that can actually be stopped. diff --git a/docs/plans/ssh-host-reachability/task-01-probe-and-classification.md b/docs/plans/ssh-host-reachability/task-01-probe-and-classification.md new file mode 100644 index 0000000000..19ea961f44 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-01-probe-and-classification.md @@ -0,0 +1,205 @@ +--- +id: "01-probe-and-classification" +title: "Single-host SSH probe and failure classification" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.4 + - AC-EXECUTORS-SSH-REACHABILITY-001.5 + - AC-EXECUTORS-SSH-REACHABILITY-001.6 + - AC-EXECUTORS-SSH-REACHABILITY-001.7 + - AC-EXECUTORS-SSH-REACHABILITY-001.8 + - AC-EXECUTORS-SSH-REACHABILITY-001.27 + - AC-EXECUTORS-SSH-REACHABILITY-001.29 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md +--- + +# Task 01: Single-Host SSH Probe and Failure Classification + +## Summary + +Export from `lifecycle` a probe that opens an authenticated SSH transport to +one resolved target using its pinned fingerprint, closes it without running a +remote command, and returns a typed outcome. Classify every failure into +exactly one reason from the closed set, walking a total order so a deadline is +never reported as a network failure and a rejected host key is never reported +as an authentication failure. Nothing calls this yet. + +## In scope + +- New `executor_ssh_reachability_probe.go` exporting the probe result type, + the reason type with its six constants, `ProbeSSHHost(ctx, target, timeout)`, + and `ClassifyDialError(err)`. The name is deliberately about the *dial*, not + the probe: task 05 classifies a launch's own dial error with the same + function, and a launch is not a probe. +- The probe wraps its own `context.WithTimeout` at the supplied deadline, dials + through the existing `DialSSH` with `PinnedFingerprint` populated, and closes + the client on every returning path so an abandoned probe leaks no connection. +- **The probe bounds the SSH handshake on its context, not only the TCP + dial.** `ssh.NewClientConn` takes no context and `ssh.ClientConfig.Timeout` + covers only `DialSSH`'s own TCP connect, so a host that completes TCP and + then stalls the handshake would otherwise hold its goroutine past the + supplied deadline and past a caller's cancellation. Set a deadline on the + underlying connection covering the handshake and clear it once the + transport is up. +- Resolution validates the pin before any dial: a target that resolves with an + empty `PinnedFingerprint` classifies as `config` without dialing, exactly + like an unresolvable configuration. `ResolveSSHTarget` itself does not + reject an empty pin and `buildClientConfig`'s host-key callback accepts any + key when the pin is empty, so skipping this check would dial unpinned, + accept any key, and record a substituted host as `reachable`. +- Classification order `config`, `timeout`, `host_key`, `auth`, `network`, + `unknown`, first match wins. `host_key` keys off the existing + `errHostKeyMismatch` rather than an upstream error string. `timeout` unwraps + to `context.DeadlineExceeded` or a `net.Error` timeout. `context.Canceled` is + **not** a `timeout` and not any other reason: the outcome reports "cancelled, + no observation" so the caller discards it rather than writing a failure. A + classifier that folds both context errors together makes every graceful + shutdown look like a host outage. +- Move `sshTargetFromExecutorConfig` out of `internal/ssh/handlers.go` into + `lifecycle` as exported `SSHTargetFromExecutorConfig(map[string]string)`, + and delegate the handler to it. A missing host or a missing + `ssh_host_fingerprint` fails resolution and classifies as `config` without a + dial. +- **AC-EXECUTORS-SSH-REACHABILITY-001.29: a bastion (`ProxyJump`) host-key + rejection classifies as `host_key`, not `network` or `unknown`.** It arrives + as `ssh: bastion dial: …` rather than `errHostKeyMismatch`, because + `dialViaJump`/`bastionHostKeyCallback` verify the bastion against + `~/.ssh/known_hosts` (accept-new) rather than the pin; the reason names what + failed, not which hop. Without this an interception on the jump path waits + out `failureThreshold` under the wrong reason. `host` in the outcome always + stays the **target** executor's host, success or failure — never the + bastion — so `AC-…-001.4` and `AC-…-002.1` stay true for a probe that + succeeds through a jump. A bastion-hop failure is disambiguated in + `message` alone: it must name the bastion and state that the failure arose + on the jump hop. See the engine design's *Security* section. +- The stored message is the dial error text only. Key material and agent-socket + contents must not reach it. + +## Out of scope + +- Any persistence, poller, ticker, hysteresis, or consecutive-failure counter. +- Any caller. `SSHExecutor.HealthCheck` still returns `nil` and + `HealthCheckAll` is untouched. +- Any change to `POST /api/v1/ssh/test`, which keeps dialing unpinned so a + first-time user can observe and trust a fingerprint. +- Re-pinning, repairing, or rewriting `executors.config`. + +## Acceptance + +- A probe against a fake SSH server whose host key matches the pinned + fingerprint returns a success outcome, opens no session channel, and leaves + no open client. +- A probe whose presented host key differs from the pinned fingerprint returns + reason `host_key`, and the target's stored pinned fingerprint is unchanged + afterwards. +- A probe against a resolved target with an empty pinned fingerprint returns + reason `config` without opening a TCP connection. +- A probe whose TCP dial completes but whose handshake never responds is + abandoned at the supplied deadline rather than blocking past it. +- A probe through a bastion whose presented key does not match + `~/.ssh/known_hosts` returns reason `host_key`, `host` set to the target + (not the bastion), and a message naming the bastion and the jump hop. +- The classification table returns exactly one reason per constructed error, + including the two ordering cases: an expired deadline over a dial error, and + a host-key mismatch over an authentication failure. + +## Verification + +Start with the ordering table as a failing test — construct an error that is +both a deadline expiry and a dial failure, assert `timeout`, and confirm it +fails before the classification function exists. Then run: + +```bash +# From apps/backend: +go test -tags fts5 -race ./internal/agent/runtime/lifecycle/ -run 'Reachability|ClassifyDialError|ProbeSSHHost' +go test -tags fts5 -race ./internal/ssh/ -run 'SSHTarget|ResolveSSHTarget' +make lint +``` + +## Files likely touched + +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe.go` +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_probe_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go` +- `apps/backend/internal/ssh/handlers.go` +- `apps/backend/internal/ssh/handlers_test.go` + +## Dependencies + +None. + +## Risks + +- The `lifecycle` package is guarded by a `goleak` `TestMain`; a probe that + closes its client on the success path but not on the classification path + will surface there rather than in the probe's own assertions. Close in a + `defer` on the dial's returning path, not at each `return`. +- `golang.org/x/crypto/ssh` error shapes are not a stable API. Confining the + `host_key` discriminator to Kandev's own `errHostKeyMismatch` keeps a library + upgrade from turning a security event into `auth`. +- The handshake-deadline fix is the one place this task can silently regress + into the bug it closes: setting the deadline only around the TCP dial (the + pre-round-5 shape) leaves a stalled handshake unbounded. Test it with a fake + server that completes TCP and then never writes, asserting the probe returns + at the deadline rather than hanging — a `t.Fatal` on timeout in the test + itself, not just an assertion after the fact. + +## Verify against source before starting + +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go`: + confirm `dialViaJump` and `bastionHostKeyCallback` still verify the bastion + against `~/.ssh/known_hosts` rather than the pin (this task's `host_key` + bastion branch depends on that split), and confirm `ssh.NewClientConn`'s + call site to plan exactly where a connection deadline covers the handshake. + +## Parallelism + +`parallel-safe` with task 02 — disjoint packages, no shared schema or +generated contract. + +## Inputs + +- System design, sections *The probe* and *Security*. +- `executor_ssh_connection.go` for `SSHTarget`, `DialSSH`, `dialDirect`, + `buildClientConfig`, and `errHostKeyMismatch`. +- `ssh_fake_server_test.go` for the in-process SSH server harness. +- `internal/ssh/handlers.go` for the projection being lifted. + +## Results + +Implemented. `executor_ssh_reachability_probe.go` adds `SSHReachabilityReason` +(6 constants), `SSHProbeOutcome` (`Success`, `Cancelled`, `Host`, `Reason`, +`Message`), `ProbeSSHHost`, `ClassifyDialError`, and +`SSHTargetFromExecutorConfig`; `internal/ssh/handlers.go`'s +`resolveSSHTarget` now delegates to it and the old unexported projection was +deleted. The handshake-timeout-bound fix landed as a shared +`handshakeWithDeadline` helper used by both `dialDirect` and `dialViaJump`'s +final hop, so the bastion hop (already routed through `dialDirect`) and the +tunneled final hop are both bounded, not just the plain direct-dial path. + +Tests added in `executor_ssh_reachability_probe_test.go`: the classification +table (including both required ordering cases), probe success (no session +channel opened), host-key mismatch (pin unchanged), empty-pin config +short-circuit (no TCP dial attempted), a stalled-handshake fake listener +proving the probe returns at the deadline instead of hanging, the +AC-…-001.29 bastion host-key mismatch case (`host_key`, `Host` stays the +target, message names the bastion), a cancelled-context case, and +`SSHTargetFromExecutorConfig` resolution/error cases. + +Verification: +``` +go test -tags fts5 -race ./internal/agent/runtime/lifecycle/ -run 'Reachability|ClassifyDialError|ProbeSSHHost' # PASS +go test -tags fts5 -race ./internal/ssh/... # PASS +golangci-lint run ./internal/agent/runtime/lifecycle/... ./internal/ssh/... --new-from-rev=845546151 # 0 issues +``` +The full `./internal/agent/runtime/lifecycle/...` suite has 9 pre-existing +failures unrelated to this change (Kubernetes retained-workspace tests, +worktree rollback tests, and a unix-socket-path-length test) — reproduced +identically on the pre-Task-01 commit via `git stash`, so not a regression +from this work. diff --git a/docs/plans/ssh-host-reachability/task-02-reachability-persistence.md b/docs/plans/ssh-host-reachability/task-02-reachability-persistence.md new file mode 100644 index 0000000000..9e5061d5f7 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-02-reachability-persistence.md @@ -0,0 +1,238 @@ +--- +id: "02-reachability-persistence" +title: "Reachability record persistence" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.16 + - AC-EXECUTORS-SSH-REACHABILITY-001.21 + - AC-EXECUTORS-SSH-REACHABILITY-001.22 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md +--- + +# Task 02: Reachability Record Persistence + +## Summary + +Add the `executor_reachability` table, its model, and its repository methods. +Exactly one row per SSH executor, surviving restart, converging on the later +observation when two writes race, and deleted when its executor is +soft-deleted. No probe writes to it yet. + +## In scope + +- `models.ExecutorReachability` with the state and reason constants, plus a + `ListSSHExecutorsForReachability` projection returning eligible executors + (`type = 'ssh'`, `deleted_at IS NULL`, `status = 'active'`) ordered ascending + by `executors.id`. +- A single additive `r.migrate.Apply("executor_reachability.table", …)` in + `base_migrations.go`, following the `repository_secret_bindings` precedent so + it is idempotent on SQLite, on Postgres, and on an already-migrated database. +- `GetExecutorReachability`, `ListExecutorReachability`, + `UpsertExecutorReachability`, `DeleteExecutorReachability` on + `ExecutorRepository` and its SQLite implementation. +- Columns per the system design, including `last_success_at` alongside + `checked_at`: the pre-launch warning needs the age of the last *successful* + probe, and `checked_at` is overwritten by every failure. `updated_at + TIMESTAMP NOT NULL` is a distinct column from `checked_at`: it advances on + every write including a reset (whose `checked_at` is `NULL`), and is the + field task 04's client reconciliation depends on. +- Last-write-wins on `checked_at`: the upsert's conflict clause writes only + when the incoming `checked_at` is strictly later than the stored one, or the + stored one is `NULL`. A write whose `checked_at` equals the stored value is + discarded, and timestamps are persisted at millisecond resolution or finer so + two probes in the same second still order. The consecutive-failure counter is + read and written inside that same statement's transaction so a streak cannot + be lost to an interleaving. +- **A second, distinct method, `ResetExecutorReachability(ctx, executorID, + host string) error`**, is its own SQL statement, not a branch of the upsert: + the upsert's `state` CASE has no `unknown` branch and its `WHERE` admits only + a strictly later `checked_at`, which a reset does not carry. It writes + `unknown`, a zero counter, cleared reason/message, the newly saved `host`, + and both `checked_at`/`last_success_at` set `NULL`, guarded only by `type = + 'ssh' AND deleted_at IS NULL AND status = 'active'` — no `checked_at` or + `updated_at` pin, because a reset is ordered by the save that caused it, not + by an observation clock, and must win over whatever is stored. An executor + that is `ssh` but not `active` is *not* reset (eligibility wins per + `AC-…-001.17`), and the statement lands zero rows for it rather than + erroring. This task owns only the statement; task 04 owns who calls it. +- `DeleteExecutor` removes the reachability row alongside the soft delete. + +## Out of scope + +- The poller, the ticker, the hysteresis rule, and the decision of *what* + counter value or state to write. This task stores what it is given. +- Any HTTP route, event, or DTO. +- Columns on `executors`. Observed state must not share the row that holds + user-authored `config` and the user-controlled `status` switch. +- Retention beyond one row per executor: no history, trend, or incident log. + +## Acceptance + +- Two upserts for one executor carrying an older and a newer `checked_at` + leave the newer values stored regardless of which transaction commits second, + including when the older one lands on a whole second and the newer one carries + a fraction of that same second. +- Soft-deleting an executor removes its reachability row, and re-reading it + returns a not-found rather than a stale record. +- A record written, then read back through a reopened repository against the + same database file, is byte-identical including both `checked_at` and + `last_success_at`. +- An upsert whose `checked_at` equals the stored value leaves the row + unchanged. +- `ResetExecutorReachability` on a record with a populated `checked_at`, + `last_success_at`, and non-zero counter leaves `state = unknown`, the + counter zeroed, reason/message cleared, `host` set to the new value, and + both timestamps `NULL`; `updated_at` still advances even though `checked_at` + does not. +- `ResetExecutorReachability` against an executor whose `status` is not + `active` (but is still `type = 'ssh'` and not soft-deleted) affects zero + rows and leaves the existing record untouched. + +## Verification + +Start with the concurrency test as a failing test — issue the newer write +first and the older write second, assert the newer survives, and confirm a +plain `ON CONFLICT DO UPDATE` without the timestamp predicate fails it. Then +run: + +```bash +# From apps/backend: +go test -tags fts5 -race ./internal/task/repository/sqlite/ -run 'ExecutorReachability|ResetExecutorReachability' +go test -tags fts5 -race ./internal/task/repository/... -run 'Migration|Schema' +make lint +``` + +Postgres parity runs through the existing `*_postgres_test.go` harness in the +same package. + +## Files likely touched + +- `apps/backend/internal/task/models/executor_reachability.go` +- `apps/backend/internal/task/repository/interface.go` +- `apps/backend/internal/task/repository/sqlite/executor_reachability.go` +- `apps/backend/internal/task/repository/sqlite/executor_reachability_test.go` +- `apps/backend/internal/task/repository/sqlite/executor_reachability_postgres_test.go` +- `apps/backend/internal/task/repository/sqlite/base_migrations.go` +- `apps/backend/internal/task/repository/sqlite/executor.go` + +## Dependencies + +None. + +## Risks + +- `MigrateLogger.Apply` swallows non-"already exists" errors, so a malformed + `CREATE TABLE` fails silently at migration time and only surfaces as a + "no such table" at first write. Assert the table's presence in a migration + test rather than trusting the apply to be loud. +- `TIMESTAMP` comparison semantics differ between SQLite and Postgres, and the + difference is not symmetric. Postgres compares its `timestamp` type + temporally; SQLite stores the driver's text encoding in a NUMERIC-affinity + column and compares it lexically. Write UTC and bind `time.Time` rather than a + pre-formatted string, per the design's *Persistence* section — an RFC3339 `Z` + string inverts the comparison for any whole-second value. A Postgres-only + assertion passes while SQLite keeps the wrong row, so the ordering test must + run on both, with a sub-second pair and a whole-second/sub-second pair. +- Adding methods to `ExecutorRepository` widens an interface with existing + fakes across the codebase; expect compile breaks in unrelated test packages + and extend those fakes rather than narrowing the interface. + +## Parallelism + +`parallel-safe` with task 01 — disjoint packages, and task 01 carries no +schema change. + +## Inputs + +- System design, section *Persistence*. +- `base_migrations.go` `repository_secret_bindings` block as the + create-table-through-migration precedent. +- `executor.go` for the existing executor CRUD and soft-delete shape. +- `models.ExecutorTypeSSH` and `models.ExecutorStatusActive`. + +## Results + +Implemented as specified: + +- `models.ExecutorReachability` (`internal/task/models/executor_reachability.go`) + with `ExecutorReachabilityState`/`ExecutorReachabilityReason` constants, + `ErrExecutorReachabilityNotFound`, and `ExecutorReachabilityObservation` (the + upsert's input struct, carrying `InitialState`/`InitialFailures` for the + insert-only path plus `SeenUpdatedAt`/`FailureThreshold` for the eligibility + and hysteresis guards). +- `executor_reachability` table added via one additive + `r.migrate.Apply("executor_reachability.table", …)` in `base_migrations.go`, + following the `repository_secret_bindings` precedent (idempotent + `CREATE TABLE IF NOT EXISTS`, no separate `init*Schema` entry needed for a + brand-new table). +- `ListSSHExecutorsForReachability`, `GetExecutorReachability`, + `ListExecutorReachability`, `UpsertExecutorReachability`, + `ResetExecutorReachability`, `DeleteExecutorReachability` added to + `ExecutorRepository` (`repository/interface.go`) and implemented in the new + `repository/sqlite/executor_reachability.go`. +- `UpsertExecutorReachability` is a single `INSERT … SELECT … WHERE EXISTS … + ON CONFLICT DO UPDATE` statement per the system design: the `WHERE EXISTS` + guard checks the executor is still active SSH with the caller's observed + `updated_at`; the conflict clause derives `consecutive_failures` and `state` + from the stored row via `CASE`, and the final `WHERE` on `checked_at` + enforces last-write-wins (a `NULL` stored `checked_at` or a strictly later + incoming one wins; an equal or older one is a no-op). `InitialState`/ + `InitialFailures` are used only by the `INSERT` branch (no prior row); every + later write derives both from the stored row. +- `ResetExecutorReachability` is its own statement (not a branch of the + upsert): unconditionally writes `unknown`/zeroed counter/cleared reason and + message/new `host`/`NULL` timestamps, guarded only by + `type='ssh' AND deleted_at IS NULL AND status='active'` — no timestamp pin, + since a reset must win over whatever is stored regardless of observation + order. A non-active-but-not-deleted SSH executor is a no-op (`AC-…-001.17`). +- `DeleteExecutor` (`repository/sqlite/executor.go`) now runs the soft-delete + `UPDATE` and `DELETE FROM executor_reachability` in one transaction. +- Both writes normalize to UTC and bind `time.Time` directly (never a + pre-formatted string), per the design's SQLite-lexical-vs-Postgres-temporal + `TIMESTAMP` comparison risk. +- Widening `ExecutorRepository` broke one pre-existing test fake + (`mockRepository` in `internal/task/handlers/process_handlers_test.go`, which + hand-implements methods rather than embedding the interface); extended it + with the six new methods rather than narrowing the interface, per the task's + own Risks section. + +Tests added (`repository/sqlite/executor_reachability_test.go`, +`executor_reachability_postgres_test.go`): + +- `TestUpsertExecutorReachabilityNewerCheckedAtWinsRegardlessOfOrder` — issues + the newer write first and the older write second (including a + whole-second/sub-second pair), asserts the newer values survive. +- `TestUpsertExecutorReachabilityEqualCheckedAtIsDiscarded` +- `TestExecutorReachabilityRoundTripAcrossReopen` — write, reopen the SQLite + file, read back byte-identical `checked_at`/`last_success_at`. +- `TestDeleteExecutorRemovesReachabilityRecord` +- `TestResetExecutorReachabilityClearsTheRecordButAdvancesUpdatedAt` +- `TestResetExecutorReachabilityIsANoOpForANonActiveExecutor` +- `TestListSSHExecutorsForReachabilityOrdersByID` +- `TestPostgresExecutorReachabilityCheckedAtOrdering` (env-gated on + `KANDEV_TEST_POSTGRES_DSN`; skipped in this environment, proves the fix + doesn't regress on Postgres — the SQLite test above is the one that actually + catches the lexical-comparison bug). + +Verification commands run from `apps/backend`, all green: + +``` +go test -tags fts5 -race ./internal/task/repository/sqlite/ -run 'Reachability' # 7 passed, 1 skipped (no Postgres DSN) +go test -tags fts5 -race ./internal/task/repository/... -run 'Migration|Schema' # all passed, several skipped (no Postgres DSN) +make lint # 0 issues +go build ./... && go vet ./... # clean +``` + +`go test -race ./internal/task/...` also run as a broader sanity check: one +unrelated pre-existing failure +(`TestRepositoryBranchPolicyServiceGitflowStarterIsAtomicAndOneTime`, a +`/tmp` vs `/private/tmp` path-canonicalization issue on this macOS +environment) reproduces identically on the task-01 commit before any task-02 +change, confirmed via a throwaway git worktree at `7ca5d4b0e`. Not caused by +this work order. diff --git a/docs/plans/ssh-host-reachability/task-03-reachability-poller.md b/docs/plans/ssh-host-reachability/task-03-reachability-poller.md new file mode 100644 index 0000000000..4494e523d3 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-03-reachability-poller.md @@ -0,0 +1,330 @@ +--- +id: "03-reachability-poller" +title: "Reachability poller, hysteresis, and configuration" +status: done +wave: 2 +depends_on: ["01-probe-and-classification", "02-reachability-persistence"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.1 + - AC-EXECUTORS-SSH-REACHABILITY-001.2 + - AC-EXECUTORS-SSH-REACHABILITY-001.3 + - AC-EXECUTORS-SSH-REACHABILITY-001.4 + - AC-EXECUTORS-SSH-REACHABILITY-001.5 + - AC-EXECUTORS-SSH-REACHABILITY-001.7 + - AC-EXECUTORS-SSH-REACHABILITY-001.9 + - AC-EXECUTORS-SSH-REACHABILITY-001.10 + - AC-EXECUTORS-SSH-REACHABILITY-001.11 + - AC-EXECUTORS-SSH-REACHABILITY-001.12 + - AC-EXECUTORS-SSH-REACHABILITY-001.13 + - AC-EXECUTORS-SSH-REACHABILITY-001.14 + - AC-EXECUTORS-SSH-REACHABILITY-001.15 + - AC-EXECUTORS-SSH-REACHABILITY-001.16 + - AC-EXECUTORS-SSH-REACHABILITY-001.17 + - AC-EXECUTORS-SSH-REACHABILITY-001.20 + - AC-EXECUTORS-SSH-REACHABILITY-001.23 + - AC-EXECUTORS-SSH-REACHABILITY-001.24 + - AC-EXECUTORS-SSH-REACHABILITY-001.25 + - AC-EXECUTORS-SSH-REACHABILITY-001.26 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md +--- + +# Task 03: Reachability Poller, Hysteresis, and Configuration + +## Summary + +Add `internal/executors/reachability`: the ticker that sweeps every eligible +SSH executor on a configurable interval with bounded concurrency, the write +path that owns asymmetric hysteresis, the operator configuration key with its +clamp and disable behavior, and the metrics. This is the first code that calls +the probe from task 01 and the store from task 02. + +## In scope + +- A probe cancelled by `Stop` is discarded, never written: a graceful shutdown + must not leave failure records behind. Test it by cancelling mid-pass and + asserting the stored records are byte-identical to before. +- The poller: an immediate pass on `Start`, then a pass per tick. `Start` and + `Stop` idempotent, `Stop` draining the in-flight pass, a `goleak` `TestMain` + guarding the package — `healthpoll`'s conventions adopted without importing + it. +- One pass: list eligible executors, dispatch in ascending `executors.id` + order over a semaphore of `passConcurrency`, await all workers. A + mutex-guarded `passRunning` flag drops an overlapping tick rather than + queueing it, and increments a skipped counter. +- The write path owning hysteresis: success writes `reachable` with a zero + counter from any prior state, an empty reason and message, and + `last_success_at` set to that probe's completion timestamp; a failure leaves + `last_success_at` untouched and keeps its reason readable until a later + probe succeeds, so a reason is empty only when the most recent completed + probe succeeded or none has; failure increments the counter and writes + `unreachable` only on the probe where it reaches `failureThreshold`. + A sticky failure (`config` or `host_key`) writes `unreachable` on that probe + regardless of the counter: both repeat until a human changes something, so + the threshold would buy no confidence and would delay a security event. +- A record that does not exist reports `unknown`; the state is never inferred. +- Compile-time constants `probeTimeout` 10s, `passConcurrency` 4, + `failureThreshold` 2, each with the rationale recorded in the design. +- Catalog entry `executors.sshReachabilityIntervalSeconds` / + `KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS`, default `60`, supported + range 15 to 3600. Only `0` disables. Absent, empty, negative, fractional or + unparsable falls back to the **default**, never to a bound, so a typo cannot + silently switch monitoring off; `1`-`14` clamps up to `15` and above `3600` + clamps down, logged once at startup. The catalog's minimum must stay `0`, or + `0` falls below it and the kill switch becomes a 60-second cadence. Requires a matching entry in + `auditedStartupEnvironmentInventory()` in `catalog_test.go`, a new + `ExecutorsConfig` section on `config.Config`, and its `SetDefault`. +- **The environment and YAML paths do not behave alike, and the clamp needs a + negative branch.** `applyBoundedIntEnv` reads the environment only and + returns early when it is unset, so a `config.yaml` value never reaches the + fallback-and-bounds logic; a non-integer YAML value instead hits + `decodeConfig`'s ordinary typed-key path and refuses boot, exactly like any + other catalog key. A negative whole number decodes cleanly from either + source, so the package's own clamp (not the catalog's) needs a `below 0` + branch yielding the default `60`, alongside `0` disables, `1`-`14` → `15`, + and above `3600` → `3600`, logged once at startup. +- **An internal off-cycle-probe dispatch primitive**, registered on the + package's own `WaitGroup`/context (not a caller's), that runs one probe for + a single executor outside the ticker and persists its result through the + same write path a scheduled pass uses. `Stop` cancels and awaits it like any + other in-flight probe. This is the only piece task 04's immediate-probe + route and its save-triggered reset consume from this package; neither task + 04 concern reimplements dispatch or persistence. +- `executor_ssh_reachability_probe_discarded_total`, incremented when `Stop` + cancels a probe and its result is dropped: a cancelled probe has no outcome, + so `probe_total` cannot represent it. +- `executor_ssh_reachability_write_refused_total`, incremented when the write + path's own guard (ineligible executor, stale `checked_at`) drops a write + rather than the probe itself failing: a rising value means configuration + churn, not a fault. +- `executor_ssh_reachability_reset_total`, incremented by + `ResetExecutorReachability` succeeding with at least one row affected — task + 04's save observer is the only caller, but the counter lives with the other + reachability metrics. +- `expvar` counters `executor_ssh_reachability_probe_total` (by outcome), + `…_state_transitions_total` (by destination state), `…_pass_skipped_total`, + `…_probe_duration_ms`, each also a structured `zap` log. Transitions log at + `Warn` downward and `Info` upward; an unchanged probe does not log. +- Wiring in `internal/backendapp/main.go` alongside the integration pollers, + with `addRuntimeCleanup` calling `Stop`. + +## Out of scope + +- Publishing the change event and the HTTP routes. Task 04 owns both; this + task's write path exposes the change signal as a return value that task 04 + consumes. +- Calling the off-cycle-probe primitive: the immediate-probe HTTP route and + the `ExecutorSaveObserver`/reset trigger both live in task 04, which only + consumes the primitive this task exposes. +- Any launch-path interaction. +- A runtime feature toggle. The interval key's `0` is the kill switch, and a + toggle would leave a retired identity to carry forever. +- Retry inside a pass. The next tick is the retry. + +## Acceptance + +- A pass probes every eligible SSH executor exactly once in ascending + `executors.id` order, probes no other executor type and no inactive + executor, and never has more than `passConcurrency` probes in flight. +- A sequence of non-sticky failures leaves the state `reachable` until the + probe on which the counter reaches `failureThreshold`, which writes + `unreachable`; the very next success writes `reachable` with a zero counter + without a confirming probe. +- A single `config` or `host_key` failure writes `unreachable` on that first + probe, with the counter still below `failureThreshold`. +- A below-threshold failure stores the new reason and counter while leaving the + state exactly as it was, so a reason is reported before a state change. +- An interval of `0` runs no scheduled pass; a negative or unparsable interval + starts the poller at the 60s default rather than disabling it; an interval + outside 15 to 3600 starts clamped with one log line rather than refusing to + boot; and `Stop` leaves no goroutine behind. +- The off-cycle-probe primitive runs a probe for one executor outside the + ticker, persists through the same write path, and is included in whatever + `Stop` cancels and awaits — a call issued just before `Stop` either completes + or is cancelled cleanly, never leaking. +- A YAML `config.yaml` value for the interval key that is not a whole number + refuses boot like any other catalog key; a negative whole number from either + YAML or the environment falls back to the 60s default rather than the + nearest bound. + +## Verification + +Start with the hysteresis table as a failing test — drive a +success/fail/fail/success sequence and assert the exact probe index on which +each direction flips — and confirm a naive flip-on-first-result implementation +fails it. Then run: + +```bash +# From apps/backend: +go test -tags fts5 -race ./internal/executors/reachability/... +go test -tags fts5 -race ./internal/common/config/ -run 'Catalog|SSHReachability' +go test -tags fts5 -race ./internal/backendapp/ -run 'Wiring|Cleanup|Poller' +make lint +``` + +Use `testing/synctest` for the ticker, matching `healthpoll_test.go`, so no +test sleeps on wall-clock time. + +## Files likely touched + +- `apps/backend/internal/executors/reachability/poller.go` +- `apps/backend/internal/executors/reachability/store.go` +- `apps/backend/internal/executors/reachability/metrics.go` +- `apps/backend/internal/executors/reachability/off_cycle_probe.go` +- `apps/backend/internal/executors/reachability/poller_test.go` +- `apps/backend/internal/executors/reachability/hysteresis_test.go` +- `apps/backend/internal/executors/reachability/goleak_test.go` +- `apps/backend/internal/common/config/config.go` +- `apps/backend/internal/common/config/catalog.go` +- `apps/backend/internal/common/config/catalog_test.go` +- `apps/backend/internal/backendapp/main.go` + +## Dependencies + +Tasks 01 and 02. The probe and the store must both exist and be tested before +anything sweeps hosts on a timer. + +## Risks + +- `auditedStartupEnvironmentInventory()` is deliberately independent of + `startupCatalog`; updating only one fails + `TestConfigurationCatalogMatchesAuditedEnvironmentInventory`. This costs a + cycle rather than shipping a defect, but it is the most likely first failure. +- A `goleak` `TestMain` on a package whose poller holds a semaphore will report + a leak for a pass abandoned by a cancelled context. `Stop` must drain, not + just cancel, or the guard turns into a flake in CI before it turns into a + local failure. +- Dropping an overlapping tick is correct but silent by design. Without the + skipped-pass counter, an interval shorter than a pass duration is + indistinguishable from a healthy poller, so the counter is not optional + instrumentation. + +## Parallelism + +`sequential` + +## Inputs + +- System design, sections *The poller*, *State transitions*, *Configuration*, + *Failure and recovery*, and *Observability*. +- `internal/integrations/healthpoll/healthpoll.go` and its `goleak_test.go` + for the lifecycle conventions. +- `internal/backendapp/main.go` around the Linear and Sentry poller wiring. +- `internal/common/config/catalog.go` and `catalog_test.go`. + +## Results + +Implemented `internal/executors/reachability` (`interval.go`, `store.go`, +`metrics.go`, `probe.go`, `poller.go`, plus `goleak_test.go`, +`interval_test.go`, `store_test.go`, `poller_test.go`), the +`executors.sshReachabilityIntervalSeconds` catalog key, and the +`internal/backendapp` wiring, all under strict TDD (RED confirmed before each +GREEN). + +**`interval.go`**: compile-time constants `probeTimeout` (10s), +`passConcurrency` (4), `failureThreshold` (2), `DefaultIntervalSeconds` (60), +`MinIntervalSeconds`/`MaxIntervalSeconds` (15/3600), and `ClampInterval`, which +owns the negative/0/below-min/above-max normalization independent of the +catalog layer. `TestClampInterval` covers all nine cases. + +**`store.go`**: the narrow `Repository` interface (the four reachability +methods this package needs, not the full `ExecutorRepository`), and `store`, +whose `Observe` brackets Task 02's `UpsertExecutorReachability` with +before/after `GetExecutorReachability` reads to detect a refused write +(`checked_at` didn't move) and a state transition, without changing Task 02's +already-committed write signature. `buildObservation` derives +`InitialState`/`InitialFailures` for the insert-only (never-before-seen row) +path, mirroring the same "sticky reason always unreachable, transient failure +never promotes below threshold" rule the SQL enforces for an existing row. +`TestStoreObserveHysteresisSequence` drives a real SQLite-backed +success/fail/fail/success sequence and asserts the exact probe index each +direction flips on (a naive flip-on-first-result implementation fails it); +`TestStoreObserveStickyFailureIsUnreachableOnFirstProbe` covers `config` and +`host_key` landing `unreachable` on the very first probe, counter still below +threshold. + +**`probe.go`**: `defaultProbe` resolves an executor's SSH target via +`lifecycle.SSHTargetFromExecutorConfig` and probes it with +`lifecycle.ProbeSSHHost(ctx, target, probeTimeout)`; a resolution failure is +reported as `SSHReachabilityReasonConfig` without dialing. + +**`poller.go`**: `Poller` with idempotent `Start`/`Stop`, a package-owned +`WaitGroup`/context so `Stop` cancels and awaits every in-flight unit of work +— the scheduled loop, the pass it dispatched, and any off-cycle `ProbeNow` +call — gated through a race-safe `acquire()` that registers new work on the +`WaitGroup` under the same mutex `Stop` locks, so a new registration can never +race a concurrent `wg.Wait()`. An interval of `0` starts the poller (so +`ProbeNow` still works) but never spawns the scheduled loop at all. Each tick +is dispatched via `dispatchPass`, a single-flight gate (`passRunning` + +mutex) that drops and counts (`pass_skipped_total`) an overlapping tick +rather than queuing it — this required dispatching each pass on its own +goroutine/WaitGroup unit rather than blocking the ticker-consuming loop +directly, so a tick firing mid-pass is actually observable rather than being +serialized away by the loop's own single-threadedness. `runPass` lists +eligible executors (already ascending by `executors.id` per Task 02's query), +fans out over a `passConcurrency`-slot semaphore, and abandons dispatching +(without abandoning in-flight probes) on cancellation. `probeAndPersist` +discards a `Cancelled` outcome (`probe_discarded_total`) instead of writing +it, satisfying "`Stop` leaves no failure record behind." + +**`metrics.go`**: `expvar` counters/maps for all six named metrics +(`probe_total`, `state_transitions_total`, `pass_skipped_total`, +`write_refused_total`, `reset_total`, `probe_discarded_total`, +`probe_duration_ms`), each paired with a structured zap log where the design +calls for one. `RecordReset` is exported (unused within this package, since +Task 04's save observer is the only caller of `ResetExecutorReachability`) so +the counter lives with the rest of this package's metrics rather than being +declared dead code. + +**`poller_test.go`**: `testing/synctest`-based coverage (a fake `probeFunc` +and an in-memory fake `Repository`, no real SSH/network/DB) for: an immediate +pass on `Start`; `Start` idempotency; `Stop` before `Start` as a no-op; an +interval of `0` running no scheduled pass while `ProbeNow` still works; +`ProbeNow` refused after `Stop`; the `passConcurrency` bound (verified exactly +met, not just not-exceeded, with more executors than slots); an overlapping +tick being skipped and counted rather than queued; `Stop` discarding a +cancelled in-flight probe's result entirely; and a listing failure abandoning +the pass without writing. Stable across `-race -count=20`. + +**Config catalog** (`catalog.go`, `source.go`, `config.go`, +`catalog_test.go`): added `executors.sshReachabilityIntervalSeconds` / +`KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS` (default `60`) and the +`ExecutorsConfig` section. Confirmed the environment/YAML split the risk +called out: `applyNonNegativeIntEnv` (the existing generic helper, no new +bespoke function needed) already gives the right ENV-path behavior — absent, +empty, unparsable, or negative falls back to 60; `0` and any other +non-negative integer pass through **unclamped** — because the 15-3600 +bound-clamping is deliberately `ClampInterval`'s job alone, applied once in +`Poller.New`. A YAML value decodes straight into the typed int field +(verified a YAML `-5` reaches the config struct unclamped, confirming +`ClampInterval` is the only place that normalizes it) and a non-integer YAML +value refuses boot via the ordinary typed-key decode path, with no extra code +required. Added the matching `auditedStartupEnvironmentInventory()` entry in +the same change as the catalog entry, per the plan's flagged risk. + +**`internal/backendapp` wiring**: extracted `startSSHReachabilityPoller` +(`ssh_reachability_wiring.go`) rather than inlining construction in +`startAgentInfrastructure`, mirroring the existing `startTaskUsageWriter` +pattern — an AST-based test (`ssh_reachability_wiring_test.go`) asserts the +composition root calls it exactly once, and a direct test proves it starts +the poller, registers exactly one `Stop` cleanup, and that the poller +actually runs its immediate pass (a fake repository signals on a channel from +inside `ListSSHExecutorsForReachability`). Wired alongside the Jira/Linear/ +Sentry/WorkflowSync/Office-config-sync poller block using `repos.Task` (which +structurally satisfies the package's narrow `Repository` interface with no +adapter) and `cfg.Executors.SSHReachabilityIntervalSeconds`. + +Verification, all green: + +```bash +go test -tags fts5 -race ./internal/executors/reachability/... +go test -tags fts5 -race ./internal/common/config/ -run 'Catalog|SSHReachability' +go test -tags fts5 -race ./internal/backendapp/ -run 'Wiring|Cleanup|Poller' +make lint +``` + +Also ran `go build ./...` and `go vet ./...` across the whole repo (clean) +after the config/backendapp changes. diff --git a/docs/plans/ssh-host-reachability/task-04-reachability-api-and-events.md b/docs/plans/ssh-host-reachability/task-04-reachability-api-and-events.md new file mode 100644 index 0000000000..8cd0e69651 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-04-reachability-api-and-events.md @@ -0,0 +1,309 @@ +--- +id: "04-reachability-api-and-events" +title: "Reachability API, change event, and immediate probe" +status: done +wave: 3 +depends_on: ["03-reachability-poller"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 + - REQ-EXECUTORS-SSH-REACHABILITY-002 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.14 + - AC-EXECUTORS-SSH-REACHABILITY-001.18 + - AC-EXECUTORS-SSH-REACHABILITY-001.19 + - AC-EXECUTORS-SSH-REACHABILITY-001.26 + - AC-EXECUTORS-SSH-REACHABILITY-001.28 + - AC-EXECUTORS-SSH-REACHABILITY-002.2 + - AC-EXECUTORS-SSH-REACHABILITY-002.3 + - AC-EXECUTORS-SSH-REACHABILITY-002.5 + - AC-EXECUTORS-SSH-REACHABILITY-002.6 + - AC-EXECUTORS-SSH-REACHABILITY-002.7 +system_design: + - ../../specs/executors/system-design/ssh-reachability-surfaces.md + - ../../specs/executors/system-design/ssh-reachability.md +--- + +# Task 04: Reachability API, Change Event, and Immediate Probe + +## Summary + +Project the reachability record over HTTP as its own resource, publish a +change event only when the state or reason actually changes, and add an +immediate-probe action that coalesces concurrent callers into one connection. +Reset an executor's record when its connection configuration is saved. + +## In scope + +- Three routes on the existing `/api/v1/ssh` group: + `GET /reachability` for every SSH executor's record, + `GET /executors/:id/reachability` for one, and + `POST /executors/:id/reachability/probe` to probe now, persist, and return + the record. +- **Do not reuse `resolveSSHTarget`'s mapping wholesale.** That helper answers + `400` both for a wrong executor type *and* for a failure to project the + executor's config into a target, and an unresolvable configuration (or one + with no pinned fingerprint) is exactly the case `AC-…-002.5` requires to + return a `200` outcome carrying reason `config` instead. On all three + routes: `404` for a nonexistent id or one that is soft-deleted; `400` only + when the id names an executor whose `type` is not `ssh`; an unresolvable + `ssh` config is a `200` with reason `config`, never a `400`. The `POST` + route additionally answers `409` when the executor is `ssh` but its `status` + is not `active` — probing a deactivated executor on request would + contradict the rule that its retained record stays unchanged. +- `GET /reachability` returns one entry per SSH executor that is not + soft-deleted, **ordered by `executor_id` ascending** — the same total order + the poller's pass uses — and includes an executor whose `status` is not + `active`, carrying its retained record (`AC-…-001.17`). +- `GET` on an eligible executor with no record returns the `unknown` shape + with every timestamp — `checked_at`, `last_success_at`, and `updated_at` — + `null`, never `404`. A client treats a null `updated_at` as older than any + real record, so this synthesized shape can never displace one that actually + exists. +- The DTO adds `updated_at` alongside `checked_at` and `last_success_at`, and + `probing_enabled`, `false` when the interval key is `0`, so a surface + distinguishes "not probed yet" from "probing is off" without inferring it + from an absent timestamp. `probe_interval_seconds` carries the **effective** + (clamped) interval, which task 06's staleness and refetch logic depends on. +- Coalescing through a `golang.org/x/sync/singleflight` group keyed by executor + id, so two overlapping `POST`s open one connection and share one result. Its + write follows the same last-write-wins rule as a scheduled probe. **The + coalesced probe does not run on any caller's request context** — a + disconnecting caller must neither cancel it nor fail the other caller + waiting on it — but it is also not `context.Background()`: it runs through + task 03's off-cycle-probe primitive, registered on the reachability + package's own `WaitGroup`/context, so `Stop` still cancels and awaits it. A + probe that ran but was not stored (the write failed, or a guard refused it) + still answers `200` with the outcome and `persisted: false`. +- The immediate probe runs and persists while the poller is disabled. +- `events.ExecutorReachabilityChanged = "executor.reachability.changed"`, a + matching `ws.ActionExecutorReachabilityChanged`, and the bridge subscription + in `task_notifications.go` beside `events.ExecutorUpdated`. Published only + when `state` or `reason` differs from the stored record. +- **Save detection lives in `internal/task/service`, not `internal/ssh`.** + `Service.CreateExecutor` and `Service.UpdateExecutor` + (`service_resources.go`) are the only two call sites that can form a + before/after connection-configuration comparison — `internal/ssh` never + sees an executor write. Each notifies an `ExecutorSaveObserver` interface + the service package declares (this task adds the interface and the two call + sites); `internal/executors/reachability` implements it and is wired at + construction (also this task). `applyExecutorUpdates` mutates the loaded + executor in place, so `UpdateExecutor` must copy the before-values out + ahead of that call, or a same-row comparison never detects a difference and + the reset never fires. On any difference in the connection-configuration + fields (a create counts as a difference by definition), the observer calls + `ResetExecutorReachability` with the newly saved host and dispatches an + out-of-band probe through task 03's off-cycle primitive, if the executor is + still eligible. + +## Out of scope + +- Any frontend file. Task 06 owns the client types, API calls, and store. +- Changing `POST /api/v1/ssh/test`, `probe-agents`, or `probe-shells`. The + immediate-probe action is a separate action against a saved executor and its + pinned fingerprint; the test endpoint keeps dialing unpinned against a + possibly-unsaved form configuration and keeps writing no record. +- Reachability fields on the executor DTO. A record that changes every interval + must not invalidate the executor list payload. +- Any launch-path behavior (task 05). +- The `ResetExecutorReachability` SQL statement itself and the off-cycle-probe + dispatch primitive. Both are implemented in tasks 02 and 03 respectively; + this task only calls them from the new observer and the `POST` route. + +## Acceptance + +- Two overlapping `POST` probes for one executor produce exactly one dial and + return one identical record to both callers. +- A probe whose result matches the stored state and reason publishes nothing; + a probe that changes either publishes exactly one event carrying the record. +- `GET /reachability` returns entries ordered by `executor_id` ascending, + includes a deactivated `ssh` executor's retained record, and omits a + soft-deleted executor entirely. +- A `GET` for a wrong-type executor id returns `400`; a `GET` for an `ssh` + executor whose config cannot resolve a target returns `200` with reason + `config`, not `400`. A `POST` against a deactivated `ssh` executor returns + `409` without probing. +- Saving a changed host on an existing SSH executor leaves its record + `unknown` with a zero counter and both timestamps `null`, and dispatches a + probe without waiting for the next scheduled pass, rather than carrying the + previous target's failure streak. Saving a change that does not touch the + connection-configuration fields (for example only a name/label change) + leaves the record untouched. +- Cancelling the client that issued a `POST` mid-probe neither cancels the + probe nor prevents its result from being persisted. + +## Verification + +Start with the coalescing test as a failing test — hold a probe open, issue a +second request, and assert one dial and two identical responses — and confirm +it fails against a handler that probes per request. Then run: + +```bash +# From apps/backend: +go test -tags fts5 -race ./internal/ssh/... +go test -tags fts5 -race ./internal/executors/reachability/... -run 'Publish|Change|SaveObserver' +go test -tags fts5 -race ./internal/task/service/ -run 'Executor.*Reachability|SaveObserver' +go test -tags fts5 -race ./internal/gateway/websocket/ -run 'Reachability|Executor' +go test -tags fts5 -race ./internal/events/... +make lint +``` + +## Files likely touched + +- `apps/backend/internal/ssh/reachability_handlers.go` +- `apps/backend/internal/ssh/reachability_handlers_test.go` +- `apps/backend/internal/ssh/handlers.go` +- `apps/backend/internal/events/types.go` +- `apps/backend/pkg/websocket/actions.go` +- `apps/backend/internal/gateway/websocket/task_notifications.go` +- `apps/backend/internal/executors/reachability/publish.go` +- `apps/backend/internal/executors/reachability/publish_test.go` +- `apps/backend/internal/executors/reachability/save_observer.go` +- `apps/backend/internal/executors/reachability/save_observer_test.go` +- `apps/backend/internal/task/service/service_resources.go` +- `apps/backend/internal/task/service/service_resources_executors_reachability_test.go` + +## Dependencies + +Task 03. The poller's write path is the single place that decides whether a +result is a change, and the immediate probe must share it rather than +reimplementing hysteresis at the handler. Task 02's `ResetExecutorReachability` +statement and task 03's off-cycle-probe primitive, both consumed by the new +`ExecutorSaveObserver`. + +## Risks + +- A `singleflight` group keyed by executor id shares the *result* as well as + the work. A caller that mutates the returned record would corrupt the other + caller's copy; return a value, not a pointer into shared state. +- The change-only publish rule is what keeps a steady host silent, and it is + also what makes `checked_at` age in an open client. Task 06 must refetch on + the interval; if that is dropped, this task's correctness becomes a + user-visible staleness bug in a different work order. +- Registering a new WS action without the bridge subscription produces no + compile error and no test failure unless the bridge is asserted directly. +- The save-reset mechanism spans two packages: the interface and its call + sites live in `internal/task/service`, the implementation in + `internal/executors/reachability`. A change to `applyExecutorUpdates`'s + field set in a later, unrelated PR can silently stop detecting a + connection-configuration change if the observer's field list is not kept in + sync; there is no compiler error for a comparison that reads a field that no + longer changes. Name the exact compared fields in the observer's own test, + not just in a comment. +- `Service.UpdateExecutor` mutates the loaded executor in place via + `applyExecutorUpdates`; capturing before-values after that call instead of + before it compares the row against itself and the reset silently never + fires. Assert this with a test that changes the host and asserts the + observer actually saw old-host/new-host, not just that it was called. + +## Parallelism + +`parallel-safe` with task 05 — disjoint files, both depend only on task 03. + +## Inputs + +- System design, sections *API and event contracts* and *Save detection*. +- `internal/ssh/handlers.go` for the route group and the WS dispatcher + registration shape (not its 404/400 mapping — see the *Do not reuse* + bullet above). +- `internal/gateway/websocket/task_notifications.go` around the + `events.Executor*` subscriptions. +- `internal/task/service/service_resources.go` around `CreateExecutor` + (line ~1581), `UpdateExecutor` (line ~1611), and `applyExecutorUpdates` + (line ~1683) for the exact before/after capture points. + +## Results + +Implemented as specified. + +- `internal/executors/reachability/publish.go` adds `RecordDTO`, `BuildRecordDTO` + (the nil-record → `unknown` placeholder, `updated_at`/`probing_enabled`/ + `probe_interval_seconds`/`persisted` projection), and `Publisher.PublishChanged`, + wired via `Poller.SetPublisher`. `Poller.probeAndPersist` and the new + `ProbeAndWait` (`probe_and_wait.go`) both route through a shared + `observeAndPublish` so every probe primitive (scheduled pass, `ProbeNow`, + `ProbeAndWait`) publishes through the one path, only on an actual state/reason + change. +- `ProbeAndWait` coalesces concurrent callers for the same executor id through a + `golang.org/x/sync/singleflight` group and copies the shared record out before + returning, so no caller can mutate another's copy. It runs on the poller's own + context/`WaitGroup` (via `acquire()`), never the caller's request context, so + `Stop` still drains it and a disconnecting HTTP client neither cancels it nor + affects a sibling still waiting on the same coalesced result. +- `internal/task/service`: added the `ExecutorSaveObserver` interface and + `SetExecutorSaveObserver`; `CreateExecutor`/`UpdateExecutor` call + `notifyExecutorSaved` unconditionally on every save (mirroring + `publishExecutorEvent`'s unconditional-fire shape) — all SSH-type/active-status/ + connection-config-diff gating lives in the observer implementation, not the + service. `UpdateExecutor` captures `before := *executor` strictly before + `applyExecutorUpdates` mutates the loaded executor in place. +- `internal/executors/reachability/save_observer.go` implements + `ExecutorSaveObserver`: eligible only for an active `ssh` executor, resets via + `ResetExecutorReachability` only when one of the eight connection-config keys + actually changed (a nil `before` — create — always counts as changed), publishes + the reset when the previous record was worth announcing, and dispatches an + off-cycle probe via `Poller.ProbeNow` without waiting for the next scheduled pass. +- `events.ExecutorReachabilityChanged`, `ws.ActionExecutorReachabilityChanged`, and + the `task_notifications.go` bridge subscription (bumping `wantSubscriptions` to + 72) route the change event to WS clients on the broadcaster's default (global) + path, matching `ExecutorUpdated`. +- `internal/ssh/reachability_handlers.go` adds the three routes. `loadSSHExecutor` + enforces 404 (nonexistent/soft-deleted) vs. 400 (wrong type) without attempting + config resolution — deliberately not reusing `resolveSSHTarget`, whose 400 also + covers an unresolvable config. `buildReachabilityDTOFromRecord` synthesizes the + `unknown` placeholder for an eligible executor with no record, or a `reason: + config` record (never a 400) when the executor's own config can't resolve a + target via `lifecycle.SSHTargetFromExecutorConfig`. `listReachability` lists all + executors and all records in one query each (no N+1), filters to type `ssh`, + sorts by `ID` ascending, and includes a deactivated executor's retained record. + `probeReachability` returns 409 for a non-active `ssh` executor without probing, + otherwise delegates to the shared `*reachability.Poller.ProbeAndWait`. +- `internal/backendapp`: `startAgentInfrastructure` now captures + `startSSHReachabilityPoller`'s return value, wires its publisher + (`reachabilitypkg.NewPublisher(eventBus)`) and the task service's + `ExecutorSaveObserver` (`reachabilitypkg.NewSaveObserver`), and threads the one + running `*reachability.Poller` instance through `startGatewayAndServe` → + `buildHTTPServer` → `routeParams.sshReachabilityPoller` → `sshhandlers.RegisterRoutes`. + A nil poller is passed as a nil `ReachabilityProber` interface (not a typed-nil + `*Poller`), avoiding the classic Go nil-interface trap in `helpers.go`. +- Process note: production code for `reachability_handlers.go` was written once, + in full, before any test existed for it — a Iron Law violation caught before + it compiled cleanly. The file was deleted and rebuilt from the work order's + mandated RED test first (`TestProbeReachability_CoalescesConcurrentProbes`, + confirmed to fail on a missing `probeReachability`/extended `NewHandler` + signature), then GREEN, then the remaining acceptance criteria were each + covered by a dedicated test (404/400/409/reason-config/ordering/filtering). + Three of those — the config-resolution branch, the ascending sort, and the 409 + short-circuit — were confirmed non-vacuous by temporarily breaking the + corresponding production logic and observing the test fail, then restoring it. + +Verification (all green): + +```bash +go test -tags fts5 -race ./internal/ssh/... +go test -tags fts5 -race ./internal/executors/reachability/... -run 'Publish|Change|SaveObserver' +go test -tags fts5 -race ./internal/task/service/ -run 'Executor.*Reachability|SaveObserver' +go test -tags fts5 -race ./internal/gateway/websocket/ -run 'Reachability|Executor' +go test -tags fts5 -race ./internal/events/... +go test -tags fts5 -race ./internal/backendapp/... +make lint +``` + +Running the full, unfiltered `internal/task/service` suite (not part of this +task's verification list, which only requires the `Executor.*Reachability| +SaveObserver` filter) surfaces a pre-existing, unrelated cluster of failures — +`TestDesktopDiscoveryRootPersistsAcrossServiceRestart`, +`TestReconnectDesktopDiscoveryRootNormalizesOldPath`, +`TestDesktopDiscoveryFailurePreservesCachedRepositories`, +`TestArchiveTaskCleanupPreservesTaskEnvironmentIdentity`, +`TestArchiveUnarchiveResumeReactivatesLocalOnlyBranch`, +`TestDeleteTaskCleanupFindsWorktreeAfterLastSessionDeletedAndRestart`, +`TestDeleteTaskCleanupRemovesEveryWorktreeAfterLastSessionDeletedAndRestart`, +`TestDeleteTaskWithDiscardConsentPersistsAndCleansDirtyWorktree`, +`TestDirtyWorktreeCleanupBatchOrderRemainsRetryable`, and +`TestRepositoryBranchPolicyServiceGitflowStarterIsAtomicAndOneTime`. All ten +are the same class of macOS `/var` vs `/private/var` `TMPDIR` symlink +resolution issue ("unsafe worktree path ... not a directory", "saved +repository path resolves to a different location"), confined to worktree +cleanup, repository discovery, and branch-policy files this task never +touches — none reference executors, reachability, or SSH. diff --git a/docs/plans/ssh-host-reachability/task-05-launch-non-gating.md b/docs/plans/ssh-host-reachability/task-05-launch-non-gating.md new file mode 100644 index 0000000000..f5cf0c0c98 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-05-launch-non-gating.md @@ -0,0 +1,316 @@ +--- +id: "05-launch-non-gating" +title: "Launch-path non-gating and failure attribution" +status: done +wave: 3 +depends_on: ["03-reachability-poller"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-003 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-003.1 + - AC-EXECUTORS-SSH-REACHABILITY-003.2 + - AC-EXECUTORS-SSH-REACHABILITY-003.3 +system_design: + - ../../specs/executors/system-design/ssh-reachability-surfaces.md + - ../../specs/executors/system-design/ssh-reachability.md +--- + +# Task 05: Launch-Path Non-Gating, Failure Attribution, and the Pre-Launch Warning Producer + +## Summary + +Keep the reachability record entirely out of the launch decision, give a +failed SSH launch Kandev's own attribution to the target host, and add the +single producer of the `session.launch.warning` event. `CreateInstance` neither +reads nor writes a record. The producer write (a launch as a probe observation) +is deferred; see the requirement document's `## Out of scope`. + +**Spec-inaccuracy follow-up:** the system design attributes the warning +producer to `internal/task/service`, at "the point a task session is bound to +an SSH executor, before `SSHExecutor.CreateInstance` is called." That package +is not in the launch call path. Per `apps/backend/AGENTS.md`'s Execution Flow +(Orchestrator → Lifecycle Manager → ExecutorBackend) and the verified call +site, the producer belongs in +`internal/agent/runtime/lifecycle/manager_launch.go`'s +`launchBuildExecutorRequest`, immediately before its existing single +`rt.CreateInstance` call site, using the `Manager`'s existing `eventPublisher` +(the same mechanism behind `PublishPrepareProgress`). This task implements the +verified location; the spec document should be corrected in a follow-up so it +matches what shipped. + +## In scope + +- `SSHExecutor.CreateInstance` reads no reachability record, and no code path + consults one to decide whether to proceed, defer, or re-route. With the + producer write deferred it writes none either, so `lifecycle` takes no new + dependency on the reachability store from `CreateInstance` itself. +- The wrapped dial error names the target host and the reason classified from + **that launch's own** attempt through `ClassifyDialError` — not the + stored record, which may predate the attempt by a full interval. This is what + addresses the 2026-09-06 report directly: the user saw an agent-generated + message naming a firewall when the cause was a stale address. +- **The single `session.launch.warning` producer.** There is no branch on + whether an interactive user is present — the pre-round-5 interactive/ + non-interactive split is retired. `Manager.launchBuildExecutorRequest`, which + already resolves the per-executor-type backend and calls `rt.CreateInstance` + at its single call site for every executor type, starts a bounded asynchronous + read of the target executor's reachability record immediately before that + call — never inside `CreateInstance` itself, so the negative test + (`CreateInstance` reads no record) stays meaningful. When the executor is + `ssh`, the record's state is `unreachable`, and either periodic probing is + enabled or the record's `checked_at` falls within three times the *default* + interval (`AC-…-001.28`'s reading stays valid even with probing off), it + publishes `session.launch.warning` for the launched session via the + `Manager`'s existing `eventPublisher`, carrying `executor_id`, + `host`, `state`, `reason`, `last_success_at` (null when none recorded), and a + timestamp. A read failure produces no warning and cannot delay the launch. + Every launch path — WS-initiated, a dependency chain, a workflow transition, + an autostart — converges on this one call site, so none can diverge from + another. +- A new WS/session event type, `session.launch.warning`, registered alongside + the existing session event vocabulary so task 07's frontend handler has a + typed contract to bind to. +- An explicit negative test asserting `CreateInstance` performs no reachability + read, and a positive test asserting the warning is published across every + launch entry point that reaches `launchBuildExecutorRequest`, not just the + interactive one. + +## Out of scope + +- Refusing, deferring, queueing, or re-routing a launch on a reachability + record. REQ-003 requires the opposite, and this task must not add a read path + that a later change could turn into a gate. +- **Deferred:** writing a reachability record from the launch path at all — + `reachable` on a successful dial, a classified failure on a failed one. + Retired IDs `AC-EXECUTORS-SSH-REACHABILITY-003.4` through `003.7` are not + reused. +- Rendering the warning. Task 07 owns both the session-stream rendering and + the inline-at-initiation rendering of the event this task publishes. +- Rewriting or intercepting text produced by an agent process. Kandev adds its + own attribution alongside; it does not edit agent output. +- Restarting, migrating, or repairing a session on an unreachable host. +- Any change to resume, reuse, or reclaim paths beyond the dial site. + +## Acceptance + +- A launch against an executor whose stored state is `unreachable` is + attempted, not refused, deferred, or re-routed. +- A create whose dial fails returns an error naming the target host and + carrying the same reason the standalone probe would have produced for that + error, classified from the launch's own attempt rather than read from the + record. +- No code path in `CreateInstance` reads or writes a reachability record, on + any branch, proven against a store fake that fails the test when touched. +- Launching a session bound to an `ssh` executor whose record is `unreachable` + and either currently probed or probed within three default intervals + publishes exactly one `session.launch.warning` carrying `executor_id`, + `host`, `state`, `reason`, and `last_success_at`, and this holds across at + least a WS-initiated launch and a non-interactive (workflow/dependency-chain) + launch, because both converge on `launchBuildExecutorRequest`. +- With probing disabled and a `checked_at` older than three times the default + interval, no warning is published and the launch still proceeds. +- A read failure while resolving the reachability record produces no warning + and does not fail or delay the launch. + +## Verification + +Start with the negative test as a failing test — a store fake that fails the +test if any read method is called from `CreateInstance` — and confirm it +passes only because no read exists, not because a read happened to be skipped +on that path. Then add the cross-launch-path positive test before implementing +the producer, so it is red for the right reason. Then run: + +```bash +# From apps/backend: +go test -tags fts5 -race ./internal/agent/runtime/lifecycle/ -run 'Reachability|CreateInstance|SSHLaunch|LaunchWarning' +go test -tags fts5 -race ./internal/events/... -run 'LaunchWarning' +make test-lifecycle-goleak LIFECYCLE_GOLEAK_COUNT=5 +make lint +``` + +## Files likely touched + +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_launch.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_launch_reachability_warning_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/executor_ssh_reachability_launch_test.go` +- `apps/backend/internal/events/types.go` + +## Dependencies + +Task 03. The warning producer and the non-interactive-equivalent read both +read the stored record (state, `checked_at`, `last_success_at`), so the store +and its poller-owned writer must exist first. The dial-error attribution +itself needs only task 01's classifier. + +## Risks + +- This is the sharpest edge in the capability. A future change that reads the + record inside `CreateInstance` to "skip a pointless launch" would silently + violate REQ-003 without breaking any pre-existing test; the negative test is + the only thing standing between the contract and that change, so it must + assert on a fake that *fails* on read rather than merely counting reads. +- The warning path does read the record, so "no read" (in `CreateInstance`) + and "a read for the warning" (in `launchBuildExecutorRequest`, before + `CreateInstance` is called) coexist in one flow at two different call sites. + Keep the read strictly before the `rt.CreateInstance` call and out of + `CreateInstance` entirely, or the negative test above becomes unassertable + and the gate REQ-003 forbids becomes one refactor away. +- `launchBuildExecutorRequest` is the single call site for every executor + type, not only `ssh`; the new read and the `state == unreachable` branch + must short-circuit immediately for a non-`ssh` executor so this task adds no + behavior change for Docker/process/Kubernetes launches. +- A test that only exercises the WS-initiated launch path can pass while a + workflow-transition or dependency-chain launch silently bypasses the + producer if a second launch entry point does not actually funnel through + `launchBuildExecutorRequest`. Verify the call graph, not just one path, + before trusting the "every launch path converges" claim in a test. +- When the deferred producer write (launch-as-observation) lands, it must + reuse task 03's write path rather than opening a second one; leaving no + recorder here now is what keeps that option open. + +## Parallelism + +`parallel-safe` with task 04 — disjoint files, both depend only on task 03. + +## Inputs + +- System design (`ssh-reachability-surfaces.md`), section *Launch interaction*. +- Plan.md's "Reconciled against spec round 5" note on the verified producer + location, and the "Launch interaction" Technical-approach section. +- Requirement document `## Out of scope` for the deferred producer write. +- `executor_ssh.go` `CreateInstance` around the `dialSSH` call site. +- `manager_launch.go`'s `launchBuildExecutorRequest` (~line 965) and its single + `rt.CreateInstance` call site (~line 1069); `newProgressCallback`'s use of + `m.eventPublisher.PublishPrepareProgress` (~line 888) as the precedent for + publishing through `Manager.eventPublisher`. +- Task 01's `ClassifyDialError`. + +## Results + +Implemented as specified. + +- `executor_ssh.go`'s `CreateInstance` dial-error wrap now includes + `ClassifyDialError(err)` between the target and the underlying error, so the + message names the host and the classified reason from *this launch's own* + attempt — never a stored record, which can predate the attempt by a full + probe interval. Covered by + `executor_ssh_reachability_launch_test.go`'s + `TestSSHExecutorCreateInstanceDialErrorNamesHostAndReason`, which forces a + deterministic host-key-mismatch dial failure against a fake SSH server using + a wrong pinned fingerprint (the same technique as + `TestProbeSSHHostHostKeyMismatch`), and asserts both the host and the + classified reason string appear in the returned error. +- `manager_launch_reachability_warning.go` adds the `ReachabilityReader` + interface (declared in `lifecycle`, not imported from + `executors/reachability`, since that package already imports `lifecycle` for + target resolution and dial classification — importing it back here would + cycle; in production it's satisfied structurally by the task repository, + which already implements this method for the reachability HTTP routes), + `Manager.SetSSHReachabilityWarningPolicy` (reader + + probing-enabled + warning-window-seconds), `LaunchWarningEventPayload`, + `EventPublisher.PublishLaunchWarning`, and + `Manager.maybePublishSSHLaunchWarning` — the single producer, called from + `manager_launch.go`'s `launchBuildExecutorRequest` immediately before its + one `rt.CreateInstance` call site, never from inside `CreateInstance` + itself. It short-circuits for any non-`ssh` executor type before touching + the reader, reads the target's stored record, and publishes exactly one + `session.launch.warning` for the launched session via the existing + `Manager.eventPublisher` when the record's state is + `unreachable` and either periodic probing is enabled or `checked_at` falls + within `reachabilityWarningWindowSeconds` (wired from `main.go` as 3x + `reachability.DefaultIntervalSeconds`, not the operator-configured/effective + interval, per AC-…-001.28). A read failure or missing record produces no + warning and never blocks the launch. +- `events/types.go` registers the `session.launch.warning` event type plus + `BuildSessionLaunchWarningSubject`/`BuildSessionLaunchWarningWildcardSubject` + following the established per-session `.` subject + convention. +- `main.go` wires `lifecycleMgr.SetSSHReachabilityWarningPolicy(repos.Task, + sshReachabilityPoller.EffectiveIntervalSeconds() != 0, + 3*reachabilitypkg.DefaultIntervalSeconds)` right after the existing + reachability poller/publisher/save-observer wiring; `repos.Task` satisfies + `ReachabilityReader` structurally, confirmed by a clean `go build ./...`. +- Eight tests in `manager_launch_reachability_warning_test.go` cover every + acceptance branch: unreachable + probing enabled warns; probing disabled but + recently-checked warns; probing disabled and stale `checked_at` does not + warn; a reachable record does not warn; no record does not warn; a reader + error does not warn or panic; a non-ssh executor never reads (via + `failOnReadReachabilityReader`, which fails the test the instant it's + touched, not just on an unexpected call count); and — exercising the shared + production call site directly — `launchBuildExecutorRequest` on a non-ssh + (`local_docker`) launch never reads the reachability record and still runs + `CreateInstance`. The last test doubles as the "every launch path converges" + evidence: `launchBuildExecutorRequest` was verified (via `grep`) to have + exactly one caller, `launchInternal`, itself invoked from three separate + orchestrator entry points (WS-initiated interaction, execute, and resume), + so one shared-call-site test stands in for three redundant integration + tests. + +**Process note.** Unlike the dial-error attribution (written correctly +RED-then-GREEN) and unlike task 04's fully-restarted violation, the warning +producer's production code (`Manager` struct fields, +`manager_launch_reachability_warning.go`, the `manager_launch.go` wiring, and +the `events/types.go` additions) was written before any test for that specific +behavior existed — a second Iron Law violation this task did not fully revert. +Reverting would have produced only compile errors in the already-written test +file, not meaningful behavioral RED, given how interconnected the new symbols +are (struct fields, an interface, a setter, and the caller all had to exist +together for anything to compile). Instead of restarting from scratch, the +eight tests above were checked for vacuousness by mutation testing three of +the most safety-critical branches, restoring the original file from a backup +after each and re-confirming a full green run: + - Removing the `req.ExecutorType != string(models.ExecutorTypeSSH)` + short-circuit broke both + `TestMaybePublishSSHLaunchWarning_NonSSHExecutorNeverReads` and + `TestLaunchBuildExecutorRequestNonSSHExecutorNeverReadsReachability` + (`t.Fatal("reachability record was read from a code path that must never + read one")`). + - Removing the `record.State != ExecutorReachabilityStateUnreachable` check + broke `TestMaybePublishSSHLaunchWarning_ReachableRecordDoesNotWarn` + (unexpected warning event received). + - Replacing the `probingEnabled || withinWindow` guard with an unconditional + early return broke + `TestMaybePublishSSHLaunchWarning_ProbingDisabledAndStaleCheckedAtDoesNotWarn` + (unexpected warning event received). + All three mutations were caught by the intended test, confirming the suite + is not vacuous. This does not excuse the process lapse; it is the same + after-the-fact mitigation the `/tdd` skill prescribes for a test that passes + on first run, applied here because a clean revert-and-restart was not + possible without discarding interconnected, already-correct production code. + +Verification (all relevant tests green; goleak stress and lint clean): + +```bash +go test -tags fts5 -race ./internal/agent/runtime/lifecycle/ -run 'Reachability|CreateInstance|SSHLaunch|LaunchWarning' +go test -tags fts5 -race ./internal/events/... -run 'LaunchWarning' +make test-lifecycle-goleak LIFECYCLE_GOLEAK_COUNT=5 +make lint +``` + +Running the unfiltered `internal/agent/runtime/lifecycle` package (not part of +this task's required verification list, which only requires the +`Reachability|CreateInstance|SSHLaunch|LaunchWarning` filter above) surfaces a +pre-existing, unrelated cluster of 9 failures — +`TestDefaultPrepareScriptKubernetesReusesRetainedPVCWorkspace`, +`TestDefaultPrepareScriptKubernetesRejectsRetainedWorkspaceFromDifferentRepository`, +`TestWorktreePreparer_MultiRepo_RollbackOnPartialFailure`, +`TestWorktreePreparer_MultiRepo_RequiredRefreshIdentifiesFailingRepository`, +`TestWorktreePreparer_MultiRepo_RollbackRemovesWorktreeCreatedForStaleReuseID`, +`TestWorktreePreparer_FreshStartRejectsStaleWorktreePathOwnedByLiveTask`, +`TestWorktreePreparer_FreshStartRejectsStaleWorktreePathOwnedByLiveTask_WorktreeIDOnly`, +`TestWorktreePreparer_FreshStartRejectsStaleWorktreePath_NestedProjectMarker`, +and `TestBuildAuthMethodsIdentityAgentOverridesEnvironment`. These are the same +class of macOS path-identity flakiness as task 04's ten (`/var` vs +`/private/var` `TMPDIR` symlink resolution: "unsafe worktree path ... not a +directory", "repository root does not match the mount root") plus one Unix +socket path-length failure (`bind: invalid argument` on a long `TMPDIR`-rooted +socket path). Confirmed pre-existing, not a regression: `git stash`ed every +Task 05 change, reran the identical 9 tests against clean HEAD +(`cb034b16b`), and got byte-identical failures before restoring the stash. +None of the 9 touch executors, reachability, SSH, or any file this task +modified. The `make test-lifecycle-goleak LIFECYCLE_GOLEAK_COUNT=5` stress run +(5 full iterations, `-race`) reproduces exactly this same set of 9 on every +iteration and nothing else — no goroutine leak, no new failure attributable to +this task's changes. diff --git a/docs/plans/ssh-host-reachability/task-06-settings-surface.md b/docs/plans/ssh-host-reachability/task-06-settings-surface.md new file mode 100644 index 0000000000..943102567f --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-06-settings-surface.md @@ -0,0 +1,258 @@ +--- +id: "06-settings-surface" +title: "Reachability data layer and settings surface" +status: done +wave: 4 +depends_on: ["04-reachability-api-and-events"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 + - REQ-EXECUTORS-SSH-REACHABILITY-002 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.25 + - AC-EXECUTORS-SSH-REACHABILITY-002.1 + - AC-EXECUTORS-SSH-REACHABILITY-002.2 + - AC-EXECUTORS-SSH-REACHABILITY-002.3 + - AC-EXECUTORS-SSH-REACHABILITY-002.4 + - AC-EXECUTORS-SSH-REACHABILITY-002.5 + - AC-EXECUTORS-SSH-REACHABILITY-002.10 + - AC-EXECUTORS-SSH-REACHABILITY-002.11 + - AC-EXECUTORS-SSH-REACHABILITY-002.12 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md + - ../../specs/executors/system-design/ssh-reachability-surfaces.md +--- + +# Task 06: Reachability Data Layer and Settings Surface + +## Summary + +Add the client types, API calls, and store slice for reachability records, and +render one on the SSH executor settings page: state, probed host, failure +reason and message, consecutive-failure count, and the age of the last probe, +with a probe-now action. Localized in all five supported locales. + +## In scope + +- `SSHReachabilityRecord` in `lib/types/http-ssh.ts` mirroring the backend + shape, including `probing_enabled`, `probe_interval_seconds` (the + **effective**, clamped interval), and `updated_at` alongside `checked_at` + and `last_success_at`. +- `getSSHReachability`, `getSSHExecutorReachability`, and + `probeSSHExecutorReachability` in `lib/api/domains/ssh-api.ts`, following the + existing options-spread-then-required-fields ordering in that file. +- Settings slice state keyed by executor id, plus the + `executor.reachability.changed` WS handler that applies a pushed record. A + refetch racing a pushed event is reconciled by keeping whichever payload + carries the **later `updated_at`**, never `checked_at` — a + connection-configuration reset clears `checked_at` (it is `null`) while + still advancing `updated_at`, so reconciling on `checked_at` would make a + reset compare as older than the record it just invalidated and get + discarded, leaving a stale `reachable`/`unreachable` state showing for a + host the user just re-pointed. A `null` `updated_at` (the synthesized + never-probed shape) always loses to a real record. +- A `SSHReachabilityCard` rendering state, the host that was actually probed, + reason and message when `unreachable`, the consecutive-failure count, and + the age of the last completed probe; marking the result stale past three + intervals; stating plainly that periodic probing is off when + `probing_enabled` is `false`, rather than presenting a retained state as + current; and offering the probe-now action. +- Refetch on mount and on the configured interval (`probe_interval_seconds`, + not a constant) while the card is open, because a steady host publishes + nothing and `checked_at` would otherwise age in place. **No timer runs at + all when `probing_enabled` is `false`** — the card fetches once on open and + again after an immediate probe; reading a `0` effective interval literally + as a cadence would be a zero-delay refetch loop against a record known not + to be changing. +- A failed load reports that reachability is not known. It must never render + as `unreachable`. +- Mobile-viewport layout, and state exposed to assistive technology as text + rather than by color alone. +- Copy through `t()` in `executors.json` across `en`, `pt-pt`, `zh-cn`, + `zh-hk`, `zh-tw`, with the failure reason rendered as translated copy rather + than the raw token. Use `pnpm run i18n:zh-hant` for the Traditional Chinese + pair. + +## Out of scope + +- The pre-launch warning. Task 07 owns it. The task-card indicator is + deferred; see the requirement document's `## Out of scope`. +- Any backend file. +- Changing the existing connection, sessions, agent-readiness, or + task-dir-reclamation cards beyond mounting the new one. +- Any history, trend, or sparkline. One record per executor is all that exists. + +## Acceptance + +- The card renders every field the settings surface owes for each of the three + states, and a record older than three intervals renders as stale rather than + current. +- A pushed `executor.reachability.changed` updates the card without a reload + and without the card polling for it. +- A pushed record with a `null` `checked_at` but a newer `updated_at` (a + reset) replaces a stored `reachable`/`unreachable` record rather than being + discarded as stale. +- With `probing_enabled` false the card says periodic probing is off, runs no + refresh timer, and a load failure says reachability is not known — neither + renders as `unreachable`. + +## Verification + +```bash +# From apps/ (once per fresh worktree): +pnpm install --frozen-lockfile + +# From apps/web: +pnpm vitest run components/settings/ssh-reachability-card.test.tsx lib/state/slices/settings/settings-slice.test.ts lib/api/domains/ssh-api.test.ts +pnpm run typecheck +pnpm run lint +pnpm run i18n:check +pnpm run i18n:ratchet +``` + +## Files likely touched + +- `apps/web/lib/types/http-ssh.ts` +- `apps/web/lib/api/domains/ssh-api.ts` +- `apps/web/lib/state/slices/settings/types.ts` +- `apps/web/lib/state/slices/settings/settings-slice.ts` +- `apps/web/lib/state/slices/settings/settings-slice.test.ts` +- `apps/web/components/settings/ssh-reachability-card.tsx` +- `apps/web/components/settings/ssh-reachability-card.test.tsx` +- `apps/web/components/settings/ssh-settings.tsx` +- `apps/web/src/locales/{en,pt-pt,zh-cn,zh-hk,zh-tw}/executors.json` + +## Dependencies + +Task 04. The record shape, the `probing_enabled` flag, and the change action +are that task's contract. + +## Risks + +- Translations gate the build: `check-i18n-keys.mjs` fails on a missing key, an + extra key, a dropped placeholder, or a value left identical to English. The + six reason tokens plus the state labels are twelve-plus keys across five + locales, so this is the largest single source of build failure in the task. +- The staleness threshold and the refresh cadence both depend on the effective + interval. The record carries it as `probe_interval_seconds`; use that value + rather than a constant, or both rules silently drift from the operator's + configuration. +- Reconciling on `checked_at` instead of `updated_at` is the one regression + that passes every test written against a normal probe sequence and only + fails on a reset, because a reset is the single write that advances + `updated_at` without advancing `checked_at`. Write the reconciliation test + around a reset payload specifically, not just a newer/older probe pair. +- `i18next/no-literal-string` is a lint error repo-wide and + `check-nonjsx-copy.mjs` scans non-JSX positions by exclusion. A reason-to-copy + lookup table written as a plain `.ts` map is exactly the shape that rule + catches. + +## Parallelism + +`sequential` + +## Inputs + +- System design, sections *API and event contracts* and *Frontend components*. +- `apps/web/AGENTS.md` for the store-slice and WS-hook conventions and the + "never fetch data directly in components" rule. +- `components/settings/ssh-agent-readiness-card.tsx` as the nearest existing + card shape. +- `docs/i18n.md` for the five-locale workflow. + +## Results + +Implemented under strict `/tdd` (RED confirmed before every GREEN +implementation): + +- `SSHReachabilityRecord` (+ `SSHReachabilityState`/`SSHReachabilityReason`) + added to `apps/web/lib/types/http-ssh.ts`, mirroring the backend + `reachability.RecordDTO` wire shape used by both the GET responses and the + `executor.reachability.changed` WS payload. +- `getSSHReachability`, `getSSHExecutorReachability`, and + `probeSSHExecutorReachability` added to `lib/api/domains/ssh-api.ts`, + covered by `lib/api/domains/ssh-api.test.ts` (GET list, GET single with + executor-id URL-encoding, POST probe-now). +- A `sshReachability: { byExecutorId }` slice in the settings store + (`lib/state/slices/settings/{types,settings-slice}.ts`), reconciled on + `updated_at` — never `checked_at` — via `setSSHReachability`. A `null` + `updated_at` (the synthesized never-probed placeholder) always loses to a + record that carries a real timestamp; a first write for an executor is + always applied. Covered by 6 tests in `settings-slice.test.ts`, including + the reset case (`checked_at: null`, newer `updated_at`) that a + `checked_at`-based reconciliation would silently discard. +- `executor.reachability.changed` registered in `BackendMessageMap` + (`lib/types/backend.ts`) and wired in + `lib/ws/handlers/executors.ts` to call `setSSHReachability` directly, so a + pushed event and a card's own fetch share one reconciliation path. +- Adding `sshReachability`/`setSSHReachability` required updating both the + settings slice's own type file *and* the hand-written `AppState` type in + `lib/state/app-state-types.ts` (which does not derive from the slice types + by intersection) — missed on the first pass, caught by `pnpm run + typecheck`. +- `SSHReachabilityCard` (`components/settings/ssh-reachability-card.tsx`), + mounted in `SSHExecutorView` + (`app/settings/executors/ssh/[executorId]/page.tsx`) between + `SSHConnectionCard` and `SSHSessionsCard`, and exported from + `ssh-settings.tsx`. Fetches on mount through the store (never into + component-local-only state), so a live WS push and its own fetch/probe-now + results converge on the same rendered state. Renders state, probed host, + reason + message when `unreachable` (reason via a translated lookup, never + the raw token), consecutive-failure count, and the age of the last + completed and last successful probe. Marks a record stale past three + probe intervals (`isReachabilityStale`, a pure exported helper, covered by + 2 direct unit tests plus integration coverage through the card). Refetches + on the record's own `probe_interval_seconds` (never a constant) while + probing is enabled, and arms no timer at all when `probing_enabled` is + false. A load failure with no prior record renders "not known" copy, never + `unreachable`. 12 tests in `ssh-reachability-card.test.tsx`, split across + 4 `describe` blocks (rendering, live updates, refresh cadence, + errors/probe-now) to stay under the per-function line lint budget, cover: + per-state field rendering, staleness on/off, the live pushed-event update + (via a `PushReachability` test helper calling the same store action the WS + handler uses — the same pattern as the existing + `archive-confirmation-settings.test.tsx` "remote update" helper), the + reset-over-stored-record case, the probing-off no-timer guarantee (a + `setInterval` spy, filtering out React Testing Library's own internal + 50ms polling interval), the enabled-cadence timer args, the load-failure + "not known" fallback, and the probe-now round trip. +- Twelve reachability keys (state labels, six reason tokens, staleness, + probing-off, probe-now, host/age/failure-count copy, load-failure copy) + added to `src/locales/en/executors.json`, hand-translated into `pt-pt` and + `zh-cn`, and propagated to `zh-hk`/`zh-tw` via `pnpm run i18n:zh-hant` + (which also incidentally re-derives unrelated already-translated `zh-hk`/ + `zh-tw`/`pseudo` catalog entries outside the `executors` namespace from + current `zh-cn`/`en` source; those out-of-scope regenerated files were + reverted with `git checkout --` before committing, keeping this commit + scoped to the `executors` namespace this task owns). +- Followed the `ssh-agent-readiness-card.tsx` precedent for the card shape + (a `useReachabilityState` hook separating fetch/refresh/probe-now from a + thin view component) and the `ssh-connection-card.test.tsx` / + `archive-confirmation-settings.test.tsx` precedent for real React Testing + Library component coverage (`StateProvider` + `useAppStoreApi`) rather than + pure-helper-only tests — justified here because the acceptance criteria are + about store-integration behavior (live WS update without reload, timer + arm/no-arm, reconciliation-through-render) that a pure helper test cannot + exercise. + +Verification (all from `apps/web`, after `pnpm install --frozen-lockfile` +from `apps/`): + +```bash +pnpm vitest run components/settings/ssh-reachability-card.test.tsx lib/state/slices/settings/settings-slice.test.ts lib/api/domains/ssh-api.test.ts +# 3 files, 32 tests passed + +pnpm run typecheck +# clean + +pnpm run lint +# clean (0 warnings; fixed sonarjs/no-duplicate-string, max-lines-per-function, +# and no-nested-ternary findings during the refactor step) + +pnpm run i18n:check +# ✓ all 6 sub-checks pass; pt-pt/zh-cn/zh-hk/zh-tw complete, pseudo in sync +# (135 pre-existing unrelated orphan-key warnings, unchanged by this task) + +pnpm run i18n:ratchet +# ✓ 0 added + 9 modified file(s) clean; guard allowlist intact +``` diff --git a/docs/plans/ssh-host-reachability/task-07-pre-launch-warning.md b/docs/plans/ssh-host-reachability/task-07-pre-launch-warning.md new file mode 100644 index 0000000000..d62ae42139 --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-07-pre-launch-warning.md @@ -0,0 +1,221 @@ +--- +id: "07-pre-launch-warning" +title: "Pre-launch reachability warning" +status: done +wave: 5 +depends_on: ["05-launch-non-gating", "06-settings-surface"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-003 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-003.2 +system_design: + - ../../specs/executors/system-design/ssh-reachability-surfaces.md +--- + +# Task 07: Pre-Launch Reachability Warning + +## Summary + +A launch against an SSH host currently recorded as unreachable warns before the +attempt and proceeds anyway. The warning informs; it never gates. **The +warning renders a `session.launch.warning` event the backend pushes (task 05's +single producer); it is never derived client-side from the reachability store +slice.** There is no interactive/non-interactive split to implement here — +that split was retired at the backend. The gateway replays the latest warning +to a later subscriber while the backend is running, and a client that initiated +that launch renders the same event inline at the initiation point as well. +Both are this task's responsibility. + +## In scope + +- **Session-stream rendering.** A session-scoped WS event handler for + `session.launch.warning`, following the existing pattern in + `apps/web/lib/ws/handlers/executor-prepare.ts`, that applies the event to + session-runtime state so the session view can render it wherever a user + opens that session's card — including a user who was not present at launch + (a dependency chain, a workflow transition, an autostart). +- **Inline-at-initiation rendering.** The client that issued the launch + request renders the same event inline at the point of initiation, naming + the host and the age of the last successful probe, or stating that none has + been recorded, without waiting for or duplicating the session-stream path. +- The launch proceeds without a confirmation step in either rendering. No + blocking dialog, no disabled action, no extra click on the path to starting + work. +- Both renderers key off the event payload alone (`executor_id`, `host`, + `state`, `reason`, `last_success_at`); neither reads or re-derives + `unreachable` from the task 06 reachability store slice, and neither issues + a fetch of its own. The backend's "when to warn" rule (probing enabled, or a + fresh-enough record while disabled) is entirely task 05's; this task cannot + express or duplicate that condition without risking drift from it. +- Mobile-viewport layout, and the warning exposed to assistive technology as + text rather than by color alone. +- Copy through `t()` in `tasks.json` across `en`, `pt-pt`, `zh-cn`, `zh-hk`, + `zh-tw`, with the failure reason rendered as translated copy, never the raw + token. + +## Out of scope + +- **Deferred:** the task-card reachability indicator. Retired IDs + `AC-EXECUTORS-SSH-REACHABILITY-002.8` and `002.9` are not reused; the + behavior is specified in the requirement document's `## Out of scope`. +- The settings card, the store slice, and the API client — task 06 owns all + three. +- The event's production, its when-to-warn condition, and the event type + registration — task 05 owns all three. This task only consumes the event. +- Blocking, confirming, deferring, or re-routing the launch. The warning is + informational and the attempt proceeds. +- Any backend file. + +## Acceptance + +- Launching against an `unreachable` host produces a `session.launch.warning` + that the session-stream handler applies to session-runtime state, and the + initiating client additionally renders it inline at the point of + initiation, naming that host and the age of the last successful probe; the + launch proceeds without a confirmation step in either rendering. +- When `last_success_at` is null in the event payload, both renderers say no + successful probe has ever been recorded, rather than rendering an empty or + zero age. +- A user who opens the session card later (not present at launch — a + dependency-chain or workflow-triggered launch) still sees the warning via + the session-stream rendering, with no inline rendering required for that + path since no client initiated it interactively. +- Absent a `session.launch.warning` event, neither renderer shows anything; + the frontend never independently concludes "unreachable" from the store + slice. + +## Verification + +```bash +# From apps/web: +pnpm vitest run lib/ws/handlers/ssh-launch-warning.test.ts components/task/launch-warning.test.tsx +pnpm run typecheck +pnpm run lint +pnpm run i18n:check +pnpm run i18n:ratchet +``` + +## Files likely touched + +- `apps/web/lib/ws/handlers/ssh-launch-warning.ts` +- `apps/web/lib/ws/handlers/ssh-launch-warning.test.ts` +- `apps/web/components/task/launch-warning.tsx` +- `apps/web/components/task/launch-warning.test.tsx` +- `apps/web/src/locales/{en,pt-pt,zh-cn,zh-hk,zh-tw}/tasks.json` + +## Dependencies + +Task 05, for the `session.launch.warning` event contract and its production. +Task 06, for the reachability record type reused for shape consistency (the +event payload is a subset of it) and the locale/i18n conventions the settings +card already established. + +## Risks + +- The pre-launch warning sits on the path to starting work. A warning that + accidentally becomes a confirmation step converts an informational surface + into the gate REQ-003 forbids; assert the launch proceeds unattended, with no + user interaction between the warning and the attempt. +- The last-success age can be absent, which is a different sentence from a zero + age. Rendering `0m` where "never" is meant misreports a host that has never + once answered as one that just did. +- Building either renderer against the store slice instead of the event + payload is the most likely regression: it would compile, might even pass a + test written against the slice's current shape, and would silently + reintroduce a second "is this unreachable" computation that can drift from + the backend's when-to-warn rule the moment task 05's condition changes. +- This work order is now a single criterion split across two render surfaces + fed by one event. Resist folding it into task 06: it is a launch surface + with its own locale namespace and its own WS handler, and merging it would + put the settings page and the launch path in one commit. + +## Parallelism + +`sequential` + +## Inputs + +- System design, sections *Frontend components* (Launch surfaces) and + *Launch interaction*. +- Plan.md's Frontend Technical-approach section, "The pre-launch warning + renders a pushed event, not a derived client computation." +- `apps/web/lib/ws/handlers/executor-prepare.ts` for the session-scoped + WS-event handler pattern this task's session-stream half follows. +- Task 05 for the event contract (`executor_id`, `host`, `state`, `reason`, + `last_success_at`) and its publication point. +- The `mobile-parity` skill for native mobile interaction expectations. + +## Results + +Implemented both renderers against the `session.launch.warning` event payload +only, with no read of the task 06 reachability store slice and no client-side +fetch. + +- **Session-stream handler:** `apps/web/lib/ws/handlers/ssh-launch-warning.ts`, + following `executor-prepare.ts`'s pattern, stores a normalized + `LaunchWarningEntry` (`executorId`, `host`, `state`, `reason`, + `lastSuccessAt`) keyed by session id in a new `launchWarning` slot on the + session-runtime slice; registered in `apps/web/lib/ws/router.ts`. +- **Inline-at-initiation renderer:** `apps/web/components/task/launch-warning.tsx` + reads that same slot via `useAppStore` and renders host, last-success age + (or "never recorded"), and a translated reason sentence. It never renders a + confirmation control. Mounted in `task-chat-panel.tsx` alongside the + existing ``, not through a virtualized-list injection + mechanism — that is where the launch surface for a session already renders + transient launch state, and it satisfies "inline at the point of + initiation" without a new mount point. +- Since the session-stream handler is the only consumer and a session card + opened later reads the same store slot, a single handler plus a single + component covers both the "present at launch" and "opened later" cases from + the Acceptance section; no separate non-interactive path was needed. + +**Scope deviations, disclosed:** + +1. **One backend file set, out of this task's stated scope.** Wiring the + bus event to the WS gateway (`GetSessionID()` on the payload, the + `session.launch.warning` action constant, and the subscription + registration in `RegisterSessionStreamNotifications`) turned out to be + missing entirely — task 05 published the event to the internal bus but + nothing forwarded it to WS clients, so no frontend handler could ever have + fired regardless of implementation. Fixed as a small, tested gap-fix + (`manager_launch_reachability_warning.go`, `session_notifications.go`, + `session_notifications_test.go`, `actions.go`) rather than leaving the + frontend half unreachable. Same shape as the disclosed deviations in tasks + 04/05. +2. **Locale namespace corrected from the work order's literal text.** The + work order names `tasks.json`; the actual convention (confirmed against + existing launch/session copy) is `task.json` (`task:` namespace). + `tasks.json` (`tasks:` namespace) is scoped to the `/tasks` list view only. + The 10 new keys were added to `task.json` in all five locales instead. +3. **No reuse of task 06's `reachabilityReasonKey()`.** Per the work order's + own Risk-section guidance against folding this into task 06's namespace, + the six reason tokens are mapped independently to fresh + `task:launchWarningReason*` keys rather than importing task 06's + `executors.json`-scoped helper. +4. **Incidental store-composition fix.** Found and fixed a pre-existing gap in + `apps/web/lib/state/store-overrides.ts`'s `buildStateOverrides`: it did not + list the new `launchWarning` field, so `StateProvider`/boot-payload + `initialState` would have silently been reset to `{bySessionId: {}}` by + the session-runtime slice's own default spread. This is a store-plumbing + fix required for the new field to hydrate correctly at all, not a design + change. + +Verification (from `apps/web`): + +``` +pnpm vitest run lib/ws/handlers/ssh-launch-warning.test.ts components/task/launch-warning.test.tsx lib/state/slices/session-runtime/purge-session.test.ts +# 3 files, 16 tests passed +pnpm run typecheck # clean +pnpm run lint # clean, 0 warnings +pnpm run i18n:check # clean, pseudo in sync, 5 locales complete +pnpm run i18n:ratchet # clean +``` + +And from `apps/backend`: + +``` +go test ./internal/gateway/websocket/... -run TestSessionStreamBroadcaster +# ok +gofmt -l # no output +``` diff --git a/docs/plans/ssh-host-reachability/task-08-e2e-and-docs.md b/docs/plans/ssh-host-reachability/task-08-e2e-and-docs.md new file mode 100644 index 0000000000..3132edc49d --- /dev/null +++ b/docs/plans/ssh-host-reachability/task-08-e2e-and-docs.md @@ -0,0 +1,183 @@ +--- +id: "08-e2e-and-docs" +title: "Container E2E coverage and operator documentation" +status: done +wave: 6 +depends_on: ["05-launch-non-gating", "07-pre-launch-warning"] +plan: "plan.md" +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 + - REQ-EXECUTORS-SSH-REACHABILITY-002 + - REQ-EXECUTORS-SSH-REACHABILITY-003 +acceptance_criteria: + - AC-EXECUTORS-SSH-REACHABILITY-001.9 + - AC-EXECUTORS-SSH-REACHABILITY-001.11 + - AC-EXECUTORS-SSH-REACHABILITY-002.1 + - AC-EXECUTORS-SSH-REACHABILITY-003.1 + - AC-EXECUTORS-SSH-REACHABILITY-003.3 +system_design: + - ../../specs/executors/system-design/ssh-reachability.md + - ../../specs/executors/system-design/ssh-reachability-surfaces.md +--- + +# Task 08: Container E2E Coverage and Operator Documentation + +## Summary + +Prove the assembled capability against a real sshd target while traffic to +port 22 is blocked and restored, covering the one thing no unit test can: that +both surfaces flip in the right direction at the right probe. Document the +single new operator configuration key. + +## In scope + +- `apps/web/e2e/tests/ssh/reachability.spec.ts` under the `containers` + project, which already gates real-SSH scenarios on `KANDEV_E2E_CONTAINERS=1` + and owns the `kandev-sshd:e2e` image. Four flows: + - A reachable host renders the settings panel row as reachable. + - Traffic to the sshd container is blocked; after the failure threshold, the panel + reports the host unreachable by name. + - Traffic is restored; the next probe clears the panel on a single + success, with no confirming probe. + - A launch started while the host is stopped is attempted and fails with a + message naming the host, rather than being refused. +- The failure threshold is driven by repeated **Probe now** requests. These + requests use the same store path as the scheduled poller and avoid a + wall-clock dependency in the test. +- `docs/public/configuration.md` gains + `executors.sshReachabilityIntervalSeconds` / + `KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS`: default, supported range, + the meaning of `0`, and the clamp behavior. +- `docs/public/executors.md` gains a short SSH reachability section describing + what the probe answers, what it deliberately does not do (it never gates a + launch and never repairs a moved host), and where the result appears. + +## Out of scope + +- Any production code change. If a flow fails, the fix belongs in the work + order that owns the behavior, not here. +- Generic QA, review, or full-verification sweeps. +- Non-container E2E projects. Every flow here needs a stoppable real host. +- Screenshots or media capture. + +## Acceptance + +- The stop flow observes the settings panel naming the host as unreachable, + and does so only after the threshold is reached, not on the first failed + probe. +- The restart flow observes the panel clearing on a single successful probe. +- Both public documentation pages describe the key and the capability's + deliberate limits, and the docs build passes. + +## Verification + +```bash +# From apps/web (requires a running Docker daemon): +KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --project=containers tests/ssh/reachability.spec.ts + +# Documentation: +pnpm run lint +``` + +Do not pass all-worker overrides: local runners enforce one worker per shard +and a memory-aware shard budget, and the `containers` project's +worker-scoped backend depends on that serialization. + +## Files likely touched + +- `apps/web/e2e/tests/ssh/reachability.spec.ts` +- `apps/web/e2e/pages/SSHSettingsPage.ts` +- `apps/web/e2e/helpers/ssh.ts` +- `docs/public/configuration.md` +- `docs/public/executors.md` + +## Dependencies + +Tasks 05 and 07. The launch flow needs the launch-path attribution and the +pre-launch warning. + +## Risks + +- These specs cannot run on a host without a Docker daemon, so they are the + one part of the work package that a contributor may not be able to execute + locally. Every acceptance criterion in the plan except the four flows here + has non-E2E evidence, so a blocked run does not blind the rest of the + package. +- Stopping a container mid-test makes timing part of the assertion. Drive the + threshold through the configured interval rather than through fixed waits, + or the spec becomes the slowest and flakiest file in the project. +- `docs/public/**` is website-ready user documentation; a configuration key + documented with the wrong default or a missing `0` semantic is worse than an + undocumented one, because an operator will trust it. + +## Parallelism + +`sequential` + +## Inputs + +- Plan sections *E2E tests* and *Tests*. +- `apps/web/e2e/README.md` and the `containers` project definition in + `e2e/playwright.config.ts`. +- `e2e/helpers/ssh.ts` and `e2e/fixtures/ssh-image.ts` for the sshd container + lifecycle. +- `docs/public/configuration.md` for the existing key-documentation shape. + +## Results + +Implemented `apps/web/e2e/tests/ssh/reachability.spec.ts` under the +`containers` project, extended `apps/web/e2e/pages/SSHSettingsPage.ts` with +reachability-card locators and a `probeNow()` helper, and documented +`executors.sshReachabilityIntervalSeconds` / +`KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS` in +`docs/public/configuration.md` plus a new "Reachability" section in +`docs/public/executors.md`. `apps/web/e2e/helpers/ssh.ts` needed no change; +`dropTrafficToPort22`/`restoreTraffic` already existed from `concurrency.spec.ts`. + +Two deliberate deviations from this work order's literal text, both verified +against real backend source before writing: + +1. **Threshold driven by repeated "Probe now" clicks, not a short configured + poller interval.** `httpProbeReachability` → `Poller.ProbeAndWait` runs the + exact same `store.Observe` hysteresis path (`consecutive_failures` + increment, threshold-gated state flip, immediate clear on success) that the + scheduled poller would, and the route is synchronous, so a click-then-await + drives the identical production logic with zero wall-clock dependency. This + more directly satisfies this file's own Risk ("timing part of the + assertion... becomes the slowest and flakiest file in the project") than + configuring a short interval and waiting on ticks would have. +2. **The launch-failure flow needed a far larger timeout budget than a + dial-timeout calculation predicts, discovered by running the spec for + real, not by reasoning from source alone.** The client's own + `sshDialTimeout` is 30s, so the first version of the launch-failure + assertion used a 60s poll (matching `error-surfacing.spec.ts`'s equivalent + assertion). It reliably timed out with the session still `CREATED`. A debug + run with `E2E_DEBUG=1` showed the real failure surfacing only after ~148s, + with the client-side error itself an SSH handshake EOF + (`ssh: handshake with 127.0.0.1:: ssh: handshake failed: EOF`), not a + dial timeout — meaning the TCP connect to the published port succeeds + (completed against Docker's own port-publishing plumbing) before the + connection stalls and is eventually torn down. That stall, not the + client's dial budget, dominates the wall-clock cost, and its length is a + property of the host's Docker networking mode rather than of this + codebase's own configured timeouts. The assertion's poll timeout is now + 200s (test timeout 240s) to cover that path with margin rather than one + tuned to a fast local failure (`error-surfacing.spec.ts`'s missing-binary + case, which fails almost immediately and never pays this cost). + +## Verification + +``` +$ cd apps/web && pnpm run lint +> @kandev/web@0.1.0 lint +> eslint --max-warnings 0 +(clean, exit 0) + +$ node scripts/validate-public-docs.mjs +Validated 46 published docs pages. + +$ cd apps/web && KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --project=containers tests/ssh/reachability.spec.ts + ✓ ssh executor — reachability › reflects reachable, then unreachable after the failure threshold, then clears on a single success + ✓ ssh executor — reachability › a launch started while the host is unreachable is attempted and fails naming the host, not refused + 2 passed +``` diff --git a/docs/public/configuration.md b/docs/public/configuration.md index 018c0bd859..3147ea5378 100644 --- a/docs/public/configuration.md +++ b/docs/public/configuration.md @@ -214,6 +214,7 @@ variable overrides YAML, and changes require a backend restart. | `agentctl.detachedEventLimit` | `KANDEV_ACP_DETACHED_EVENT_LIMIT` | integer `1`-`10000`, `100` | Per-instance agent events retained while no backend is attached. An invalid or out-of-range environment value fails startup. | | `planning.coalesceWindowMs` | `KANDEV_PLAN_COALESCE_WINDOW_MS` | integer `>= 0`, `300000` | Same-author plan revision coalescing window in milliseconds. | | `observability.otlpEndpoint` | `OTEL_EXPORTER_OTLP_ENDPOINT` | URL, empty | OTLP tracing endpoint. Treat the value and emitted spans as sensitive. | +| `executors.sshReachabilityIntervalSeconds` | `KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS` | integer `>= 0`, `60` | Interval between background SSH-executor reachability probes, in seconds. `0` disables the poller entirely; a probed executor then keeps its last recorded state rather than going stale. The effective cadence is otherwise clamped to `15`-`3600`: a value from `1` to `14` becomes `15`, and a value above `3600` becomes `3600`, each logged once at startup. Valid environment and YAML values use this clamp. An invalid environment value falls back to `60`; an invalid YAML value fails startup like any other typed catalog key. | | `office.schedulerTickMs` | `KANDEV_OFFICE_SCHEDULER_TICK_MS` | positive integer, `5000` | Office queued/retry run safety-net interval in milliseconds. | | `launcher.webPort` | `KANDEV_WEB_PORT` | automatic, `0` | Development web-server port. It is used with `dev` and ignored by embedded-asset launches. | | `launcher.healthTimeoutMs` | `KANDEV_HEALTH_TIMEOUT_MS` | positive integer, `45000` | Launcher startup-health timeout in milliseconds. Development and E2E profiles use a longer default. | diff --git a/docs/public/executors.md b/docs/public/executors.md index 821dbf7f10..1bb1e32ed9 100644 --- a/docs/public/executors.md +++ b/docs/public/executors.md @@ -442,6 +442,30 @@ Stop attempts to kill the session's remote `agentctl` and remove only the remote +### Reachability + +A background poller probes each active SSH executor's configured host on +`executors.sshReachabilityIntervalSeconds` (default 60s; see +[Configuration](configuration.md)) and records whether the TCP dial and SSH +handshake succeeded. The result (reachable, unreachable with a failure +reason, or unknown before the first probe) appears on the executor's +**Settings > Executors > SSH** page, including the probed host, how long ago +the last successful probe was (or that none has ever succeeded), and a manual +**Probe now** action. + +If a task launches against a host currently recorded as unreachable, Kandev +publishes a `session.launch.warning` event for the launched session. The task's +chat panel shows an inline warning naming the host and the age of the last +successful probe before the attempt proceeds. The gateway keeps the latest +warning while the backend is running, so a later session subscriber receives +it too. Workflow and dependency launches use the same event path. + +The probe deliberately does two things and no more: it reports what it +observed, and it never blocks or delays a launch on that basis. It does not +retry a failed connection immediately, and it does not move, reconfigure, or +otherwise repair a host that has become unreachable or changed address; +fixing the host or the connection settings remains an operator action. + ## Lifecycle and cleanup The task environment reports `creating`, `ready`, `stopped`, or `failed`; individual execution records have finer states. Stop is deliberately not synonymous with destroy for resumable Docker, Kubernetes, and Sprites environments. diff --git a/docs/specs/executors/README.md b/docs/specs/executors/README.md index ab85b435f1..44bff51444 100644 --- a/docs/specs/executors/README.md +++ b/docs/specs/executors/README.md @@ -45,6 +45,7 @@ across a backend restart, and executor-specific failure and recovery contracts. - [Kubernetes startup timing](requirements/kubernetes-startup-timing.md) - [Kubernetes retained compute visibility](requirements/kubernetes-retained-compute.md) - [SSH Session Transport Liveness](requirements/ssh-transport-liveness.md) +- [SSH Host Reachability](requirements/ssh-reachability.md) ### System design @@ -64,6 +65,8 @@ across a backend restart, and executor-specific failure and recovery contracts. - [Kubernetes startup timing](system-design/kubernetes-startup-timing.md) - [Kubernetes retained compute visibility](system-design/kubernetes-retained-compute.md) - [SSH Session Transport Liveness](system-design/ssh-transport-liveness.md) +- [SSH Host Reachability](system-design/ssh-reachability.md) +- [SSH Host Reachability Surfaces](system-design/ssh-reachability-surfaces.md) ## Migration record diff --git a/docs/specs/executors/requirements/ssh-reachability.md b/docs/specs/executors/requirements/ssh-reachability.md new file mode 100644 index 0000000000..f0bd4beb3f --- /dev/null +++ b/docs/specs/executors/requirements/ssh-reachability.md @@ -0,0 +1,178 @@ +--- +status: draft +system: executors +created: 2026-09-07 +owners: + - tbd +--- +# SSH Host Reachability Requirements + +## Overview + +An SSH executor points at a host Kandev does not own, which can move, sleep, or +change address at any time. `SSHExecutor.HealthCheck` returns `nil` +unconditionally and the registry's health sweep runs once at backend startup, so +no configured host is re-checked while Kandev runs. The manual "Test +connection" action is the only reachability signal, and it sits on a settings +page nobody has open when a host goes away. + +On 2026-09-06 an orchestrator host's LAN address changed four times in one day. +Every agent bound to the SSH runner failed with a message naming a firewall +while the cause was a stale address; two cards died mid-task. + +This capability gives Kandev a continuous, cheap answer to one question per +configured SSH host: can the backend still open an authenticated SSH connection +to the host we trusted? The answer goes on the executor's settings page. + +The executor system owns this because the durable contract is an executor's +availability and its failure and recovery behavior, which the +[executor system boundary](../README.md) already claims. + +## Terminology + +- **SSH executor:** an `executors` row of type `ssh`, carrying the host, port, + user, identity source, optional `ProxyJump`, and the pinned host fingerprint a + user trusted at creation time. Exactly those fields are its **connection + configuration**; no other column of the row is. +- **Reachability probe:** one attempt to open an authenticated SSH transport to + an executor's resolved target using its pinned fingerprint, and close it. It + runs no remote command. +- **Reachability record:** the persisted outcome of the most recent probe for + one executor: state, failure reason, message, consecutive-failure count, + probed host, that probe's completion timestamp, the timestamp of the most + recent probe that succeeded, and the timestamp at which the record itself was + last written. +- **Reachability state:** `unknown`, `reachable`, or `unreachable`. +- **Probe pass:** one sweep of the poller across every eligible SSH executor. +- **Eligible executor:** an SSH executor that is not soft-deleted and whose + `status` is `active`. +- **Effective interval:** the interval in force after the normalization required + by AC-EXECUTORS-SSH-REACHABILITY-001.24. Every rule below referring to "the + interval" means this value. +- **Sticky failure reason:** `config` or `host_key`. Both repeat on every later + probe until a human changes something, so neither is a transient blip. + +## Requirements + +### REQ-EXECUTORS-SSH-REACHABILITY-001: Continuous SSH host reachability probing + +**Intent:** Turn "is this SSH host still there?" from a question answered only +by a failing launch into a fact Kandev already holds, cheap enough to pay +continuously, so an operator learns the host moved before a card dies on it. + +#### Acceptance criteria + +- **AC-EXECUTORS-SSH-REACHABILITY-001.1:** When the poller is enabled, the system shall run one probe pass immediately at backend start and then repeat a probe pass once per effective interval for as long as the backend runs. +- **AC-EXECUTORS-SSH-REACHABILITY-001.2:** When a probe pass runs, the system shall probe every eligible SSH executor exactly once in that pass, and shall probe no other executor type. +- **AC-EXECUTORS-SSH-REACHABILITY-001.3:** When a probe pass runs, the system shall probe eligible executors in a stable total order that is the same for every pass over the same set of executors. +- **AC-EXECUTORS-SSH-REACHABILITY-001.4:** When a probe opens an authenticated SSH transport whose host key matches the executor's pinned fingerprint, the system shall record state `reachable`, an empty reason and message, a consecutive-failure count of `0`, the probed host, and the completion timestamp as both the completion and last-success timestamps, then close the transport without running any remote command. +- **AC-EXECUTORS-SSH-REACHABILITY-001.5:** When a probe does not reach an authenticated transport, the system shall record the failure with exactly one machine-readable reason from the closed set `config`, `network`, `timeout`, `auth`, `host_key`, `unknown`, together with a human-readable message, shall increment the consecutive-failure count, and shall leave the last-success timestamp unchanged. The recorded reason shall be reported until a later probe succeeds, including while the state is still `reachable` or `unknown` below the threshold, so the reason is empty only when the most recent completed probe succeeded or none has completed. +- **AC-EXECUTORS-SSH-REACHABILITY-001.6:** When an executor's configuration cannot be resolved into a connection target, or resolves without a pinned fingerprint, the system shall record reason `config` without attempting any network connection. A probe shall never dial a host whose fingerprint is unpinned: such a dial accepts any key presented and would record a substituted host as `reachable`. +- **AC-EXECUTORS-SSH-REACHABILITY-001.7:** When the presented host key does not match the executor's pinned fingerprint, the system shall record reason `host_key`, shall record state `unreachable` on that first mismatch without waiting for the failure threshold, and shall not update the executor's stored fingerprint. +- **AC-EXECUTORS-SSH-REACHABILITY-001.8:** When an individual probe has not completed within its probe deadline, the system shall abandon that probe, record reason `timeout`, and continue the pass. +- **AC-EXECUTORS-SSH-REACHABILITY-001.9:** When a probe fails with a reason that is not a sticky failure reason, the system shall record state `unreachable` if the resulting consecutive-failure count is at or above the failure threshold, and otherwise shall leave the state exactly as it was, so that a below-threshold failure changes the reason and the counter but never the state. +- **AC-EXECUTORS-SSH-REACHABILITY-001.10:** When a probe fails with a sticky failure reason, the system shall record state `unreachable` on that probe regardless of the consecutive-failure count. +- **AC-EXECUTORS-SSH-REACHABILITY-001.11:** When the current state is `unreachable` and a probe succeeds, the system shall record state `reachable` on that single success, without requiring a second confirming probe. +- **AC-EXECUTORS-SSH-REACHABILITY-001.12:** When an executor has no reachability record, the system shall report state `unknown`, and shall keep reporting `unknown` for that executor until its first probe completes. +- **AC-EXECUTORS-SSH-REACHABILITY-001.13:** When a probe pass is still running at the moment the interval elapses, the system shall skip that tick rather than starting a second concurrent pass or queueing the tick for later. +- **AC-EXECUTORS-SSH-REACHABILITY-001.14:** The system shall bound reachability probes running concurrently across the whole backend by one fixed limit, however many executors are configured, saved at once, or probed on request together. A launch's own SSH connection is not a probe and is not bounded by this limit. +- **AC-EXECUTORS-SSH-REACHABILITY-001.15:** When no eligible SSH executor exists, a probe pass shall complete without opening a connection and without emitting a warning or error log. +- **AC-EXECUTORS-SSH-REACHABILITY-001.16:** When an executor is soft-deleted, the system shall delete its reachability record and stop probing it. +- **AC-EXECUTORS-SSH-REACHABILITY-001.17:** When an executor's `status` is not `active`, the system shall not probe it and shall retain its last reachability record unchanged. +- **AC-EXECUTORS-SSH-REACHABILITY-001.18:** When an SSH executor is created, or a save leaves at least one of its connection-configuration fields different from the stored value, the system shall reset that executor's record to state `unknown` with a consecutive-failure count of `0`, an empty failure reason, an empty message, the newly saved host, no completion timestamp, and no last-success timestamp, and shall then probe it without waiting for the next scheduled pass if it is eligible, and not otherwise. A save that leaves every such field equal to the stored value shall reset nothing and probe nothing. +- **AC-EXECUTORS-SSH-REACHABILITY-001.19:** When an executor's connection configuration is saved while a probe against that executor is already in flight, the system shall discard that probe's result rather than write it: it describes the pre-change target. +- **AC-EXECUTORS-SSH-REACHABILITY-001.20:** When an executor becomes ineligible after a pass has listed it and before that probe's result is written, the system shall discard the result, so it cannot re-create a deleted record or write one for an ineligible executor. +- **AC-EXECUTORS-SSH-REACHABILITY-001.21:** When two probe results for the same executor are written concurrently, the system shall retain the one whose completion timestamp is strictly later and discard the other. Completion timestamps shall be recorded at millisecond resolution or finer, and a result whose timestamp equals the stored one shall be discarded. +- **AC-EXECUTORS-SSH-REACHABILITY-001.22:** When the backend restarts, the system shall preserve each executor's last reachability record and both its timestamps rather than clearing them to `unknown`, and shall report the preserved record until a new probe replaces it. +- **AC-EXECUTORS-SSH-REACHABILITY-001.23:** The probe interval, the per-probe deadline, the concurrent-probe limit, and the failure threshold shall each have a single documented default; the interval shall be operator-configurable through one documented configuration key whose value `0` disables the poller entirely. +- **AC-EXECUTORS-SSH-REACHABILITY-001.24:** When the interval is absent, empty, negative, or not a whole number in the environment variable, the system shall use the documented default and start the backend normally. When the configuration file supplies a value that is not a whole number, the system shall refuse to boot with a configuration error, as it does for every other typed key; a negative whole number from either source shall yield the documented default. When the value is `0` the system shall disable the poller. When it is above `0` but outside the documented supported range, the system shall clamp it to the nearer bound and log the clamp once at startup. Every rule that depends on the interval shall use the resulting effective value. +- **AC-EXECUTORS-SSH-REACHABILITY-001.25:** When the poller is disabled by configuration, no probe pass shall run, including the pass at backend start, and each surface that renders a reachability record shall state that periodic probing is off rather than presenting a retained state as a current result. +- **AC-EXECUTORS-SSH-REACHABILITY-001.26:** When the backend shuts down, the system shall stop the poller and shall cancel and await every probe it started, whether scheduled, off-cycle, or requested, leaving no probe goroutine and no probe connection alive once shutdown returns. +- **AC-EXECUTORS-SSH-REACHABILITY-001.27:** When more than one reason could describe a single probe failure, the system shall select the first that applies in the order `config`, `timeout`, `host_key`, `auth`, `network`, `unknown`. +- **AC-EXECUTORS-SSH-REACHABILITY-001.29:** When a probe is rejected because a jump host's key does not match the entry recorded for that host, the system shall record reason `host_key` and shall record state `unreachable` on that first rejection, exactly as for a mismatch on the target host: the reason describes what failed, not which hop failed. +- **AC-EXECUTORS-SSH-REACHABILITY-001.28:** When the poller is disabled by configuration, the immediate-probe action shall still run on request and shall persist its result under the same reason, threshold, and conflict rules as a scheduled probe. + +### REQ-EXECUTORS-SSH-REACHABILITY-002: Reachability surfacing + +**Intent:** Put the answer where the user already is. The 2026-09-06 outage was +diagnosed from backend logs; a user whose agent is failing should see that the +runner is unreachable without opening one, and stop debugging the agent. + +#### Acceptance criteria + +- **AC-EXECUTORS-SSH-REACHABILITY-002.1:** The SSH executor's settings page shall present the executor's reachability state, the host it was probed against, the failure reason and message when the state is `unreachable`, the consecutive-failure count, the age of the last completed probe, and the age of the last successful probe or a statement that none has been recorded. +- **AC-EXECUTORS-SSH-REACHABILITY-002.2:** When an executor's reachability state or failure reason changes, connected clients shall receive that change pushed to them, without reloading the page and without polling for it. +- **AC-EXECUTORS-SSH-REACHABILITY-002.3:** When a probe completes without changing the state or the failure reason, the system shall not push a client update for it, so a steady host produces no per-interval broadcast. A client displaying one executor's reachability detail may refresh that single record for itself while that view is open, and shall keep whichever payload carries the later record-update timestamp, discarding an older one however it arrived. Record-update time orders these, not completion time, so a configuration reset, which clears the completion timestamp, is not dropped as older. +- **AC-EXECUTORS-SSH-REACHABILITY-002.4:** When periodic probing is enabled and the age of the last completed probe exceeds three times the effective interval, the surface shall mark the displayed result as stale rather than presenting it as current. When periodic probing is disabled, the surface shall state that probing is off and shall not compute or present staleness. +- **AC-EXECUTORS-SSH-REACHABILITY-002.5:** The SSH executor's settings page shall offer an explicit action that runs one probe immediately, persists its result under the same rules as a scheduled probe, and returns that result to the caller. When the result is not persisted, because the write failed or a guard refused it, the action shall still return the probe outcome and shall mark it as not persisted rather than reporting the probe itself as failed. +- **AC-EXECUTORS-SSH-REACHABILITY-002.6:** When two immediate-probe requests for the same executor overlap, the system shall run one probe and return its single result to both callers, rather than opening two connections to the same host. The shared probe shall run to completion and persist its result even when the caller that started it disconnects first. +- **AC-EXECUTORS-SSH-REACHABILITY-002.7:** When an immediate probe is requested for an executor that exists and is of type `ssh` but whose `status` is not `active`, the system shall refuse the request, shall open no connection, and shall leave the stored record unchanged. +- **AC-EXECUTORS-SSH-REACHABILITY-002.10:** When a reachability surface cannot load a result, it shall report that the reachability of the host is not known and shall not present the failure to load as an unreachable host. +- **AC-EXECUTORS-SSH-REACHABILITY-002.11:** Every reachability surface shall be operable and legible on mobile viewports and shall expose its state to assistive technology as text, not by color alone. +- **AC-EXECUTORS-SSH-REACHABILITY-002.12:** All reachability copy shall be localized through the product's translation layer in every supported locale, with the failure reason rendered as translated copy rather than the raw reason token. + +### REQ-EXECUTORS-SSH-REACHABILITY-003: Probe informs launches without gating them + +**Intent:** A probe result is evidence, not permission. A host can recover +between the last probe and the launch, so a stale or mistaken probe must never +stand between a user and their own machine. + +#### Acceptance criteria + +- **AC-EXECUTORS-SSH-REACHABILITY-003.1:** When a launch targets an SSH executor whose state is `unreachable` or `unknown`, or whose record is stale or absent, the system shall attempt the launch and shall not refuse, defer, or re-route it on the basis of the reachability record. +- **AC-EXECUTORS-SSH-REACHABILITY-003.2:** When a launch targets an SSH executor whose state is `unreachable` and periodic probing is enabled, the system shall start a best-effort warning lookup before the attempt, naming the host and the age of the last successful probe or stating that none has been recorded, and shall proceed without confirmation or delay. The warning shall be published on the launched session event stream, and the gateway shall replay the latest warning to a later subscriber while the backend is running. The initiating client may render the same event inline. When periodic probing is disabled, the system shall warn only if the record's completion timestamp is within three times the default interval, because an older retained record is not a current measurement while a record just refreshed on request is. +- **AC-EXECUTORS-SSH-REACHABILITY-003.3:** When a launch on an SSH executor fails because Kandev could not open its own SSH connection, the reported failure shall name the target host and the failure reason classified from that launch's own connection attempt. + +## Prior art + +Two external legs were required and neither was available: the `wiki-query` +skill and its vault are absent here, and the `saas-kb` / `search_fsm_docs` MCP +server is not exposed. Both are recorded as unavailable, not as empty results. +In-repository prior art did inform this specification: +`internal/integrations/healthpoll` (the Jira and Linear auth-health poller) and +the existing SSH test-connection endpoint; the four departures are recorded in +the system design's `## Prior art and departures`. + +## Out of scope + +The first two entries are *deferred*, not rejected; the rest are permanent. +A retired criterion ID is never reused. + +- **Deferred: task-card reachability indicator.** Cards bound to an + `unreachable` executor name that host and clear on return to `reachable`, + showing nothing in any other state. No polling and no new join: cards carry the + primary executor's id and type, and every `unreachable` entry and exit is + already pushed. Retires AC-EXECUTORS-SSH-REACHABILITY-002.8 and 002.9. +- **Deferred: launch path as a probe producer.** A launch's own dial is stronger + evidence than a probe: success writes `reachable`, a dial-step failure writes a + classified failure, both under the scheduled probe's rules, while a launch that + never dials writes nothing. Until it lands the poller is the only writer, so a + host dying between passes lags the settings page by up to the failure threshold + times the effective interval. Retires + AC-EXECUTORS-SSH-REACHABILITY-003.4 through 003.7. +- **Health probing for non-SSH executors.** Local, Docker, Kubernetes, and cloud + executors have their own availability signals; `HealthCheckAll` is unchanged + for those runtimes. +- **Repairing what the probe finds.** Re-resolving a moved address, re-pinning + a changed fingerprint, and migrating a session off an unreachable host are + excluded: the probe reports, a human decides. +- **Gating, deferring, queueing, or re-routing launches on a probe result.** + Excluded by REQ-EXECUTORS-SSH-REACHABILITY-003, which requires the opposite. +- **Alerting outside the product surface.** No notification, email, webhook or + chat message is raised when a host goes unreachable. +- **Probing remote capability on the cadence.** Platform detection, `git` + presence, agent binary readiness, shell discovery and the agentctl cache check + stay on the manual test-connection endpoint. +- **Changing the existing manual test-connection endpoint.** It keeps dialing + unpinned, form-supplied configuration for a host that may not be saved yet, and + writes no record. AC-EXECUTORS-SSH-REACHABILITY-002.5 is a separate action + against a saved executor and its pin. +- **Rewriting error text produced by an agent.** The 2026-09-06 firewall message + came from the agent. This contract adds Kandev's own attribution alongside it + rather than intercepting agent output. +- **Per-session or per-forward liveness.** Whether a running session's port + forward still carries traffic is a different question from whether the host + accepts a new connection. +- **History or trend.** Only the most recent result per executor is retained, + plus the timestamp of the most recent success. No time series, uptime + percentage or incident log is kept. diff --git a/docs/specs/executors/system-design/ssh-reachability-surfaces.md b/docs/specs/executors/system-design/ssh-reachability-surfaces.md new file mode 100644 index 0000000000..419bdf0f4c --- /dev/null +++ b/docs/specs/executors/system-design/ssh-reachability-surfaces.md @@ -0,0 +1,233 @@ +--- +status: draft +system: executors +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-002 + - REQ-EXECUTORS-SSH-REACHABILITY-003 +created: 2026-09-13 +owners: + - tbd +--- +# SSH Host Reachability Surfaces System Design + +## Purpose and boundaries + +The [engine design](ssh-reachability.md) owns how a reachability record is +produced, decided and stored. This document owns how that record reaches a +user: the HTTP and WebSocket contracts, the web components that render it, and +the launch interaction. Nothing here decides reachability; every surface renders +state the engine defines. + +The task-card indicator and the launch path's record *write* are deferred. Both +are specified in the requirement document's `## Out of scope`, and their +criterion IDs are retired rather than reused. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-EXECUTORS-SSH-REACHABILITY-002` | [API and event contracts](#api-and-event-contracts), [Frontend components](#frontend-components) | +| `REQ-EXECUTORS-SSH-REACHABILITY-003` | [Launch interaction](#launch-interaction) | + +## API and event contracts + +Reachability is its own resource, not fields on the executor DTO, so a record +changing every interval does not invalidate the executor list payload. + +| Route | Purpose | +| --- | --- | +| `GET /api/v1/ssh/reachability` | Every SSH executor's current record in one request. | +| `GET /api/v1/ssh/executors/:id/reachability` | One executor's record. | +| `POST /api/v1/ssh/executors/:id/reachability/probe` | Run one probe now, persist it, return the record. | + +The record shape, mirrored in `apps/web/lib/types/http-ssh.ts`: + +```json +{ + "executor_id": "…", + "state": "unknown | reachable | unreachable", + "reason": "'' | config | network | timeout | auth | host_key | unknown", + "message": "…", + "consecutive_failures": 0, + "host": "…", + "checked_at": "RFC3339 | null", + "last_success_at": "RFC3339 | null", + "updated_at": "RFC3339 | null", + "probing_enabled": true, + "probe_interval_seconds": 60, + "persisted": true +} +``` + +`reason` is empty exactly when the most recent completed probe succeeded, or +when none has completed; a `reachable` or `unknown` state can therefore carry a +non-empty reason left by a below-threshold failure. It is a member of the type, +not an absent field, so a typed client is not surprised by the success shape. `probing_enabled` is `false` when the effective +interval is `0`, distinguishing "not probed yet" from "probing is off" without +inferring it from an absent timestamp. `probe_interval_seconds` carries the +**effective** interval after clamping, because the client cannot otherwise +satisfy the three-times-the-interval staleness rule or set its own refresh +cadence; without it both silently become hardcoded guesses that drift from the +operator's configuration. + +Status codes are the same on all three routes: `404` when no such executor +exists or it is soft-deleted, and `400` when the id names an executor that is +not of type `ssh`. The probe route adds `409` when the executor is `ssh` but its +`status` is not `active`; probing a deactivated executor on request would +contradict the rule that its retained record stays unchanged, so the route +refuses rather than inventing a third behavior. + +**Do not reuse `resolveSSHTarget`'s mapping wholesale.** That helper +(`internal/ssh/handlers.go`) answers `400` both for a wrong executor type *and* +for a failure to project the executor's config into a target — and a +configuration that will not resolve, or that carries no pinned fingerprint, is +exactly the case `AC-EXECUTORS-SSH-REACHABILITY-002.5` says must return an +outcome. A probe route built on it would answer `400` for precisely the broken +executors whose `config` reading matters most. Only the wrong-type branch maps +to `400` here; an unresolvable configuration is a `200` carrying reason +`config`. + +`GET` on an eligible executor with no record returns the `unknown` shape with +null timestamps, not `404`. `updated_at` is null there too: the shape is +synthesized, no row was written, and there is no version to report. A client +treats a null `updated_at` as older than any real record, so a synthesized +placeholder can never displace a record that actually exists. That covers an executor that exists and has simply +not been probed yet; a nonexistent, soft-deleted or non-`ssh` id takes the +status above, so a wrong id cannot be rendered to the user as "reachability +unknown". + +`GET /api/v1/ssh/reachability` returns one entry for every SSH executor that is +not soft-deleted, **ordered by `executor_id` ascending** — the same total order +the pass uses, and the slice keys by id anyway, so no tiebreak is needed. +Executors whose `status` is not `active` are included, carrying their retained +record, because `AC-EXECUTORS-SSH-REACHABILITY-001.17` preserves it and the +settings page for a deactivated executor still renders it. An executor never +probed appears in the `unknown` shape rather than being omitted, so the client +never has to distinguish "absent from the list" from "has no record". + +The immediate-probe route coalesces: a per-executor +`golang.org/x/sync/singleflight` group keyed by executor id makes overlapping +requests share one probe and one result. **The shared probe does not run on any +caller's request context.** It runs on a context derived from the reachability +package's own, carrying the probe deadline, so a caller that disconnects — a +closed tab, a cancelled fetch — neither cancels the probe nor fails the other +caller waiting on it, and the result is persisted either way. It is **not** +`context.Background()`: the probe is registered on the package `WaitGroup` and +cancelled by `Stop` like every other probe, which is what the engine design's +drain contract and its `goleak` guard require. A caller whose request already ended receives +nothing. A probe that ran but was not stored — the write failed, or a guard +refused it — still answers `200` with the outcome and `persisted: false`: the +reading is true even when its durability is not, and the client cannot act on the +difference. Only a failure to run the probe at all is an error status. + +Change propagation reuses the existing event bus to WebSocket bridge. A new +event `executor.reachability.changed` is published only when `state` or `reason` +differs from the stored record, subscribed in +`internal/gateway/websocket/task_notifications.go` alongside +`events.ExecutorUpdated`. Publishing per probe would put one message per executor +per interval on every client forever; publishing on change keeps a steady host +silent. The payload is the record above. The configuration reset uses the same +rule, so a state change has one propagation path. + +Because a silent steady state means `checked_at` ages in the client, the +settings page refetches **its own single record** once per effective interval +while open. That is a per-view refresh by the one client looking, not a +broadcast, so a steady host still produces no per-interval traffic to any other +client. + +When `probing_enabled` is `false` the effective interval is `0` and there is no +cadence to follow: the page fetches once on open and again after an immediate +probe, and runs no timer at all. Reading the interval literally in that state +would be a zero-delay refetch loop against the one record known not to be +changing. + +## Frontend components + +Everything here is `apps/web`. No component defines reachability; each renders +state the backend owns. + +**Types and client.** The record type lands in `apps/web/lib/types/http-ssh.ts` +next to `SSHTestResult`; the three routes are wrapped in +`apps/web/lib/api/domains/ssh-api.ts`, which already wraps SSH settings. + +**Store slice.** A `reachability` slice keyed by executor id, one record each. +It hydrates from `GET /api/v1/ssh/reachability` and applies +`executor.reachability.changed` events in place. Both inputs race — a refetch +issued before an event can answer after it — so the slice keeps whichever payload +carries the later **`updated_at`**. + +`updated_at`, not `checked_at`, is the reconciliation key, and the distinction is +load-bearing. The server orders *observations* by `checked_at`, which is when a +probe completed. A client orders *record versions*, and one record version has +no `checked_at` at all: the configuration reset clears it. Reconciling on +`checked_at` would make a reset — whose payload carries `checked_at: null` — +compare as older than every result it supersedes, so the client would discard it +and keep showing `reachable` or `unreachable` for an executor the user had just +re-pointed, in exactly the window this capability exists to illuminate. +`updated_at` advances on every write, the reset included, so it orders all of +them. + +**Settings page.** `apps/web/app/settings/executors/ssh/[executorId]/page.tsx` +gains `apps/web/components/settings/ssh-reachability-card.tsx` alongside the +existing `SSHConnectionCard` and `SSHSessionsCard`. It shows the state, probed +host, the reason and message when unreachable, the failure count, the age of the +last completed and last successful probe (or that none is recorded), the stale +marker, the probing-is-off notice, and the immediate-probe button. It owns the +once-per-effective-interval refetch above and is the only component to refetch. + +**Launch surfaces.** The backend always emits `session.launch.warning` for the +launched session; the session view renders it there, and a client that +initiated the launch itself also renders it inline at the initiation point, +naming the host and the last-success age. The gateway caches the latest warning +for each session and replays it to a later subscriber until the session reaches +`RUNNING` or is removed. The launch-failure attribution renders what the +backend reports; the frontend adds no text of its own. + +**Cross-cutting.** All copy goes through `t()` with keys in all five locales; +reason tokens render through a translated label map, never raw. State is text as +well as color. Every surface is verified against `/mobile-parity`. + +## Launch interaction + +The launch path is unchanged in its decision-making: `CreateInstance` reads no +reachability record and nothing consults one to decide whether to proceed. A +probe is evidence about the recent past, and a host can recover before a +launch. + +The launch path writes no reachability record either: the poller is the only +writer. Making a launch's own dial a probe observation is deferred, so a launch +that fails against a dead host updates no record, and the settings page can lag +that host by up to the failure threshold times the effective interval. + +**Where the warning is produced.** In +`internal/agent/runtime/lifecycle/manager_launch.go`, the shared launch path +starts a bounded, best-effort read immediately before its single +`rt.CreateInstance` call. The read runs asynchronously, so a slow repository +cannot delay the launch. `CreateInstance` remains free of any record read, and a +read failure produces no warning rather than blocking or retrying. + +**One mechanism, not two.** Rather than branch on whether a human is watching, +the warning is *always* published as a `session.launch.warning` event carrying +`executor_id`, `host`, `state`, `reason`, `last_success_at` (null when none has +been recorded), and a timestamp. A client that initiated that launch +interactively renders the same event inline as well. The gateway keeps the +latest warning per session and replays it to a later subscriber while the +backend is running. Every other path — a dependency chain, a workflow +transition, or an autostart — uses the same event path. One producer and one +payload keep all launch paths aligned. + +**When it is raised.** Only when the record's state is `unreachable`. While +periodic probing is enabled, any `unreachable` record qualifies. While probing +is disabled, only a record whose `checked_at` falls within three times the +*default* interval qualifies, per `AC-EXECUTORS-SSH-REACHABILITY-003.2`. An +operator who switched the poller off and then ran an immediate probe seconds ago +holds a current measurement and should be warned from it — +`AC-EXECUTORS-SSH-REACHABILITY-001.28` keeps that action working precisely so +that reading exists. A record retained from before the poller was switched off +measures nothing, and warning from it would train the user to distrust the +warning that matters. + +**The launch-failure message** names the target host and the reason classified +from **that launch's own connection attempt**, not the stored record, which may +predate the attempt by a full interval. Kandev never intercepts or edits agent +output; it adds its own attribution alongside. diff --git a/docs/specs/executors/system-design/ssh-reachability.md b/docs/specs/executors/system-design/ssh-reachability.md new file mode 100644 index 0000000000..39e6b1501b --- /dev/null +++ b/docs/specs/executors/system-design/ssh-reachability.md @@ -0,0 +1,588 @@ +--- +status: draft +system: executors +requirements: + - REQ-EXECUTORS-SSH-REACHABILITY-001 + - REQ-EXECUTORS-SSH-REACHABILITY-002 + - REQ-EXECUTORS-SSH-REACHABILITY-003 +created: 2026-09-07 +owners: + - tbd +--- +# SSH Host Reachability System Design + +## Purpose and boundaries + +The executor system owns SSH lifecycle and executor failure and recovery +contracts, so it owns the path answering "is this SSH host still reachable". +This design covers the poller, the probe, its classification, persistence and +configuration. The API, event and web projections are in +[SSH Host Reachability Surfaces](ssh-reachability-surfaces.md). + +Contracts this design uses but does not own: + +- `internal/agent/runtime/lifecycle` owns SSH target resolution + (`ResolveSSHTarget`), dialing (`DialSSH`), fingerprint pinning, `ProxyJump`, + and the dial error types. The probe consumes those, and does not reimplement. +- The task system owns `executors`, `executor_profiles`, and `task_sessions`, + and the event bus to WebSocket bridge in `internal/gateway/websocket`. +- `internal/common/config` owns the operator configuration catalog. +- `internal/integrations/healthpoll` is precedent, not a dependency: its + `Prober` is shaped around one integration with many workspaces, while this + poller sweeps many independent hosts. Its conventions are kept, not imported. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-EXECUTORS-SSH-REACHABILITY-001` | [The poller](#the-poller), [The probe](#the-probe), [State transitions](#state-transitions), [Persistence](#persistence), [Configuration](#configuration) | +| `REQ-EXECUTORS-SSH-REACHABILITY-002` | [Surfaces design](ssh-reachability-surfaces.md) | +| `REQ-EXECUTORS-SSH-REACHABILITY-003` | [Surfaces design](ssh-reachability-surfaces.md) | + +The task-card indicator and the launch-path record write are deferred; see the +requirement document's `## Out of scope`. + +## Components and responsibilities + +**`internal/agent/runtime/lifecycle` (extended).** `SSHExecutor.HealthCheck` +keeps returning `nil`: the runtime is available even when an individual host is +not, and the registry's `HealthCheckAll` sweep is a per-runtime concept called +once from `Manager.Start`. Conflating per-host reachability with runtime health +there would mark the whole SSH runtime down because one of several hosts moved. + +This package gains **three** exported functions and the two types they need: + +- `SSHTargetFromExecutorConfig(map[string]string) (*SSHTarget, error)` — lifted + from `internal/ssh`; projects `executors.config` into the `SSHConnConfig` that + `ResolveSSHTarget` takes. That function needs the typed struct, not a map, so + without this the reachability package must import `internal/ssh` to read an + executor's own config. +- `ProbeSSHHost(ctx, target *SSHTarget, timeout time.Duration) SSHProbeOutcome` + — dials with the pinned fingerprint, closes immediately, runs no remote + command, and reports success, cancellation, target host, reason, and a + sanitized message. **It bounds the handshake on the derived probe context, + not only the TCP dial.** `ssh.NewClientConn` takes no context and + `ssh.ClientConfig.Timeout` covers only `ssh.Dial`'s own TCP connect, so a host + that completes TCP and then stalls the handshake would hold its goroutine + past `probeTimeout` and past `Stop`. `handshakeWithDeadline` sets a deadline + when the transport supports it and also closes the owner connection from a + context cancellation callback, which covers ProxyJump channel connections. +- `ClassifyDialError(err error) SSHReachabilityReason` — maps a dial error to + exactly one reason from the closed set. + +**Classification lives here, not in the reachability package, and that is the +load-bearing decision in this section.** The error types it must distinguish are +`lifecycle`'s own and unexported: `errHostKeyMismatch` is lowercase, so no other +package can `errors.As` it. Exporting the *classifier* rather than the *error +types* keeps that boundary intact and gives the launch path identical +classification without a second implementation. +`Reason` is a string enum whose members are exactly `config`, `network`, +`timeout`, `auth`, `host_key`, `unknown`, plus the empty string for success. + +**`internal/executors/reachability` (new).** Owns the poller, the concurrency +limiter, the hysteresis rule and the store interface: it enumerates eligible +executors, orders them, runs a bounded pass, calls `ProbeSSHHost`, applies the +state rule, writes results and publishes change events. It depends on a narrow +repository interface, not the full task repository. + +**`internal/task/repository/sqlite` (extended).** Owns the +`executor_reachability` table, its additive migration, its queries and the +explicit record delete in [Persistence](#persistence). + +**`internal/task/service` (extended).** Owns save detection, because it is the +only layer holding both the stored executor and the incoming change. +`Service.UpdateExecutor` (`service_resources.go`) loads the row, applies the +request and bumps `UpdatedAt`; `Service.CreateExecutor` is the creation call +site. Each notifies an `ExecutorSaveObserver` — an interface the service +declares and the reachability package implements, wired at construction, so the +service depends on an interface rather than on the reachability package. The +observer is handed the connection-configuration fields as they stood before the +save and as they stand after, compares exactly the fields Terminology names, +and on any difference performs the reset in [Persistence](#persistence) and +schedules the off-cycle probe. A create is a difference by definition. + +**`applyExecutorUpdates` mutates the loaded executor in place**, so the before +values must be copied out before it runs. Read afterwards they compare the row +against itself, every save looks like a no-op, the reset never fires, and a +re-pointed executor keeps reporting the previous host's state indefinitely. + +**`internal/ssh` and `apps/web`.** The HTTP and WebSocket projection, the web +components and the launch interaction. Specified in the +[surfaces design](ssh-reachability-surfaces.md), which owns them; this document +states no contract for them. + +## The poller + +A single ticker loop, started where the lifecycle manager and the integration +pollers start and stopped on backend shutdown. Both `Start` and `Stop` are +idempotent. Every probe the package starts is registered on one package-owned +`WaitGroup` and derives from one package context, so `Stop` cancels and awaits +*all* of them — the in-flight pass, an off-cycle probe after a save, a coalesced +immediate probe — not only the pass. That is what makes an immediate probe's +package-owned context safe: it outlives its caller but not the package. Draining +is bounded because every probe carries `probeTimeout` *and* because +`ProbeSSHHost` bounds the handshake phase as well as the TCP dial; without that +second half a stalled handshake outlives both its deadline and `Stop`. A `goleak` +`TestMain` guards the package, matching `healthpoll`. + +When the effective interval is `0` the poller does not start: no startup pass +and no ticker. The immediate-probe route still works, not going through the +poller loop. The package context, `WaitGroup` and semaphore are still created +and registered for cleanup; only the pass and ticker are skipped. A `Start` +returning early before creating them leaves immediate probes unowned at +shutdown, breaking the drain contract and the `goleak` guard. + +One pass: + +1. List eligible executors: `type = 'ssh'`, `deleted_at IS NULL`, + `status = 'active'`, capturing each row's `updated_at` with its id. +2. Sort ascending by the `executors.id` column. `id` is the primary key and is + never reused, so the order is total; no tiebreak is needed or defined. +3. Acquire the shared probe semaphore per executor, preserving dispatch order. +4. Await all workers, then release the pass. + +A `sync.Mutex`-guarded `passRunning` flag makes an overlapping tick a no-op: the +tick is dropped, not queued, and increments a skipped-pass counter. Queueing it +would compound the overload that caused the overlap, and the next tick carries +identical information. + +**The semaphore is process-global, not per-pass.** One `passConcurrency`-slot +semaphore, owned by the package, is acquired by every probe whoever asked for +it: a scheduled pass, an off-cycle probe after a save, an immediate-probe +request. Saving twenty executors in a script therefore opens at most +`passConcurrency` connections, not twenty. A launch's own `DialSSH` is not a +probe and never takes a slot: a launch must not queue behind background probing. + +Cost per pass is `N` TCP connections and handshakes for `N` hosts, at most +`passConcurrency` at a time, each bounded by `probeTimeout`. At the defaults +below, one host costs one handshake per minute. + +## The probe + +Per executor, inside its own `context.WithTimeout(ctx, probeTimeout)`: + +1. Project `executors.config` into an `SSHConnConfig` and resolve it with + `lifecycle.ResolveSSHTarget` — the same projection the existing handler + performs in `sshTargetFromExecutorConfig`, lifted into a shared helper so the + poller and the handlers cannot drift. A resolution failure classifies as + `config` and returns without touching the network. **`ResolveSSHTarget` does + not validate the pin, so the probe checks it separately** and returns `config` + when the resolved target's `PinnedFingerprint` is empty, before any dial. + Leaving this to resolution is a security defect rather than a gap: + `buildClientConfig`'s host-key callback returns `nil` unconditionally when the + pin is empty, so the probe would dial unpinned, accept whatever key answered, + and record a substituted host as `reachable`. +2. `lifecycle.ProbeSSHHost(ctx, target)`, dialing with the pinned fingerprint + set, so a changed host key fails the handshake rather than being silently + accepted. That is the substantive difference from the manual test path, which + dials unpinned to observe a fingerprint for first-time trust. +3. On success the client is closed immediately and the outcome is `reachable`. + +`ClassifyDialError` walks a total order and returns the first match. The table +below is written **in that order**, and the order is the contract: + +| Order | Reason | Condition | +| --- | --- | --- | +| 1 | `config` | Target resolution failed, or resolved with no pinned fingerprint, before any dial. | +| 2 | `timeout` | The probe deadline expired, or the error unwraps to a net timeout. | +| 3 | `host_key` | The handshake failed host-key verification against the pin. | +| 4 | `auth` | Transport negotiation completed; authentication failed. | +| 5 | `network` | TCP dial failed: refused, no route, DNS, network unreachable. | +| 6 | `unknown` | Any other error. | + +`config` is decided before any dial. `timeout` precedes `network` because a +deadline expiry surfaces as a dial error and would otherwise read as a refused +connection. `host_key` precedes `auth` because both arrive as handshake failures +and only `host_key` is a security event. The stored message is the error text, +which `dialDirect` already stamps with host and port. + +## State transitions + +Three states, and one counter that is part of the contract rather than an +implementation detail: + +- `unknown`: no record, or a record explicitly reset. Never inferred. +- `reachable`: the last probe succeeded. +- `unreachable`: a sticky failure, or a transient failure streak that reached + `failureThreshold`. + +**On success:** `state = reachable`, `consecutive_failures = 0`, reason and +message cleared, `last_success_at = checked_at`. One success is enough from any +prior state: a host that just answered is reachable, and delaying that +conclusion only prolongs a false alarm. + +**On a sticky failure (`config` or `host_key`):** `state = unreachable` +immediately, whatever the counter says. The counter still increments and +`last_success_at` is left alone. Both conditions are deterministic and will +repeat on the next probe, so the threshold would buy no confidence — it would +only delay a possible host-key compromise by a full interval. + +**On a transient failure (`network`, `timeout`, `auth`, `unknown`):** +`consecutive_failures += 1`, and the state becomes `unreachable` on the probe +where the counter reaches `failureThreshold`. Until then the state is left +**exactly as it was**: `reachable` stays `reachable` and `unknown` stays +`unknown`. A failing probe never *promotes* a state, so it cannot turn `unknown` +into `reachable`; it changes the reason and the counter while the state stands +still. That pair, a non-empty reason under `reachable` or `unknown`, is the +intended representation of a below-threshold failure, not a contradiction: one +dropped packet must not paint a healthy host as down, while recovery is reported +at once. + +A save that changes the connection configuration resets the record to `unknown`, +then probes out of band if the executor is still eligible, because the prior +streak and success describe a different target. A save that changes nothing +resets nothing; the mechanism is in [Persistence](#persistence). + +## Persistence + +New table, one row per SSH executor: + +```sql +CREATE TABLE IF NOT EXISTS executor_reachability ( + executor_id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'unknown', + reason TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + consecutive_failures INTEGER NOT NULL DEFAULT 0, + host TEXT NOT NULL DEFAULT '', + checked_at TIMESTAMP, + last_success_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL +); +``` + +`last_success_at` exists because the pre-launch warning must name the age of the +last *successful* probe, and `checked_at` is overwritten by every failure. It is +`NULL` until the first success, and only the configuration reset clears it. + +A separate table rather than columns on `executors`: `config` is user-authored +input and `status` a user-controlled switch, so observed state must not share +either, and a per-interval write stays off the row every read touches. + +**Deletion is explicit, not a cascade.** `Repository.DeleteExecutor` is a *soft* +delete (`UPDATE executors SET deleted_at = ?, updated_at = ?`), so no foreign-key +cascade can fire and the table declares none. `DeleteExecutor` issues `DELETE +FROM executor_reachability WHERE executor_id = ?` in the same transaction as the +soft delete, and is the only call site that removes a record. + +**One statement decides every observation.** The write is a single upsert; the caller +supplies the observation (reason, message, host, `checked_at`) and the statement +derives the counter and the state from the row's own prior values, so no +read-modify-write in Go can lose a streak to an interleaving: + +```sql +INSERT INTO executor_reachability (executor_id, state, reason, message, + consecutive_failures, host, checked_at, last_success_at, updated_at) +SELECT :id, :state, :reason, :message, :failures, :host, :checked_at, + :last_success_at, :now + WHERE EXISTS (SELECT 1 FROM executors e + WHERE e.id = :id AND e.deleted_at IS NULL + AND e.status = 'active' AND e.updated_at = :seen_updated_at) +ON CONFLICT (executor_id) DO UPDATE SET + consecutive_failures = CASE WHEN :reason = '' THEN 0 + ELSE executor_reachability.consecutive_failures + 1 END, + state = CASE WHEN :reason = '' THEN 'reachable' + WHEN :reason IN ('config','host_key') THEN 'unreachable' + WHEN executor_reachability.consecutive_failures + 1 >= :threshold + THEN 'unreachable' + ELSE executor_reachability.state END, + reason = :reason, message = :message, host = :host, + checked_at = :checked_at, + last_success_at = CASE WHEN :reason = '' THEN :checked_at + ELSE executor_reachability.last_success_at END, + updated_at = :now +WHERE executor_reachability.checked_at IS NULL + OR excluded.checked_at > executor_reachability.checked_at; +``` + +On the insert path there is no prior row, so the caller supplies the initial +`state` and `failures`; every later write derives both in SQL from the stored +row. The `ON CONFLICT` `WHERE` is last-write-wins on `checked_at`: a scheduled +pass and an immediate probe racing on one executor converge on the later +observation rather than on whichever transaction committed second, and an equal +timestamp is not later, so the incoming row loses. `checked_at` is stored at +millisecond resolution or finer, keeping the tie rule a backstop. + +**That `>` is a text comparison, so the encoding is part of the contract.** +`TIMESTAMP` takes NUMERIC affinity and the driver binds `time.Time` as text +(`2006-01-02 15:04:05.999999999-07:00`), so SQLite orders these lexically, not +temporally. Two rules make lexical order equal time order, and both are +required: **normalize to UTC** before writing, because the driver keeps whatever +offset it is handed and a DST fall-back then sorts an earlier observation later; +and **bind `time.Time`, never a pre-formatted string**, because RFC3339 renders +UTC as `Z`, which sorts after `.` and would let a whole-second observation beat +a later sub-second one. Postgres compares temporally and is unaffected, which is +why this belongs in the SQLite tests: the parity test passes on Postgres while +SQLite keeps the wrong row. + +The eligibility guard is the `EXISTS` above, and `:seen_updated_at` is the +`updated_at` the probe captured before it dialed. Every mutation of the executor +row — configuration save, status change, soft delete — bumps `updated_at`, so one +comparison covers all three: it stops a probe in flight against the *old* target +from landing after a reset, and stops a worker from re-creating a deleted record +or writing one for an executor deactivated mid-pass. The `EXISTS` gates the +insert; the `ON CONFLICT` arm inherits it, because a row can only conflict if the +insert was admitted. A refused write is logged at debug and dropped. + +**The reset is a second statement, and deliberately not that one.** A +configuration reset is not an observation; it invalidates one. The upsert above +cannot express it — its `state` CASE has no `unknown` branch, and its `WHERE` +admits only a strictly later `checked_at`, which a reset does not carry. Nor may +it be expressed as a delete: `DeleteExecutor` is the only call site permitted to +remove a row. + +```sql +INSERT INTO executor_reachability (executor_id, state, reason, message, + consecutive_failures, host, checked_at, last_success_at, updated_at) +SELECT :id, 'unknown', '', '', 0, :host, NULL, NULL, :now + WHERE EXISTS (SELECT 1 FROM executors e + WHERE e.id = :id AND e.type = 'ssh' + AND e.deleted_at IS NULL AND e.status = 'active') +ON CONFLICT (executor_id) DO UPDATE SET + state = 'unknown', reason = '', message = '', consecutive_failures = 0, + host = :host, checked_at = NULL, last_success_at = NULL, updated_at = :now; +``` + +The `EXISTS` carries a decision, not just a safety check: an executor whose +`status` is not `active` is **not** reset by a save, because +`AC-EXECUTORS-SSH-REACHABILITY-001.17` retains its record unchanged and +eligibility wins over the reset trigger. The statement lands zero rows and the +retained record stands until the executor is reactivated and probed. + +`:host` is the newly saved host, so the settings page never renders a pre-change +host beside `unknown` — the window this capability exists to illuminate is +exactly the one that would otherwise lie. There is deliberately no `checked_at` +guard and no `:seen_updated_at` pin: a reset is ordered by the save that caused +it, not by an observation clock, so it must win over whatever is stored. A probe +already in flight against the pre-save target cannot overwrite it afterwards, +because that probe's own write still carries the pre-save `updated_at` and the +upsert's `EXISTS` rejects it. That is the mechanism +`AC-EXECUTORS-SSH-REACHABILITY-001.19` requires, and it is why the reset needs no +clock of its own. + +The reset advances the row's `updated_at` like any other write, which is what +lets a client order it against a concurrent probe result: see the surfaces +design's reconciliation rule. + +Retention is one row per executor. The table survives restart and the surface +reports the preserved record until a new probe replaces it: a blank panel after +every restart would be worse than a timestamped result marked stale. + +The migration is a single additive `CREATE TABLE IF NOT EXISTS` applied through +the existing idempotent mechanism in `internal/task/repository/sqlite`, so it is +safe on SQLite and Postgres and on an already-migrated database. + +## Configuration + +One catalog entry in `internal/common/config/catalog.go`: + +| Key | Environment variable | Default | +| --- | --- | --- | +| `executors.sshReachabilityIntervalSeconds` | `KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS` | `60` | + +Supported range 15 to 3600 seconds; `0` disables the poller. **The catalog does +not clamp.** `applyNonNegativeIntEnv` falls back to the *default* — never to a +bound — for a value that is absent, empty, negative, fractional or unparsable, so +every one of those yields `60`. Its minimum must stay `0`: widening it to `15` to +express the supported range would push `0` below the minimum and silently turn +the documented kill switch into a 60-second cadence. + +**That normalization is the environment path only, and the two sources do not +behave alike.** `applyBoundedIntEnv`, which `applyNonNegativeIntEnv` wraps, reads +`nonEmptyEnv(env, entry.EnvVars...)` and returns early when no environment +variable is set, so a value written in `config.yaml` never reaches its +fallback-and-bounds logic at all. A YAML value is decoded by `decodeConfig`, +which sets no `WeaklyTypedInput`, and `Load` returns the decode error, so a +non-integer in `config.yaml` refuses boot exactly as it does for every other +typed key in the catalog. That is the documented behavior rather than an +exception carved out for this key, and +`AC-EXECUTORS-SSH-REACHABILITY-001.24` states it that way instead of promising a +leniency the loader does not implement. A *negative whole number* does decode +cleanly and so does reach the package, which is why the clamp below needs a +negative branch of its own. + +Clamping is therefore the package's job, applied to whatever value reaches it: +below `0` yields the default `60`, `0` disables, `1`-`14` becomes `15`, above +`3600` becomes `3600`, and a clamp logs once at startup rather than refusing to +boot. + +**The clamped value is the effective interval, and every consumer uses it.** +Staleness (three times the interval), the `probing_enabled` projection, and the +settings page's own refresh cadence all read the effective interval, never the +raw configured number. An operator who writes `5` gets a 15-second cadence and a +45-second staleness horizon, consistently across the backend and the UI. + +The other three knobs are compile-time constants, not configuration, because no +operator scenario has been identified that needs them independently and each new +key is a permanent contract: + +| Constant | Value | Rationale | +| --- | --- | --- | +| `probeTimeout` | 10s | `sshDialTimeout` is 30s, which would let one hung host hold a slot for half the interval. | +| `passConcurrency` | 4 | Bounds simultaneous outbound connections and file descriptors while keeping a pass short. | +| `failureThreshold` | 2 | Smallest value that suppresses a single-probe blip; worst-case detection is two intervals. | + +This capability adds no runtime feature toggle: the interval key already +provides the kill switch through the designated operator-startup mechanism, and +a toggle would leave a retired identity to carry forever. + +## Control flow + +```text +ticker -> poller.pass() (only if effective interval > 0) + | list eligible SSH executors by executors.id, with updated_at + v +global semaphore(passConcurrency) <- off-cycle and immediate probes + v +probe() SSHTargetFromExecutorConfig > ProbeSSHHost > ClassifyDialError + v +store.Upsert(observation) guards: eligible, updated_at, checked_at + |- state or reason changed -> bus.Publish -> WS -> store slice + '- unchanged -> stop + +save -> ExecutorSaveObserver (do the connection-config fields differ?) + '- yes -> store.Reset(host) guard: eligible -> bus.Publish -> off-cycle probe + +launch -> session creation reads record, emits session.launch.warning + '- SSHExecutor.CreateInstance (reads no record, writes no record) + '- failed at its dial -> ClassifyDialError -> error names host +``` + +## Failure and recovery + +The poller is best-effort and never a source of truth for a caller. Every +failure mode degrades to a stale or `unknown` record and a log line: + +- Listing executors fails: the pass is abandoned with a warning, nothing is + written, the next tick retries. +- A probe panics or errors unexpectedly: it classifies `unknown` and the pass + continues; no other executor is affected. +- A record write fails: logged, the pass completes, the next probe rewrites, + costing one interval of freshness. +- A write refused by a guard is expected, not an error: logged at debug and + dropped. +- A probe exceeding `probeTimeout` is abandoned by its context; the client is + closed on the returning path so it leaks no connection, and its slot released. +- A probe cancelled by `Stop` writes **nothing**, which is not the case above. + Shutdown cancels the package context, so an in-flight dial returns + `context.Canceled`, and no guard catches it: a shutdown mutates no executor + row, so `EXISTS` and `updated_at` still admit the write. Recorded, it would + increment `consecutive_failures` for every probe in flight — up to + `passConcurrency` hosts per restart — and at `failureThreshold` a second + restart would flip a healthy host to `unreachable`, turning a routine deploy + into exactly the false alarm this capability exists to prevent. + `ClassifyDialError` must therefore separate the two context errors: only a + deadline is a `timeout` observation, and a cancellation is not an observation. +- The store is unreachable on an API read: the route errors and the surface + reports reachability as not known, never `unreachable`. + +There are no retries inside a pass: the next tick is the retry, and an immediate +retry against a host that just refused a connection adds cost, not information. + +## Security + +The probe uses the pinned fingerprint on the dial to the **target**, so a +host-key change there is a detected `host_key` failure and never a silent +re-pin; the stored fingerprint is read-only to this path. `host_key` reports +`unreachable` on its first occurrence rather than waiting out the threshold: a +mismatch is possible interception, and delaying it a full interval is a security +cost with no accuracy benefit. Credentials are unchanged: the probe reuses the +executor's configured identity source and holds no secret of its own. + +`message` carries an SSH error string, which can include host, port, username +and identity path — all values the user configured and can already see on the +same settings page, so the record is not sensitive. `ClassifyDialError` must not +place key material or agent socket contents into `message`. + +The probe opens no remote shell and runs no remote command, so it grants nothing +beyond what the manual test already exercises, and adds no new trust boundary. + +**`ProxyJump` is a second hop with a weaker trust model, and the reason set must +not paper over it.** An executor pins one fingerprint and it belongs to the +target; `dialViaJump` verifies the bastion against `~/.ssh/known_hosts` instead, +accepting an absent host with a logged warning (OpenSSH's +`StrictHostKeyChecking=accept-new`). Two consequences the probe inherits: + +- A bastion key *mismatch* is rejected, but surfaces as `ssh: bastion dial: …`, + not as `errHostKeyMismatch`, so `ClassifyDialError` would call it `network` or + `unknown` and wait out `failureThreshold` under the wrong reason. The + classifier must map a bastion host-key rejection to `host_key`, which + `AC-EXECUTORS-SSH-REACHABILITY-001.29` now requires observably. +- Every bastion-path error is about the *bastion*, while the record's `host` is + the target. A bastion that is down reports the target unreachable and sends an + operator to a machine that is fine — the same misattribution as the firewall + message of 2026-09-06, from a different cause. **`host` stays the target in + every case**: it identifies the executor's subject, the column keeps one + meaning for success and failure alike, and `AC-…-001.4` and `AC-…-002.1` stay + true for a probe that succeeds through a jump. The disambiguation belongs in + `message`, which must name the bastion and state that the failure arose on the + jump hop whenever it did. `message` is rendered beside the state wherever an + unreachable record is shown, so the operator reads "unreachable … via bastion + X" instead of being sent to a healthy target. + +## Observability + +Structured `zap` logs plus `expvar` counters under `/debug/vars`, following the +`office_stall_*` and `routing_*` precedent. Every name below carries the prefix +`executor_ssh_reachability_`: + +- `probe_total`, labelled by outcome (`reachable` plus each failure reason). +- `state_transitions_total`, by destination state. +- `pass_skipped_total`, when a tick is dropped because a pass is still running: + the interval is shorter than a pass. +- `write_refused_total`, when a guard drops a write. A rising value means + configuration churn, not a fault. +- `reset_total`, when a save invalidates a record. +- `probe_discarded_total`, when `Stop` cancelled a probe. `probe_total` is + labelled by outcome and a cancelled probe has none, so without this the + shutdown discard is the package's only uncounted drop. +- `probe_duration_ms`, the last pass's aggregate probe duration. + +A transition logs at `Warn` for `reachable` to `unreachable` and `Info` for the +reverse, with executor id, host, reason and failure count. Unchanged results do +not log, so a steady host stays silent. + +## Prior art and departures + +`internal/integrations/healthpoll` is the closest in-repo precedent: an +immediate probe on start, a fixed cadence, a configured-or-skip gate, +best-effort semantics, idempotent `Start`/`Stop`, and a goroutine-leak test. +Those conventions are kept. Its persistence shape — a timestamp, a boolean and an +error string on the config row — is not, in four places, because an SSH host is +not an HTTP API: a **closed reason set** rather than an error string, since five +failures are acted on differently and only a typed reason lets the UI say +"fingerprint changed"; **asymmetric hysteresis** rather than flipping on the +first result, which would let one dropped packet paint a healthy host as down +(see [State transitions](#state-transitions)); a **separate record** rather than +a column, since observed state must not share a row with user-authored input +(see [Persistence](#persistence)); and **changes pushed, results not**, since +pushing every result would put one message per executor per interval on the wire +forever. + +## Testing + +Strategy only; the plan's `## Tests` table owns the per-criterion evidence. + +- Poller tests drive the ticker with `testing/synctest`, matching + `healthpoll_test.go`, under a `goleak` `TestMain`. +- Classification and hysteresis are table-driven — over constructed errors and + over probe sequences — so the ordering rules and the exact probe each state + flips on are asserted, not inferred. +- The upsert's ordering rule is exercised on **both** dialects, including the + whole-second versus sub-second pair the text-encoding trap turns into a silent + SQLite-only failure. +- The guards get a negative test each: soft-deleted, deactivated or reconfigured + mid-probe writes nothing. The reset gets its own: it lands with no + `checked_at`, and a probe in flight from before the save does not overwrite it. +- The immediate-probe route is tested for caller cancellation and for a + guard-refused write returning `persisted: false`. +- Web E2E lives in `apps/web/e2e/tests/ssh/` under the `containers` project, + gated on `KANDEV_E2E_CONTAINERS=1`. + +## Related decisions + +No new ADR: this design creates no system boundary and no ownership rule, and +extends an existing contract through the repository's designated mechanisms for +configuration, persistence, events and metrics. +([0018](../../../decisions/0018-runtime-settings-overrides.md) covers the +adjacent runtime-override tier.)