feat(executors): add continuous SSH host reachability probing (#3747)

Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com>
Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
Henry
2026-09-22 17:31:00 +08:00
committed by GitHub
parent 7b2a4c0712
commit c7cc92382b
121 changed files with 11067 additions and 170 deletions
@@ -256,7 +256,11 @@ func (r *SSHExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateReq
}
client, err := dialSSH(baseCtx, target)
if err != nil {
return nil, fmt.Errorf("ssh: connect to %s@%s: %w", target.User, target.Host, err)
// Classified from this launch's own attempt, never from a stored
// reachability record — that record can predate this dial by a full
// probe interval and would misattribute a stale cause to a fresh
// failure (see task-05-launch-non-gating.md).
return nil, fmt.Errorf("ssh: connect to %s@%s: %s: %w", target.User, target.Host, ClassifyDialError(err), err)
}
var runtimeAPITunnel *sshRuntimeAPITunnel
released := false
@@ -495,7 +495,7 @@ func dialDirect(ctx context.Context, addr string, cfg *ssh.ClientConfig) (*ssh.C
if err != nil {
return nil, fmt.Errorf("ssh: tcp dial %s: %w", addr, err)
}
sshConn, chans, reqs, err := ssh.NewClientConn(tcpConn, addr, cfg)
sshConn, chans, reqs, err := handshakeWithDeadline(ctx, tcpConn, addr, cfg)
if err != nil {
_ = tcpConn.Close()
return nil, fmt.Errorf("ssh: handshake with %s: %w", addr, err)
@@ -530,6 +530,37 @@ func (c *sshProxyJumpConn) Close() error {
return c.closeErr
}
// handshakeWithDeadline runs the SSH handshake over conn, bounding it by
// ctx's deadline when one is set. ssh.NewClientConn itself takes no context —
// without this, a peer that completes the TCP handshake but never speaks SSH
// (or stalls partway through key exchange) blocks the caller indefinitely.
// Callers that pass a context with no deadline see unchanged behavior.
func handshakeWithDeadline(ctx context.Context, conn net.Conn, addr string, cfg *ssh.ClientConfig) (ssh.Conn, <-chan ssh.NewChannel, <-chan *ssh.Request, error) {
cancelDone := make(chan struct{})
stopCancellation := context.AfterFunc(ctx, func() {
// ProxyJump tunnels wrap an x/crypto/ssh channel. That channel
// reports unsupported deadlines, so closing the owner transport is
// the only reliable way to unblock NewClientConn on cancellation.
_ = conn.Close()
close(cancelDone)
})
defer func() {
if !stopCancellation() {
<-cancelDone
}
}()
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
defer func() { _ = conn.SetDeadline(time.Time{}) }()
}
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg)
if ctxErr := ctx.Err(); ctxErr != nil {
_ = conn.Close()
return nil, nil, nil, ctxErr
}
return sshConn, chans, reqs, err
}
// dialViaJump implements ProxyJump as a single bastion hop. The bastion is
// resolved from its own ~/.ssh/config Host block, defaulting to the same
// identity source as the target (passes the user's agent / key through).
@@ -568,7 +599,7 @@ func dialViaJump(ctx context.Context, target *SSHTarget, finalAddr string, final
return nil, fmt.Errorf("ssh: bastion tunnel to %s: %w", finalAddr, err)
}
jumpConn := &sshProxyJumpConn{Conn: tunnel, bastion: bastionClient}
sshConn, chans, reqs, err := ssh.NewClientConn(jumpConn, finalAddr, finalCfg)
sshConn, chans, reqs, err := handshakeWithDeadline(ctx, jumpConn, finalAddr, finalCfg)
if err != nil {
_ = jumpConn.Close()
return nil, fmt.Errorf("ssh: handshake with %s via %s: %w", finalAddr, target.ProxyJump, err)
@@ -2,6 +2,8 @@ package lifecycle
import (
"context"
"errors"
"net"
"net/http"
"os"
"os/exec"
@@ -17,6 +19,39 @@ import (
"golang.org/x/crypto/ssh"
)
type unsupportedDeadlineConn struct{ net.Conn }
func (unsupportedDeadlineConn) SetDeadline(time.Time) error {
return errors.New("deadlines are not supported")
}
func TestHandshakeWithDeadlineClosesConnectionsWhenDeadlinesAreUnsupported(t *testing.T) {
clientConn, serverConn := net.Pipe()
defer func() { _ = serverConn.Close() }()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
done := make(chan error, 1)
go func() {
_, _, _, err := handshakeWithDeadline(ctx, unsupportedDeadlineConn{Conn: clientConn}, "test", &ssh.ClientConfig{
User: "test",
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
})
done <- err
}()
select {
case err := <-done:
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("handshake error = %v, want context deadline exceeded", err)
}
case <-time.After(500 * time.Millisecond):
_ = clientConn.Close()
<-done
t.Fatal("handshake did not stop when its context deadline expired")
}
}
func TestSSHControlRequestUsesBearerToken(t *testing.T) {
req, err := http.NewRequest(http.MethodPost, "http://127.0.0.1/api/v1/instances", nil)
if err != nil {
@@ -0,0 +1,45 @@
package lifecycle
import (
"context"
"strconv"
"strings"
"testing"
)
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2
//
// A launch's own dial failure must be attributed with the target host and
// the same reason ClassifyDialError would produce for a standalone probe —
// not read from any stored record, since the stored record can predate this
// attempt by a full interval.
func TestSSHExecutorCreateInstanceDialErrorNamesHostAndReason(t *testing.T) {
server := newFakeSSHServer(t, nil)
newTestHomeDir(t)
identity := writeTestIdentityFile(t)
exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger())
t.Cleanup(func() { _ = exec.Close() })
_, err := exec.CreateInstance(context.Background(), &ExecutorCreateRequest{
InstanceID: "instance-1",
Metadata: map[string]interface{}{
MetadataKeySSHHost: "127.0.0.1",
MetadataKeySSHPort: strconv.Itoa(server.port()),
MetadataKeySSHUser: "kandev",
MetadataKeySSHHostFingerprint: "SHA256:not-the-server-key",
MetadataKeySSHIdentitySource: string(SSHIdentitySourceFile),
MetadataKeySSHIdentityFile: identity,
},
})
if err == nil {
t.Fatal("expected a dial failure, got nil error")
}
if !strings.Contains(err.Error(), "127.0.0.1") {
t.Fatalf("error = %q, want it to name the target host", err.Error())
}
if !strings.Contains(err.Error(), string(SSHReachabilityReasonHostKey)) {
t.Fatalf("error = %q, want it to carry the classified reason %q", err.Error(), SSHReachabilityReasonHostKey)
}
}
@@ -0,0 +1,188 @@
package lifecycle
import (
"context"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
"unicode/utf8"
"golang.org/x/crypto/ssh/knownhosts"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
)
const maxSSHReachabilityMessageBytes = 512
// SanitizeSSHReachabilityMessage keeps dial diagnostics useful while making
// sure credentials, local paths, and oversized provider text do not cross the
// reachability API boundary.
func SanitizeSSHReachabilityMessage(err error) string {
if err == nil {
return ""
}
message := routingerr.Sanitize(err.Error())
if len(message) <= maxSSHReachabilityMessageBytes {
return message
}
cut := maxSSHReachabilityMessageBytes - len("…")
for cut > 0 && !utf8.RuneStart(message[cut]) {
cut--
}
return message[:cut] + "…"
}
// SSHReachabilityReason classifies why a probe or launch-time SSH dial
// failed. The set is closed: ClassifyDialError always returns one of these
// six values, walking a total order so a deadline is never reported as a
// network failure and a rejected host key is never reported as an
// authentication failure.
type SSHReachabilityReason string
const (
SSHReachabilityReasonConfig SSHReachabilityReason = "config"
SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout"
SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key"
SSHReachabilityReasonAuth SSHReachabilityReason = "auth"
SSHReachabilityReasonNetwork SSHReachabilityReason = "network"
SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown"
)
// SSHProbeOutcome is the result of a single reachability probe, or of a
// launch's own dial attempt classified through the same rules. Host always
// names the target executor's host, even when a failure occurred on a
// bastion hop. Cancelled means the caller's context ended the attempt before
// it produced an observation about the host; callers must discard a
// cancelled outcome rather than recording it as a failure.
type SSHProbeOutcome struct {
Success bool
Cancelled bool
Host string
Reason SSHReachabilityReason
Message string
}
// ProbeSSHHost opens an authenticated SSH transport to target using its
// pinned fingerprint, closes it without running a remote command, and
// returns a typed outcome. It bounds both the TCP dial and the SSH handshake
// by timeout, and never dials when the target has no pinned fingerprint to
// verify against.
func ProbeSSHHost(ctx context.Context, target *SSHTarget, timeout time.Duration) SSHProbeOutcome {
outcome := SSHProbeOutcome{Host: target.Host}
if strings.TrimSpace(target.PinnedFingerprint) == "" {
outcome.Reason = SSHReachabilityReasonConfig
outcome.Message = "no pinned host key fingerprint is configured"
return outcome
}
probeCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
client, err := dialSSH(probeCtx, target)
if err != nil {
if errors.Is(err, context.Canceled) {
outcome.Cancelled = true
return outcome
}
outcome.Reason = ClassifyDialError(err)
outcome.Message = SanitizeSSHReachabilityMessage(err)
return outcome
}
defer func() { _ = client.Close() }()
outcome.Success = true
return outcome
}
// ClassifyDialError maps a non-nil SSH dial error to exactly one reason from
// the closed set, walking timeout, host_key, auth, network, unknown in that
// order — first match wins. context.Canceled is deliberately not classified
// here: it is not an observation about the host, and callers check for it
// (see ProbeSSHHost) before reaching this function.
func ClassifyDialError(err error) SSHReachabilityReason {
switch {
case isSSHTimeoutError(err):
return SSHReachabilityReasonTimeout
case isSSHHostKeyMismatch(err):
return SSHReachabilityReasonHostKey
case isSSHAuthFailure(err):
return SSHReachabilityReasonAuth
case isSSHNetworkError(err):
return SSHReachabilityReasonNetwork
default:
return SSHReachabilityReasonUnknown
}
}
func isSSHTimeoutError(err error) bool {
if errors.Is(err, context.DeadlineExceeded) {
return true
}
var netErr net.Error
return errors.As(err, &netErr) && netErr.Timeout()
}
// isSSHHostKeyMismatch confines the host_key discriminator to Kandev's own
// errHostKeyMismatch (the pinned-fingerprint check) and knownhosts.KeyError
// with a non-empty Want (a genuine mismatch, not an unknown-host TOFU
// accept, which returns nil and never reaches this function) — the bastion
// host-key check. golang.org/x/crypto/ssh has no stable exported error type
// for a rejected key; keying off our own types keeps a library upgrade from
// turning a security event into auth or unknown.
func isSSHHostKeyMismatch(err error) bool {
var mismatch *errHostKeyMismatch
if errors.As(err, &mismatch) {
return true
}
var keyErr *knownhosts.KeyError
return errors.As(err, &keyErr) && len(keyErr.Want) > 0
}
// isSSHAuthFailure relies on string-matching because the pinned
// golang.org/x/crypto/ssh v0.57.0 dependency has no exported error type for
// an authentication failure — it returns a plain fmt.Errorf. Revisit this
// classifier when upgrading the dependency if it adds a stable type.
func isSSHAuthFailure(err error) bool {
return strings.Contains(err.Error(), "unable to authenticate")
}
func isSSHNetworkError(err error) bool {
var opErr *net.OpError
return errors.As(err, &opErr)
}
// SSHTargetFromExecutorConfig projects an SSH executor's persisted config map
// into a resolved SSHTarget, applying the same ~/.ssh/config inheritance as
// ResolveSSHTarget. A missing host or fingerprint fails resolution here,
// which callers should classify as SSHReachabilityReasonConfig without
// attempting a dial.
func SSHTargetFromExecutorConfig(cfg map[string]string) (*SSHTarget, error) {
if cfg == nil {
return nil, fmt.Errorf("ssh executor has no config")
}
fingerprint := strings.TrimSpace(cfg["ssh_host_fingerprint"])
if fingerprint == "" {
return nil, fmt.Errorf("ssh host fingerprint is required for reachability probes")
}
port := 0
if p := strings.TrimSpace(cfg["ssh_port"]); p != "" {
n, err := strconv.Atoi(p)
if err != nil || n < 1 || n > 65535 {
return nil, fmt.Errorf("invalid ssh_port %q", p)
}
port = n
}
return ResolveSSHTarget(SSHConnConfig{
HostAlias: cfg["ssh_host_alias"],
Host: cfg["ssh_host"],
Port: port,
User: cfg["ssh_user"],
IdentitySource: SSHIdentitySource(cfg["ssh_identity_source"]),
IdentityFile: cfg["ssh_identity_file"],
ProxyJump: cfg["ssh_proxy_jump"],
PinnedFingerprint: fingerprint,
})
}
@@ -0,0 +1,372 @@
package lifecycle
import (
"context"
"crypto/ed25519"
"crypto/rand"
"errors"
"fmt"
"net"
"strconv"
"strings"
"sync"
"testing"
"time"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
// fakeTimeoutError is a minimal net.Error whose Timeout() reports true,
// wrapped inside a *net.OpError below to model a timeout that is also
// structurally shaped like a network dial error — @covers
// AC-EXECUTORS-SSH-REACHABILITY-001.27 (timeout must win the ordering, not
// fall through to network).
type fakeTimeoutError struct{}
func (fakeTimeoutError) Error() string { return "fake timeout" }
func (fakeTimeoutError) Timeout() bool { return true }
func (fakeTimeoutError) Temporary() bool { return true }
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.27 — classification order
// config, timeout, host_key, auth, network, unknown, first match wins,
// including the two explicit ordering cases.
func TestClassifyDialError(t *testing.T) {
tests := []struct {
name string
err error
want SSHReachabilityReason
}{
{
name: "a context deadline classifies as timeout",
err: fmt.Errorf("ssh: tcp dial x: %w", context.DeadlineExceeded),
want: SSHReachabilityReasonTimeout,
},
{
name: "a timeout-shaped dial error wins over the network fallback",
err: &net.OpError{Op: "dial", Net: "tcp", Err: fakeTimeoutError{}},
want: SSHReachabilityReasonTimeout,
},
{
name: "a pinned host key mismatch classifies as host_key",
err: fmt.Errorf("ssh: handshake with x: %w", &errHostKeyMismatch{Expected: "A", Got: "B"}),
want: SSHReachabilityReasonHostKey,
},
{
name: "a bastion known_hosts mismatch classifies as host_key",
err: fmt.Errorf("ssh: bastion dial: %w", &knownhosts.KeyError{Want: []knownhosts.KnownKey{{}}}),
want: SSHReachabilityReasonHostKey,
},
{
name: "host_key wins over an auth-shaped message in the same error",
err: fmt.Errorf("ssh: handshake failed: ssh: unable to authenticate: %w",
&errHostKeyMismatch{Expected: "A", Got: "B"}),
want: SSHReachabilityReasonHostKey,
},
{
name: "an authentication failure classifies as auth",
err: errors.New("ssh: unable to authenticate, attempted methods [publickey], no supported methods remain"),
want: SSHReachabilityReasonAuth,
},
{
name: "a connection-refused dial error classifies as network",
err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")},
want: SSHReachabilityReasonNetwork,
},
{
name: "an unrecognized error classifies as unknown",
err: errors.New("something unexpected happened"),
want: SSHReachabilityReasonUnknown,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ClassifyDialError(tt.err); got != tt.want {
t.Fatalf("ClassifyDialError(%v) = %q, want %q", tt.err, got, tt.want)
}
})
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.4, 001.5
func TestProbeSSHHostSuccess(t *testing.T) {
server := newFakeSSHServer(t, nil)
newTestHomeDir(t)
identity := writeTestIdentityFile(t)
target := &SSHTarget{
Host: "127.0.0.1", Port: server.port(), User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
PinnedFingerprint: server.hostKeyFingerprint(),
}
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
if !outcome.Success {
t.Fatalf("outcome = %+v, want Success", outcome)
}
if outcome.Host != "127.0.0.1" {
t.Fatalf("Host = %q", outcome.Host)
}
if outcome.Reason != "" || outcome.Message != "" {
t.Fatalf("a successful outcome must carry no reason/message: %+v", outcome)
}
if len(server.execCalls()) != 0 {
t.Fatalf("execCalls = %v, want the probe to open no session channel", server.execCalls())
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.5
func TestProbeSSHHostHostKeyMismatch(t *testing.T) {
server := newFakeSSHServer(t, nil)
newTestHomeDir(t)
identity := writeTestIdentityFile(t)
target := &SSHTarget{
Host: "127.0.0.1", Port: server.port(), User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
PinnedFingerprint: "SHA256:not-the-server-key",
}
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
if outcome.Success {
t.Fatalf("outcome = %+v, want failure", outcome)
}
if outcome.Reason != SSHReachabilityReasonHostKey {
t.Fatalf("Reason = %q, want host_key", outcome.Reason)
}
if target.PinnedFingerprint != "SHA256:not-the-server-key" {
t.Fatalf("PinnedFingerprint mutated: %q", target.PinnedFingerprint)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.6
func TestProbeSSHHostEmptyPinDoesNotDial(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
dialed := false
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
conn, err := ln.Accept()
if err == nil {
dialed = true
_ = conn.Close()
}
}()
t.Cleanup(func() {
_ = ln.Close()
wg.Wait()
})
addr := ln.Addr().(*net.TCPAddr)
target := &SSHTarget{
Host: addr.IP.String(), Port: addr.Port, User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: "/does/not/matter",
PinnedFingerprint: "",
}
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
if outcome.Reason != SSHReachabilityReasonConfig {
t.Fatalf("Reason = %q, want config", outcome.Reason)
}
if outcome.Success {
t.Fatal("outcome must not be a success")
}
_ = ln.Close()
wg.Wait()
if dialed {
t.Fatal("ProbeSSHHost dialed TCP despite an empty pinned fingerprint")
}
}
// newStallHandshakeListener accepts exactly one TCP connection and holds it
// open without ever writing the SSH version banner, so a client's handshake
// blocks on read until its own deadline fires. Used to prove the probe
// bounds the handshake, not only the TCP dial — @covers
// AC-EXECUTORS-SSH-REACHABILITY-001.7.
func newStallHandshakeListener(t *testing.T) *net.TCPAddr {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
done := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
conn, err := ln.Accept()
if err != nil {
return
}
<-done
_ = conn.Close()
}()
t.Cleanup(func() {
close(done)
_ = ln.Close()
wg.Wait()
})
return ln.Addr().(*net.TCPAddr)
}
func TestProbeSSHHostHandshakeStallIsBoundedByTheDeadline(t *testing.T) {
newTestHomeDir(t)
identity := writeTestIdentityFile(t)
addr := newStallHandshakeListener(t)
target := &SSHTarget{
Host: addr.IP.String(), Port: addr.Port, User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
PinnedFingerprint: "SHA256:irrelevant",
}
const probeTimeout = 300 * time.Millisecond
start := time.Now()
done := make(chan SSHProbeOutcome, 1)
go func() { done <- ProbeSSHHost(context.Background(), target, probeTimeout) }()
select {
case outcome := <-done:
elapsed := time.Since(start)
if elapsed > 5*time.Second {
t.Fatalf("probe took %s, want it bounded near the %s deadline", elapsed, probeTimeout)
}
if outcome.Reason != SSHReachabilityReasonTimeout {
t.Fatalf("Reason = %q, want timeout; message=%q", outcome.Reason, outcome.Message)
}
case <-time.After(5 * time.Second):
t.Fatal("ProbeSSHHost did not return within 5s of a 300ms deadline — handshake is unbounded")
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.29
func TestProbeSSHHostBastionHostKeyMismatchClassifiesAsHostKey(t *testing.T) {
home := newTestHomeDir(t)
identity := writeTestIdentityFile(t)
final := newFakeSSHServer(t, nil)
bastion := newFakeSSHServer(t, nil)
_, otherPriv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatalf("generate other key: %v", err)
}
otherSigner, err := ssh.NewSignerFromKey(otherPriv)
if err != nil {
t.Fatalf("other signer: %v", err)
}
writeKnownHosts(t, home, fmt.Sprintf("[127.0.0.1]:%d", bastion.port()), otherSigner.PublicKey())
proxyJump := "jump@127.0.0.1:" + strconv.Itoa(bastion.port())
target := &SSHTarget{
Host: "127.0.0.1", Port: final.port(), User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
PinnedFingerprint: final.hostKeyFingerprint(),
ProxyJump: proxyJump,
}
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
if outcome.Reason != SSHReachabilityReasonHostKey {
t.Fatalf("Reason = %q, want host_key; message=%q", outcome.Reason, outcome.Message)
}
if outcome.Host != "127.0.0.1" {
t.Fatalf("Host = %q, want the target host, not the bastion", outcome.Host)
}
if !strings.Contains(outcome.Message, "bastion") {
t.Fatalf("Message = %q, want it to name the bastion", outcome.Message)
}
}
func TestProbeSSHHostCancelledContextProducesNoObservation(t *testing.T) {
server := newFakeSSHServer(t, nil)
newTestHomeDir(t)
identity := writeTestIdentityFile(t)
target := &SSHTarget{
Host: "127.0.0.1", Port: server.port(), User: "kandev",
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
PinnedFingerprint: server.hostKeyFingerprint(),
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
outcome := ProbeSSHHost(ctx, target, 5*time.Second)
if !outcome.Cancelled {
t.Fatalf("outcome = %+v, want Cancelled", outcome)
}
if outcome.Success {
t.Fatal("a cancelled probe must not report success")
}
if outcome.Reason != "" {
t.Fatalf("Reason = %q, want no reason for a cancelled probe", outcome.Reason)
}
}
func TestSSHTargetFromExecutorConfig(t *testing.T) {
newTestHomeDir(t)
t.Run("a valid config resolves", func(t *testing.T) {
target, err := SSHTargetFromExecutorConfig(map[string]string{
"ssh_host": "target.internal",
"ssh_port": "2222",
"ssh_user": "deploy",
"ssh_identity_source": "file",
"ssh_identity_file": "/keys/id_ed25519",
"ssh_host_fingerprint": "SHA256:abc",
})
if err != nil {
t.Fatalf("SSHTargetFromExecutorConfig: %v", err)
}
if target.Host != "target.internal" || target.Port != 2222 || target.User != "deploy" {
t.Fatalf("target = %+v", target)
}
if target.PinnedFingerprint != "SHA256:abc" {
t.Fatalf("PinnedFingerprint = %q", target.PinnedFingerprint)
}
})
t.Run("a nil config fails resolution", func(t *testing.T) {
if _, err := SSHTargetFromExecutorConfig(nil); err == nil {
t.Fatal("expected an error for a nil config")
}
})
t.Run("an invalid port fails resolution", func(t *testing.T) {
if _, err := SSHTargetFromExecutorConfig(map[string]string{
"ssh_host": "target.internal",
"ssh_port": "not-a-port",
}); err == nil {
t.Fatal("expected an error for an invalid port")
}
})
t.Run("a missing pinned fingerprint fails resolution", func(t *testing.T) {
if _, err := SSHTargetFromExecutorConfig(map[string]string{
"ssh_host": "target.internal",
"ssh_user": "deploy",
}); err == nil {
t.Fatal("expected an error when the pinned host fingerprint is missing")
}
})
}
func TestSanitizeSSHReachabilityMessageBoundsAndRedacts(t *testing.T) {
secret := "ssh: unable to authenticate, password=super-secret-token " + strings.Repeat("x", 700)
got := SanitizeSSHReachabilityMessage(errors.New(secret))
if len(got) > maxSSHReachabilityMessageBytes {
t.Fatalf("message length = %d, want at most %d", len(got), maxSSHReachabilityMessageBytes)
}
if strings.Contains(got, "super-secret-token") {
t.Fatalf("message contains the raw credential: %q", got)
}
}
@@ -250,6 +250,20 @@ type Manager struct {
// SkillDeployer hook. Nil → skill deploy is skipped.
agentProfileReader AgentProfileReader
// reachabilityReader resolves an ssh executor's stored reachability
// record for the launch-time session.launch.warning producer. Nil →
// no warning is ever published (feature not wired). See
// manager_launch_reachability_warning.go and SetSSHReachabilityWarningPolicy.
reachabilityReader ReachabilityReader
// reachabilityProbingEnabled mirrors whether the reachability poller's
// periodic sweep is on (interval != 0). When true, a stored unreachable
// record is always warning-eligible regardless of how old checked_at is.
reachabilityProbingEnabled bool
// reachabilityWarningWindowSeconds is 3x the reachability package's own
// default interval (not the configured/effective one), evaluated even
// with probing disabled per AC-EXECUTORS-SSH-REACHABILITY-001.28.
reachabilityWarningWindowSeconds int
// skillDeployer materialises per-profile skills + custom prompt before
// the agent process starts. Defaults to a no-op deployer; office wires
// its concrete implementation via SetSkillDeployer.
@@ -1096,6 +1096,8 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st
return nil, nil, nil, err
}
m.scheduleSSHLaunchWarning(launchCtx, reqWithWorktree, metadata, reqWithWorktree.SessionID)
execInstance, err := rt.CreateInstance(launchCtx, execReq)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to create execution: %w", err)
@@ -0,0 +1,145 @@
package lifecycle
import (
"context"
"strings"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
// ReachabilityReader is the narrow read accessor launchBuildExecutorRequest
// uses to look up an ssh executor's stored reachability record immediately
// before calling CreateInstance. It is declared here — not imported from
// executors/reachability — because that package already imports lifecycle
// for SSH target resolution and dial classification; importing it back here
// would cycle. In production this is satisfied directly by the task
// repository, which already implements this exact method for the
// reachability HTTP routes (see internal/ssh.ReachabilityLister).
type ReachabilityReader interface {
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
}
// SetSSHReachabilityWarningPolicy wires the launch-time session.launch.warning
// producer (see task-05-launch-non-gating.md). reader resolves an ssh
// executor's stored record; probingEnabled mirrors whether the reachability
// poller's periodic sweep is on; warningWindowSeconds is 3x the reachability
// package's own default interval, evaluated even with probing disabled.
// Leaving this unset (reader nil) disables the producer entirely.
func (m *Manager) SetSSHReachabilityWarningPolicy(reader ReachabilityReader, probingEnabled bool, warningWindowSeconds int) {
m.reachabilityReader = reader
m.reachabilityProbingEnabled = probingEnabled
m.reachabilityWarningWindowSeconds = warningWindowSeconds
}
// LaunchWarningEventPayload is the session.launch.warning payload. It is
// published once, immediately before an SSH launch's CreateInstance call,
// when the target executor's stored reachability record is unreachable and
// still within the probing window.
type LaunchWarningEventPayload struct {
TaskID string `json:"task_id"`
SessionID string `json:"session_id"`
ExecutorID string `json:"executor_id"`
Host string `json:"host"`
State string `json:"state"`
Reason string `json:"reason"`
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
Timestamp string `json:"timestamp"`
}
// sshLaunchWarningReadTimeout bounds the best-effort record lookup started by
// a launch. The lookup must not hold up CreateInstance, but it also must not
// leave a database call running forever after a launch has finished.
const sshLaunchWarningReadTimeout = 2 * time.Second
// GetSessionID satisfies the session-routing accessor
// internal/gateway/websocket.extractSessionID uses to broadcast this event
// to the correct session's subscribers.
func (p LaunchWarningEventPayload) GetSessionID() string {
return p.SessionID
}
// PublishLaunchWarning publishes a session.launch.warning event to the
// launched session's own event stream.
func (p *EventPublisher) PublishLaunchWarning(sessionID string, payload *LaunchWarningEventPayload) {
if p.eventBus == nil {
return
}
if payload.Timestamp == "" {
payload.Timestamp = time.Now().UTC().Format(time.RFC3339Nano)
}
subject := events.BuildSessionLaunchWarningSubject(sessionID)
event := bus.NewEvent(events.SessionLaunchWarning, "lifecycle", payload)
if err := p.eventBus.Publish(context.Background(), subject, event); err != nil {
p.logger.Error("failed to publish session launch warning event",
zap.String("session_id", sessionID),
zap.Error(err))
}
}
// scheduleSSHLaunchWarning starts the best-effort record lookup without
// delaying CreateInstance. The request and metadata values used by the
// goroutine are copied because the launch path may reuse or mutate its input
// maps after it returns.
func (m *Manager) scheduleSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) {
if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) ||
m.reachabilityReader == nil || m.eventPublisher == nil {
return
}
warningReq := *req
warningMetadata := map[string]interface{}{
"executor_id": getMetadataString(metadata, "executor_id"),
}
warningCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sshLaunchWarningReadTimeout)
go func() {
defer cancel()
m.maybePublishSSHLaunchWarning(warningCtx, &warningReq, warningMetadata, sessionID)
}()
}
// maybePublishSSHLaunchWarning reads the target executor's reachability
// record and, when it is unreachable and still warning-eligible, publishes
// exactly one session.launch.warning. It is a strict read-only accessor call,
// made from launchBuildExecutorRequest alongside its single CreateInstance
// call site, never from inside CreateInstance itself. A read failure or a
// missing record produces no warning.
func (m *Manager) maybePublishSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) {
if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) ||
m.reachabilityReader == nil || m.eventPublisher == nil {
return
}
executorID := strings.TrimSpace(getMetadataString(metadata, "executor_id"))
if executorID == "" {
return
}
record, err := m.reachabilityReader.GetExecutorReachability(ctx, executorID)
if err != nil || record == nil || record.State != models.ExecutorReachabilityStateUnreachable {
return
}
if !m.reachabilityProbingEnabled && !withinReachabilityWarningWindow(record.CheckedAt, m.reachabilityWarningWindowSeconds) {
return
}
m.eventPublisher.PublishLaunchWarning(sessionID, &LaunchWarningEventPayload{
TaskID: req.TaskID,
SessionID: sessionID,
ExecutorID: executorID,
Host: record.Host,
State: string(record.State),
Reason: string(record.Reason),
LastSuccessAt: record.LastSuccessAt,
})
}
// withinReachabilityWarningWindow reports whether checkedAt is recent enough
// to still warn even with periodic probing disabled. A nil checkedAt (never
// probed) or a non-positive window never qualifies.
func withinReachabilityWarningWindow(checkedAt *time.Time, windowSeconds int) bool {
if checkedAt == nil || windowSeconds <= 0 {
return false
}
return time.Since(*checkedAt) <= time.Duration(windowSeconds)*time.Second
}
@@ -0,0 +1,305 @@
package lifecycle
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/agent/executor"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
type fakeReachabilityReader struct {
records map[string]*models.ExecutorReachability
err error
calls []string
}
func (f *fakeReachabilityReader) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
f.calls = append(f.calls, executorID)
if f.err != nil {
return nil, f.err
}
return f.records[executorID], nil
}
// failOnReadReachabilityReader fails the test the moment it is touched at
// all — used to prove a code path performs no reachability read, rather than
// merely counting reads after the fact.
type failOnReadReachabilityReader struct {
t *testing.T
}
func (f failOnReadReachabilityReader) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) {
f.t.Fatal("reachability record was read from a code path that must never read one")
return nil, nil
}
type blockingReachabilityReader struct {
started chan struct{}
release chan struct{}
}
func (r *blockingReachabilityReader) GetExecutorReachability(ctx context.Context, _ string) (*models.ExecutorReachability, error) {
close(r.started)
select {
case <-r.release:
return nil, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func newWarningTestManager(t *testing.T, reader ReachabilityReader, probingEnabled bool, windowSeconds int) (*Manager, *bus.MemoryEventBus) {
t.Helper()
log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"})
if err != nil {
t.Fatalf("new logger: %v", err)
}
memBus := bus.NewMemoryEventBus(log)
mgr := &Manager{
logger: log,
eventPublisher: NewEventPublisher(memBus, log),
}
mgr.SetSSHReachabilityWarningPolicy(reader, probingEnabled, windowSeconds)
return mgr, memBus
}
func subscribeLaunchWarnings(t *testing.T, memBus *bus.MemoryEventBus, sessionID string) <-chan *bus.Event {
t.Helper()
received := make(chan *bus.Event, 1)
sub, err := memBus.Subscribe(events.BuildSessionLaunchWarningSubject(sessionID), func(_ context.Context, ev *bus.Event) error {
received <- ev
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
return received
}
const warningWindowSeconds = 180 // 3x reachability.DefaultIntervalSeconds
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_UnreachableAndProbingEnabledWarns(t *testing.T) {
old := time.Now().Add(-time.Hour)
lastSuccess := old.Add(-time.Hour)
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
"executor-1": {
ExecutorID: "executor-1",
State: models.ExecutorReachabilityStateUnreachable,
Reason: models.ExecutorReachabilityReasonTimeout,
Host: "build.example",
CheckedAt: &old,
LastSuccessAt: &lastSuccess,
},
}}
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{
ExecutorType: string(models.ExecutorTypeSSH),
TaskID: "task-1",
SessionID: "session-1",
}
metadata := map[string]interface{}{"executor_id": "executor-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, metadata, req.SessionID)
select {
case ev := <-received:
payload, ok := ev.Data.(*LaunchWarningEventPayload)
if !ok {
t.Fatalf("payload type = %T", ev.Data)
}
if payload.ExecutorID != "executor-1" || payload.Host != "build.example" ||
payload.State != string(models.ExecutorReachabilityStateUnreachable) ||
payload.Reason != string(models.ExecutorReachabilityReasonTimeout) ||
payload.LastSuccessAt == nil || !payload.LastSuccessAt.Equal(lastSuccess) {
t.Fatalf("payload = %+v", payload)
}
default:
t.Fatal("expected a session.launch.warning event, got none")
}
if len(reader.calls) != 1 || reader.calls[0] != "executor-1" {
t.Fatalf("reader.calls = %v", reader.calls)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_ProbingDisabledButRecentlyCheckedWarns(t *testing.T) {
recent := time.Now().Add(-time.Minute)
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
"executor-1": {
ExecutorID: "executor-1",
State: models.ExecutorReachabilityStateUnreachable,
Reason: models.ExecutorReachabilityReasonNetwork,
Host: "build.example",
CheckedAt: &recent,
},
}}
mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
select {
case <-received:
default:
t.Fatal("expected a session.launch.warning event when checked_at is within the warning window")
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_ProbingDisabledAndStaleCheckedAtDoesNotWarn(t *testing.T) {
stale := time.Now().Add(-time.Hour)
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
"executor-1": {
ExecutorID: "executor-1",
State: models.ExecutorReachabilityStateUnreachable,
Host: "build.example",
CheckedAt: &stale,
},
}}
mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
select {
case ev := <-received:
t.Fatalf("expected no warning for a stale checked_at with probing disabled, got %+v", ev)
default:
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_ReachableRecordDoesNotWarn(t *testing.T) {
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
"executor-1": {ExecutorID: "executor-1", State: models.ExecutorReachabilityStateReachable, Host: "build.example"},
}}
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
select {
case ev := <-received:
t.Fatalf("expected no warning for a reachable record, got %+v", ev)
default:
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_NoRecordDoesNotWarn(t *testing.T) {
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{}}
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
select {
case ev := <-received:
t.Fatalf("expected no warning when no record exists yet, got %+v", ev)
default:
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
func TestMaybePublishSSHLaunchWarning_ReadFailureDoesNotWarnOrPanic(t *testing.T) {
reader := &fakeReachabilityReader{err: errors.New("boom")}
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
received := subscribeLaunchWarnings(t, memBus, "session-1")
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
select {
case ev := <-received:
t.Fatalf("expected no warning on a read failure, got %+v", ev)
default:
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// Non-ssh launches must never trigger a reachability read at all — the
// failOnReadReachabilityReader fails the test the instant it is touched,
// which is what proves the short-circuit rather than merely observing zero
// warnings (a bug that read the record but decided not to warn would still
// pass a call-count-only assertion).
func TestMaybePublishSSHLaunchWarning_NonSSHExecutorNeverReads(t *testing.T) {
mgr, _ := newWarningTestManager(t, failOnReadReachabilityReader{t: t}, true, warningWindowSeconds)
req := &LaunchRequest{ExecutorType: "local_docker", SessionID: "session-1"}
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
}
func TestScheduleSSHLaunchWarningDoesNotWaitForRead(t *testing.T) {
reader := &blockingReachabilityReader{started: make(chan struct{}), release: make(chan struct{})}
mgr, _ := newWarningTestManager(t, reader, true, warningWindowSeconds)
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
started := time.Now()
mgr.scheduleSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
t.Fatalf("scheduleSSHLaunchWarning waited %v for a blocked read", elapsed)
}
select {
case <-reader.started:
case <-time.After(time.Second):
t.Fatal("warning read did not start asynchronously")
}
close(reader.release)
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// launchBuildExecutorRequest is the single production call site for every
// executor type (verified: it has exactly one caller, launchInternal, which
// every launch entry point — WS-initiated, resume, and workflow/dependency
// launches via orchestrator/executor's LaunchAgent — funnels through). This
// exercises that shared call site directly with a non-ssh backend and proves
// the reachability reader is never touched and CreateInstance still runs.
func TestLaunchBuildExecutorRequestNonSSHExecutorNeverReadsReachability(t *testing.T) {
log := newTestRegistryLogger()
registry := NewExecutorRegistry(log)
backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameDocker}}
registry.Register(backend)
memBusLog, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"})
memBus := bus.NewMemoryEventBus(memBusLog)
mgr := &Manager{
executorRegistry: registry,
executorFallbackPolicy: ExecutorFallbackDeny,
logger: log,
eventPublisher: NewEventPublisher(memBus, memBusLog),
}
mgr.SetSSHReachabilityWarningPolicy(failOnReadReachabilityReader{t: t}, true, warningWindowSeconds)
req := &LaunchRequest{
ExecutorType: "local_docker",
EnvironmentFinalized: true,
SessionID: "session-1",
Metadata: map[string]interface{}{"executor_id": "executor-1"},
}
_, _, _, err := mgr.launchBuildExecutorRequest(
context.Background(), "instance-1", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil,
)
if err != nil {
t.Fatalf("launchBuildExecutorRequest: %v", err)
}
if backend.lastRequest == nil {
t.Fatal("expected CreateInstance to run")
}
}
@@ -0,0 +1,92 @@
package runtime
import (
"context"
"time"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
)
// SSHReachabilityReason classifies why an SSH reachability probe failed. It
// mirrors lifecycle.SSHReachabilityReason so callers outside the runtime
// package never import the lifecycle tier directly.
type SSHReachabilityReason string
const (
SSHReachabilityReasonConfig SSHReachabilityReason = "config"
SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout"
SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key"
SSHReachabilityReasonAuth SSHReachabilityReason = "auth"
SSHReachabilityReasonNetwork SSHReachabilityReason = "network"
SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown"
)
// SSHProbeOutcome is the runtime-facade mirror of lifecycle.SSHProbeOutcome.
type SSHProbeOutcome struct {
Success bool
Cancelled bool
Host string
Reason SSHReachabilityReason
Message string
}
// SanitizeSSHReachabilityMessage exposes the runtime-safe SSH diagnostic
// projection without making higher-level adapters import lifecycle directly.
func SanitizeSSHReachabilityMessage(err error) string {
return lifecycle.SanitizeSSHReachabilityMessage(err)
}
// SSHReachabilityProber is the runtime seam for a single SSH reachability
// probe. Higher-level packages depend on this contract instead of the
// lifecycle implementation details.
type SSHReachabilityProber interface {
// Probe resolves an SSH executor's target from its persisted config and
// probes it, bounded by timeout. A resolution failure (missing host,
// invalid port, unresolvable alias) never dials — it is reported as
// SSHReachabilityReasonConfig directly, the same reason a target with no
// pinned fingerprint uses.
Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome
// ValidateConfig reports whether config resolves to a dialable SSH
// target, without dialing it.
ValidateConfig(config map[string]string) error
}
type sshReachabilityProber struct {
resolveTarget func(map[string]string) (*lifecycle.SSHTarget, error)
probe func(context.Context, *lifecycle.SSHTarget, time.Duration) lifecycle.SSHProbeOutcome
}
// NewSSHReachabilityProber creates the production adapter over the lifecycle
// SSH target resolver and reachability prober.
func NewSSHReachabilityProber() SSHReachabilityProber {
return &sshReachabilityProber{
resolveTarget: lifecycle.SSHTargetFromExecutorConfig,
probe: lifecycle.ProbeSSHHost,
}
}
func (a *sshReachabilityProber) Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome {
target, err := a.resolveTarget(config)
if err != nil {
return SSHProbeOutcome{
Host: config["ssh_host"],
Reason: SSHReachabilityReasonConfig,
Message: lifecycle.SanitizeSSHReachabilityMessage(err),
}
}
outcome := a.probe(ctx, target, timeout)
return SSHProbeOutcome{
Success: outcome.Success,
Cancelled: outcome.Cancelled,
Host: outcome.Host,
Reason: SSHReachabilityReason(outcome.Reason),
Message: outcome.Message,
}
}
func (a *sshReachabilityProber) ValidateConfig(config map[string]string) error {
_, err := a.resolveTarget(config)
return err
}
var _ SSHReachabilityProber = (*sshReachabilityProber)(nil)
@@ -220,32 +220,32 @@ func stringContains(value, want string) bool {
func newCanvasEditTestDependencies() (*fakeCanvasEditTaskStore, *fakeCanvasEditCanvasStore, *fakeCanvasEditReleaseStore, *fakeCanvasEditArtifactStore) {
return &fakeCanvasEditTaskStore{
workspace: &models.Workspace{
ID: "workspace-a",
DefaultAgentProfileID: stringPointer("profile-default"),
DefaultExecutorID: stringPointer("executor-default"),
},
}, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{
ID: "canvas-a",
PluginInstanceID: "instance-a",
WorkspaceID: "workspace-a",
ScopeKind: canvas.ScopeWorkspace,
Status: instances.StatusActive,
ActiveReleaseID: "release-a",
ActiveReleaseStatus: instances.ValidationValid,
EffectiveGrants: []canvas.GrantProjection{{
PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace,
}},
}}, &fakeCanvasEditReleaseStore{release: instances.Release{
ID: "release-a",
InstanceID: "instance-a",
PackageDigest: "digest-a",
ArtifactPath: "releases/digest-a",
ValidationStatus: instances.ValidationValid,
}}, &fakeCanvasEditArtifactStore{files: map[string][]byte{
"manifest.yaml": []byte("manifest"),
"index.html": []byte("index"),
}}
workspace: &models.Workspace{
ID: "workspace-a",
DefaultAgentProfileID: stringPointer("profile-default"),
DefaultExecutorID: stringPointer("executor-default"),
},
}, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{
ID: "canvas-a",
PluginInstanceID: "instance-a",
WorkspaceID: "workspace-a",
ScopeKind: canvas.ScopeWorkspace,
Status: instances.StatusActive,
ActiveReleaseID: "release-a",
ActiveReleaseStatus: instances.ValidationValid,
EffectiveGrants: []canvas.GrantProjection{{
PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace,
}},
}}, &fakeCanvasEditReleaseStore{release: instances.Release{
ID: "release-a",
InstanceID: "instance-a",
PackageDigest: "digest-a",
ArtifactPath: "releases/digest-a",
ValidationStatus: instances.ValidationValid,
}}, &fakeCanvasEditArtifactStore{files: map[string][]byte{
"manifest.yaml": []byte("manifest"),
"index.html": []byte("index"),
}}
}
func stringPointer(value string) *string { return &value }
@@ -46,6 +46,7 @@ import (
editorhandlers "github.com/kandev/kandev/internal/editors/handlers"
"github.com/kandev/kandev/internal/entityrefs"
"github.com/kandev/kandev/internal/events/bus"
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
"github.com/kandev/kandev/internal/failedinbox"
gateways "github.com/kandev/kandev/internal/gateway/websocket"
"github.com/kandev/kandev/internal/github"
@@ -729,6 +730,7 @@ type routeParams struct {
planCoalesceWindowConfigured bool
homeDir string
interimSettingsInterlockToken string
sshReachabilityPoller *reachabilitypkg.Poller
log *logger.Logger
progress *startup.Reporter
}
@@ -1467,6 +1469,14 @@ func registerSecondaryRoutes(
)
p.log.Debug("Registered Kubernetes handlers (HTTP + WebSocket)")
// A nil *reachabilitypkg.Poller must not be passed directly as the
// sshhandlers.ReachabilityProber interface parameter — that would
// produce a non-nil interface holding a nil pointer, defeating the
// handler's own nil check and panicking on first use.
var reachabilityPoller sshhandlers.ReachabilityProber
if p.sshReachabilityPoller != nil {
reachabilityPoller = p.sshReachabilityPoller
}
sshhandlers.RegisterRoutes(
p.router,
p.gateway.Dispatcher,
@@ -1475,6 +1485,8 @@ func registerSecondaryRoutes(
p.agentRegistry,
lifecycle.NewAgentctlResolver(p.log),
p.log,
p.taskRepo,
reachabilityPoller,
)
p.log.Debug("Registered SSH handlers (HTTP + WebSocket)")
}
@@ -22,7 +22,7 @@ func TestBuildHTTPServerAbortsWhenInterlockTokenGenerationFails(t *testing.T) {
newInterimSettingsInterlockToken = func() (string, error) { return "", wantErr }
t.Cleanup(func() { newInterimSettingsInterlockToken = original })
server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
if !errors.Is(err, wantErr) {
t.Fatalf("buildHTTPServer error = %v, want %v", err, wantErr)
}
+38 -2
View File
@@ -43,6 +43,9 @@ import (
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
// SSH executor reachability poller
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
// GitHub integration
azuredevopspkg "github.com/kandev/kandev/internal/azuredevops"
githubpkg "github.com/kandev/kandev/internal/github"
@@ -881,6 +884,36 @@ func startAgentInfrastructure(
log.Info("Office config sync poller started")
}
// Start SSH executor reachability poller: sweeps every eligible SSH
// executor on a configurable interval, probing reachability and
// persisting results through a hysteresis-owning write path. Wired with
// a publisher (state/reason changes reach WS clients) and registered as
// the task service's executor-save observer (a changed host resets the
// record and dispatches an off-cycle probe) so both the poller and the
// reachability HTTP routes below share the one running instance.
sshReachabilityPoller := startSSHReachabilityPoller(
ctx,
repos.Task,
cfg.Executors.SSHReachabilityIntervalSeconds,
log,
reachabilitypkg.NewPublisher(eventBus, log),
addRuntimeCleanup,
)
services.Task.SetExecutorSaveObserver(reachabilitypkg.NewSaveObserver(sshReachabilityPoller))
// Launch-time session.launch.warning producer (task 05): repos.Task
// already implements the narrow read accessor (same method used by the
// reachability HTTP routes). probingEnabled mirrors the poller's own
// effective interval so a configured 0 (disabled) keeps the warning
// gated on staleness alone. warningWindowSeconds is 3x the reachability
// package's own default interval, not the configured one, per
// AC-EXECUTORS-SSH-REACHABILITY-001.28.
lifecycleMgr.SetSSHReachabilityWarningPolicy(
repos.Task,
sshReachabilityPoller.EffectiveIntervalSeconds() != 0,
3*reachabilitypkg.DefaultIntervalSeconds,
)
// Start the plugin system's event delivery and health monitor
// background loops.
if services.Plugins != nil {
@@ -943,7 +976,7 @@ func startAgentInfrastructure(
return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services,
agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath,
sessionCapacityEnvironment, storageStore, func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelWorkers,
restoreCleanups, databaseQuiesce)
restoreCleanups, databaseQuiesce, sshReachabilityPoller)
}
// startOrchestratorAndAutomationConsumers establishes the startup chain in
@@ -1013,6 +1046,7 @@ func startGatewayAndServe(
cancelWorkers context.CancelFunc,
restoreCleanups []func() error,
databaseQuiesce func() error,
sshReachabilityPoller *reachabilitypkg.Poller,
) bool {
// ============================================
// WEBSOCKET GATEWAY
@@ -1350,7 +1384,7 @@ func startGatewayAndServe(
builtServer, err := buildHTTPServer(cfg, log, gateway, repos, services, agentSettingsController,
lifecycleMgr, eventBus, orchestratorSvc, notificationCtrl, msgCreator, agentRegistry, hostUtilityMgr,
addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer,
storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, startup.FromContext(ctx), persistenceHealth)
storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, sshReachabilityPoller, startup.FromContext(ctx), persistenceHealth)
if err != nil {
log.Error("Failed to build HTTP server", zap.Error(err))
closeBoundListeners(server, listeners, log)
@@ -2599,6 +2633,7 @@ func buildHTTPServer(
temporaryArtifacts *tempartifacts.Registry,
dbPool *db.Pool,
agentRuntimeAvailability *agentctlclient.Availability,
sshReachabilityPoller *reachabilitypkg.Poller,
progress *startup.Reporter,
persistenceHealth ...*requiredstores.Health,
) (*http.Server, error) {
@@ -2730,6 +2765,7 @@ func buildHTTPServer(
planCoalesceWindowConfigured: true,
homeDir: cfg.ResolvedHomeDir(),
interimSettingsInterlockToken: interimSettingsInterlockToken,
sshReachabilityPoller: sshReachabilityPoller,
log: log,
progress: progress,
})
@@ -0,0 +1,26 @@
package backendapp
import (
"context"
"github.com/kandev/kandev/internal/common/logger"
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
)
// startSSHReachabilityPoller launches the SSH executor reachability poller
// and registers its Stop on addCleanup, mirroring the third-party
// integration poller wiring alongside it in startAgentInfrastructure.
func startSSHReachabilityPoller(
ctx context.Context,
repo reachabilitypkg.Repository,
intervalSeconds int,
log *logger.Logger,
publisher *reachabilitypkg.Publisher,
addCleanup func(func() error) func() error,
) *reachabilitypkg.Poller {
poller := reachabilitypkg.New(repo, intervalSeconds, log)
poller.SetPublisher(publisher)
poller.Start(ctx)
addCleanup(func() error { poller.Stop(); return nil })
return poller
}
@@ -0,0 +1,108 @@
package backendapp
import (
"context"
"go/ast"
"go/parser"
"go/token"
"testing"
"time"
"github.com/kandev/kandev/internal/common/logger"
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
"github.com/kandev/kandev/internal/task/models"
)
type sshReachabilityWiringRepo struct {
listed chan struct{}
}
func (r *sshReachabilityWiringRepo) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) {
select {
case r.listed <- struct{}{}:
default:
}
return nil, nil
}
func (r *sshReachabilityWiringRepo) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) {
return nil, models.ErrExecutorReachabilityNotFound
}
func (r *sshReachabilityWiringRepo) UpsertExecutorReachability(context.Context, models.ExecutorReachabilityObservation) error {
return nil
}
func (r *sshReachabilityWiringRepo) ResetExecutorReachability(context.Context, string, string, time.Time) error {
return nil
}
// TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper mirrors
// TestInitOfficeServicesUsesTaskUsageWiringHelper: the composition root
// (startAgentInfrastructure, too large to exercise directly in a unit test)
// must delegate to the narrow, independently-testable wiring helper exactly
// once rather than inlining the poller construction.
func TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper(t *testing.T) {
fset := token.NewFileSet()
file, err := parser.ParseFile(fset, "main.go", nil, 0)
if err != nil {
t.Fatalf("parse main.go: %v", err)
}
var target *ast.FuncDecl
for _, decl := range file.Decls {
fn, ok := decl.(*ast.FuncDecl)
if ok && fn.Name.Name == "startAgentInfrastructure" {
target = fn
break
}
}
if target == nil {
t.Fatal("startAgentInfrastructure not found in main.go")
}
var helperCalls int
ast.Inspect(target, func(node ast.Node) bool {
call, ok := node.(*ast.CallExpr)
if !ok {
return true
}
if name, ok := call.Fun.(*ast.Ident); ok && name.Name == "startSSHReachabilityPoller" {
helperCalls++
}
return true
})
if helperCalls != 1 {
t.Fatalf("startSSHReachabilityPoller calls = %d, want exactly one composition call", helperCalls)
}
}
func TestStartSSHReachabilityPoller_StartsPollerAndRegistersCleanup(t *testing.T) {
repo := &sshReachabilityWiringRepo{listed: make(chan struct{}, 1)}
var cleanups []func() error
addCleanup := func(fn func() error) func() error {
cleanups = append(cleanups, fn)
return fn
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
poller := startSSHReachabilityPoller(ctx, repo, 3600, logger.Default(), (*reachabilitypkg.Publisher)(nil), addCleanup)
if poller == nil {
t.Fatal("startSSHReachabilityPoller returned a nil poller")
}
if len(cleanups) != 1 {
t.Fatalf("cleanups registered = %d, want exactly 1", len(cleanups))
}
select {
case <-repo.listed:
case <-time.After(time.Second):
t.Fatal("poller never ran its immediate pass")
}
if err := cleanups[0](); err != nil {
t.Fatalf("cleanup: %v", err)
}
}
@@ -111,6 +111,7 @@ var startupCatalog = []CatalogEntry{
{Key: "launcher.webPort", EnvVars: []string{"KANDEV_WEB_PORT"}, Owner: "launcher", Default: "automatic"},
{Key: "launcher.healthTimeoutMs", EnvVars: []string{"KANDEV_HEALTH_TIMEOUT_MS"}, Owner: "launcher", Default: "45000"},
{Key: "launcher.noBrowser", EnvVars: []string{"KANDEV_NO_BROWSER"}, Owner: "launcher", Default: "false"},
{Key: "executors.sshReachabilityIntervalSeconds", EnvVars: []string{"KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS"}, Owner: "executors/reachability", Default: "60"},
}
var startupExclusions = []CatalogExclusion{
@@ -134,6 +134,7 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment {
{envVar: "KANDEV_WEB_PORT", class: "catalog"},
{envVar: "KANDEV_HEALTH_TIMEOUT_MS", class: "catalog"},
{envVar: "KANDEV_NO_BROWSER", class: "catalog"},
{envVar: "KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", class: "catalog"},
{envVar: InternalConfigFileEnv, class: "exclusion"},
{envVar: InternalConfigHomeFileEnv, class: "exclusion"},
{envVar: InternalAgentctlStartupConfigEnv, class: "exclusion"},
@@ -53,6 +53,7 @@ type Config struct {
Office OfficeConfig `mapstructure:"office"`
Features FeaturesConfig `mapstructure:"features"`
GitHubCredentialBroker GitHubCredentialBrokerConfig `mapstructure:"githubCredentialBroker"`
Executors ExecutorsConfig `mapstructure:"executors"`
Source ConfigSource `mapstructure:"-" json:"-"`
}
@@ -71,6 +72,15 @@ type CredentialsConfig struct {
File string `mapstructure:"file"`
}
// ExecutorsConfig contains executor-related startup settings.
type ExecutorsConfig struct {
// SSHReachabilityIntervalSeconds is the raw configured value, not yet
// clamped into the reachability package's own 15-3600 bound: 0 disables
// the poller, and reachability.ClampInterval owns the rest of the
// normalization at construction time.
SSHReachabilityIntervalSeconds int `mapstructure:"sshReachabilityIntervalSeconds"`
}
// LimitsConfig contains process and protocol capacity limits.
type LimitsConfig struct {
GHMaxConcurrent int `mapstructure:"ghMaxConcurrent"`
+25 -22
View File
@@ -65,28 +65,29 @@ type configSelection struct {
// AutomaticEnv during Unmarshal: legacy invalid environment values must retain
// their fallback behavior instead of turning into typed-config startup errors.
var yamlOnlyStartupKeys = map[string]struct{}{
"server.trustedProxies": {},
"tasks.preparationTimeout": {},
"tasks.stallDetectionThreshold": {},
"credentials.file": {},
"limits.ghMaxConcurrent": {},
"limits.gitMaxConcurrent": {},
"limits.lspMaxConnections": {},
"messageQueue.maxPerSession": {},
"agentctl.idleTimeout": {},
"agentctl.idleReaperInterval": {},
"agentctl.notificationQueueCapacity": {},
"agentctl.recoveryDeadline": {},
"agentctl.recoveryReadTimeout": {},
"agentctl.recoveryReadRetries": {},
"agentctl.unownedPeriod": {},
"agentctl.detachedEventLimit": {},
"planning.coalesceWindowMs": {},
"office.schedulerTickMs": {},
"observability.otlpEndpoint": {},
"launcher.webPort": {},
"launcher.healthTimeoutMs": {},
"launcher.noBrowser": {},
"server.trustedProxies": {},
"tasks.preparationTimeout": {},
"tasks.stallDetectionThreshold": {},
"credentials.file": {},
"limits.ghMaxConcurrent": {},
"limits.gitMaxConcurrent": {},
"limits.lspMaxConnections": {},
"messageQueue.maxPerSession": {},
"agentctl.idleTimeout": {},
"agentctl.idleReaperInterval": {},
"agentctl.notificationQueueCapacity": {},
"agentctl.recoveryDeadline": {},
"agentctl.recoveryReadTimeout": {},
"agentctl.recoveryReadRetries": {},
"agentctl.unownedPeriod": {},
"agentctl.detachedEventLimit": {},
"planning.coalesceWindowMs": {},
"office.schedulerTickMs": {},
"observability.otlpEndpoint": {},
"launcher.webPort": {},
"launcher.healthTimeoutMs": {},
"launcher.noBrowser": {},
"executors.sshReachabilityIntervalSeconds": {},
}
func isYAMLOnlyStartupKey(key string) bool {
@@ -313,6 +314,7 @@ func applyStartupDefaults(cfg *Config, yamlKeys map[string]bool, profileDefaults
if !yamlKeys["launcher.noBrowser"] {
cfg.Launcher.NoBrowser = false
}
setDefaultInt("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60)
}
func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources map[string]SettingSource) {
@@ -343,6 +345,7 @@ func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources
applyBoundedIntEnv("launcher.webPort", &cfg.Launcher.WebPort, 0, 1, 65535, envSnapshot, sources)
applyPositiveIntEnv("launcher.healthTimeoutMs", &cfg.Launcher.HealthTimeoutMs, launcherHealthTimeoutDefault(), envSnapshot, sources)
applyBoolEnv("launcher.noBrowser", &cfg.Launcher.NoBrowser, false, envSnapshot, sources)
applyNonNegativeIntEnv("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60, envSnapshot, sources)
}
func launcherHealthTimeoutDefault() int {
@@ -307,6 +307,74 @@ func TestHomeConfigInvalidFirstDoesNotFallThrough(t *testing.T) {
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
//
// The catalog's own fallback-and-bounds logic only ever falls back to the
// default (60): the 0-disables/15-3600-clamp behavior belongs to the
// reachability package's own ClampInterval, not this layer, per the design's
// split between catalog-level normalization and package-level clamping.
func TestExecutorsSSHReachabilityIntervalSecondsEnvironment(t *testing.T) {
tests := []struct {
name string
env string
wantValue int
wantSource SettingSource
}{
{"absent uses default", "", 60, SourceDefault},
{"valid value passes through unclamped", "45", 45, SourceEnvironment},
{"zero passes through as the disable sentinel", "0", 0, SourceEnvironment},
{"negative falls back to the default", "-5", 60, SourceDefault},
{"unparsable falls back to the default", "not-a-number", 60, SourceDefault},
{"a value outside 15-3600 still passes through unclamped here", "9999", 9999, SourceEnvironment},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.env != "" {
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", tt.env)
} else {
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "")
}
t.Setenv("KANDEV_SERVER_PORT", "")
cfg, err := LoadWithPath(t.TempDir())
if err != nil {
t.Fatalf("LoadWithPath: %v", err)
}
if cfg.Executors.SSHReachabilityIntervalSeconds != tt.wantValue {
t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want %d", cfg.Executors.SSHReachabilityIntervalSeconds, tt.wantValue)
}
if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != tt.wantSource {
t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, tt.wantSource)
}
})
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
//
// A YAML value is not passed through the environment's fallback-and-bounds
// logic at all — it decodes straight into the typed field, negative values
// included, leaving ClampInterval as the sole place that normalizes it.
func TestExecutorsSSHReachabilityIntervalSecondsYAMLPassesThroughUnclamped(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte("executors:\n sshReachabilityIntervalSeconds: -5\n"), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "")
t.Setenv("KANDEV_SERVER_PORT", "")
cfg, err := LoadWithPath(dir)
if err != nil {
t.Fatalf("LoadWithPath: %v", err)
}
if cfg.Executors.SSHReachabilityIntervalSeconds != -5 {
t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want -5 (unclamped)", cfg.Executors.SSHReachabilityIntervalSeconds)
}
if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != SourceConfiguration {
t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, SourceConfiguration)
}
}
func nestedField(t *testing.T, value any, names ...string) reflect.Value {
t.Helper()
current := reflect.ValueOf(value)
+24
View File
@@ -192,6 +192,10 @@ const (
ExecutorCreated = "executor.created"
ExecutorUpdated = "executor.updated"
ExecutorDeleted = "executor.deleted"
// ExecutorReachabilityChanged is published only when a probe or a
// connection-configuration reset actually changes the stored state or
// reason — a steady host never publishes.
ExecutorReachabilityChanged = "executor.reachability.changed"
)
// Event types for executor profiles
@@ -310,6 +314,16 @@ const (
AvailableCommandsUpdated = "available_commands.updated" // Available slash commands updated
)
// Event types for session launch warnings
const (
// SessionLaunchWarning is published once, immediately before an SSH
// launch's CreateInstance call, when the target executor's stored
// reachability record is unreachable and the record is still within the
// probing window. It carries Kandev's own attribution of the target
// host, independent of whatever an agent process itself reports.
SessionLaunchWarning = "session.launch.warning"
)
// Event types for session mode
const (
SessionModeChanged = "session_mode.changed" // Agent session mode changed
@@ -512,6 +526,16 @@ func BuildAgentCapabilitiesSubject(sessionID string) string {
return AgentCapabilitiesUpdated + "." + sessionID
}
// BuildSessionLaunchWarningSubject creates a session launch warning subject for a specific session
func BuildSessionLaunchWarningSubject(sessionID string) string {
return SessionLaunchWarning + "." + sessionID
}
// BuildSessionLaunchWarningWildcardSubject creates a wildcard subscription for all session launch warning events
func BuildSessionLaunchWarningWildcardSubject() string {
return SessionLaunchWarning + ".*"
}
// BuildAgentCapabilitiesWildcardSubject creates a wildcard subscription for all agent capabilities events
func BuildAgentCapabilitiesWildcardSubject() string {
return AgentCapabilitiesUpdated + ".*"
@@ -0,0 +1,15 @@
package reachability
import (
"testing"
"go.uber.org/goleak"
)
// TestMain asserts that no goroutines from this package outlive the test
// process. The Poller owns the scheduled loop plus any off-cycle ProbeNow
// call, all registered on one WaitGroup — goleak catches a Start path that
// forgets to register, or a Stop path that returns before every probe drains.
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
@@ -0,0 +1,51 @@
// Package reachability sweeps every eligible SSH executor on a configurable
// interval, probing each with bounded concurrency and persisting the result
// through a hysteresis-owning write path. See
// docs/specs/executors/system-design/ssh-reachability.md.
package reachability
import "time"
const (
// probeTimeout bounds a single probe's TCP dial and SSH handshake.
// sshDialTimeout is 30s; a shorter bound here keeps one hung host from
// holding a semaphore slot for half the interval.
probeTimeout = 10 * time.Second
// passConcurrency bounds simultaneous outbound connections and file
// descriptors while keeping a pass short.
passConcurrency = 4
// failureThreshold is the smallest value that suppresses a single-probe
// blip; worst-case detection is two intervals.
failureThreshold = 2
// DefaultIntervalSeconds is the effective interval used when the
// configured value is absent, negative, fractional, or unparsable.
DefaultIntervalSeconds = 60
// MinIntervalSeconds and MaxIntervalSeconds bound the poller's cadence.
// 0 is a separate "disabled" sentinel and is never clamped into this
// range.
MinIntervalSeconds = 15
MaxIntervalSeconds = 3600
)
// ClampInterval applies this package's bounds to a raw configured interval
// (in seconds) and reports whether the value was adjusted — worth one
// startup log line rather than a refused boot. The environment path in
// internal/common/config already normalizes an absent, empty, fractional, or
// unparsable value to DefaultIntervalSeconds; a YAML value reaches here
// unnormalized; either way this function still owns the below-minimum,
// above-maximum, and negative cases directly.
func ClampInterval(seconds int) (effective int, clamped bool) {
switch {
case seconds == 0:
return 0, false
case seconds < 0:
return DefaultIntervalSeconds, true
case seconds < MinIntervalSeconds:
return MinIntervalSeconds, true
case seconds > MaxIntervalSeconds:
return MaxIntervalSeconds, true
default:
return seconds, false
}
}
@@ -0,0 +1,31 @@
package reachability
import "testing"
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.23, AC-EXECUTORS-SSH-REACHABILITY-001.24
func TestClampInterval(t *testing.T) {
tests := []struct {
name string
in int
wantEffect int
wantClamped bool
}{
{"zero disables and is not a clamp", 0, 0, false},
{"negative falls back to the default", -5, DefaultIntervalSeconds, true},
{"below minimum clamps up to 15", 1, MinIntervalSeconds, true},
{"just below minimum clamps up to 15", 14, MinIntervalSeconds, true},
{"minimum passes through unchanged", 15, 15, false},
{"default passes through unchanged", 60, 60, false},
{"maximum passes through unchanged", 3600, 3600, false},
{"just above maximum clamps down to 3600", 3601, MaxIntervalSeconds, true},
{"far above maximum clamps down to 3600", 100000, MaxIntervalSeconds, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
effective, clamped := ClampInterval(tt.in)
if effective != tt.wantEffect || clamped != tt.wantClamped {
t.Errorf("ClampInterval(%d) = (%d, %v), want (%d, %v)", tt.in, effective, clamped, tt.wantEffect, tt.wantClamped)
}
})
}
}
@@ -0,0 +1,60 @@
package reachability
import (
"expvar"
"strings"
)
// expvar counters exposed via /debug/vars, following the office_stall_* and
// routing_* precedent: "key=value;key=value..." labels so a Prometheus
// translation layer can split on ';' and '='. Every name carries the
// executor_ssh_reachability_ prefix.
var (
probeTotal = expvar.NewMap("executor_ssh_reachability_probe_total")
stateTransitionsTotal = expvar.NewMap("executor_ssh_reachability_state_transitions_total")
passSkippedTotal = expvar.NewInt("executor_ssh_reachability_pass_skipped_total")
writeRefusedTotal = expvar.NewInt("executor_ssh_reachability_write_refused_total")
readBackFailedTotal = expvar.NewInt("executor_ssh_reachability_read_back_failed_total")
publishFailedTotal = expvar.NewInt("executor_ssh_reachability_publish_failed_total")
resetTotal = expvar.NewInt("executor_ssh_reachability_reset_total")
probeDiscardedTotal = expvar.NewInt("executor_ssh_reachability_probe_discarded_total")
probeDurationMsLastRun = expvar.NewInt("executor_ssh_reachability_probe_duration_ms")
)
// reachabilityLabel builds a "k1=v1;k2=v2;..." expvar map key. Returns an
// empty label for an odd number of arguments rather than guessing.
func reachabilityLabel(pairs ...string) string {
if len(pairs)%2 != 0 {
return ""
}
parts := make([]string, 0, len(pairs)/2)
for i := 0; i < len(pairs); i += 2 {
parts = append(parts, pairs[i]+"="+pairs[i+1])
}
return strings.Join(parts, ";")
}
// recordProbeOutcome increments probeTotal labelled by outcome: "reachable"
// on success, or the failure reason string otherwise.
func recordProbeOutcome(success bool, reason string) {
outcome := reason
if success {
outcome = "reachable"
}
probeTotal.Add(reachabilityLabel("outcome", outcome), 1)
}
// recordStateTransition increments stateTransitionsTotal labelled by the
// destination state.
func recordStateTransition(destination string) {
stateTransitionsTotal.Add(reachabilityLabel("state", destination), 1)
}
// RecordReset increments resetTotal. This package never calls
// ResetExecutorReachability itself — task 04's save observer is the only
// caller of that method — but the counter is exported here so it lives with
// the rest of this package's reachability metrics rather than being
// duplicated at the call site.
func RecordReset() {
resetTotal.Add(1)
}
@@ -0,0 +1,297 @@
package reachability
import (
"context"
"sync"
"time"
"go.uber.org/zap"
"golang.org/x/sync/singleflight"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
)
// probeFunc is the injectable seam over a single probe attempt. defaultProbe
// is the production implementation; tests substitute a fake so poller
// lifecycle/concurrency behavior can be verified under testing/synctest
// without real SSH or network I/O.
type probeFunc func(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome
// Poller sweeps every eligible SSH executor on intervalSeconds, probing each
// with bounded concurrency and persisting results through store. An
// intervalSeconds of 0 disables the scheduled sweep entirely — the poller
// still starts (so ProbeNow keeps working) but never runs a pass on its own.
//
// Start and Stop are idempotent. Stop cancels every in-flight unit of work
// registered on wg — the scheduled loop, the pass it is running, and any
// off-cycle ProbeNow call — and waits for all of them to drain before
// returning, so a probe cancelled by Stop is guaranteed to never be
// persisted (see probeAndPersist).
type Poller struct {
store *store
log *logger.Logger
intervalSeconds int
probe probeFunc
// publisher is optional: nil until SetPublisher wires the event bus, so
// task 03's own tests and callers that never call it keep working
// unchanged. Set once at construction time in production.
publisher *Publisher
// inflight coalesces concurrent ProbeAndWait callers for the same
// executor id into a single dial. Zero value is ready to use.
inflight singleflight.Group
// probeSem is owned by the poller rather than by a scheduled pass. This
// keeps scheduled, manual, and synchronous probes under one cap.
probeSem chan struct{}
// mu guards started/stopping/ctx/cancel/wg against concurrent
// Start/Stop/ProbeNow calls. acquire() registers new work on wg under mu
// so it can never race a concurrent Stop's wg.Wait() observing the
// counter transition from zero.
mu sync.Mutex
started bool
stopping bool
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
stopDone chan struct{}
// passMu guards passRunning, the single-flight gate for the scheduled
// pass: an overlapping tick is dropped and counted, never queued.
passMu sync.Mutex
passRunning bool
}
// New builds a Poller. intervalSeconds is clamped via ClampInterval; a
// clamp is logged once rather than refusing to start.
func New(repo Repository, intervalSeconds int, log *logger.Logger) *Poller {
effective, clamped := ClampInterval(intervalSeconds)
if clamped {
log.Warn("executor ssh reachability: interval clamped",
zap.Int("configured", intervalSeconds), zap.Int("effective", effective))
}
return &Poller{
store: &store{repo: repo, log: log},
log: log,
intervalSeconds: effective,
probe: defaultProbe,
probeSem: make(chan struct{}, passConcurrency),
}
}
// SetPublisher wires the event publisher used to announce a state/reason
// change. Optional — a Poller with no publisher still probes and persists
// exactly as before, it just never publishes.
func (p *Poller) SetPublisher(publisher *Publisher) {
p.publisher = publisher
}
// Start launches the background loop. Calling Start more than once without
// an intervening Stop is a no-op.
func (p *Poller) Start(ctx context.Context) {
p.mu.Lock()
if p.started {
p.mu.Unlock()
return
}
runCtx, cancel := context.WithCancel(ctx)
p.started = true
p.stopping = false
p.ctx = runCtx
p.cancel = cancel
p.stopDone = make(chan struct{})
if p.intervalSeconds > 0 {
p.wg.Add(1)
}
p.mu.Unlock()
if p.intervalSeconds > 0 {
go p.loop(runCtx)
}
}
// Stop cancels the loop and every in-flight probe, then waits for all of
// them to drain. Calling Stop before Start, or twice in a row, is a no-op.
func (p *Poller) Stop() {
p.mu.Lock()
if !p.started {
p.mu.Unlock()
return
}
stopDone := p.stopDone
if p.stopping {
p.mu.Unlock()
<-stopDone
return
}
p.stopping = true
cancel := p.cancel
p.mu.Unlock()
cancel()
p.wg.Wait()
p.mu.Lock()
p.started = false
p.stopping = false
p.ctx = nil
p.cancel = nil
if p.stopDone == stopDone {
close(stopDone)
p.stopDone = nil
}
p.mu.Unlock()
}
// ProbeNow runs one off-cycle probe for executor, outside the scheduled
// cadence, tracked on the same WaitGroup/context as the scheduled loop so
// Stop drains it too. Returns false without doing anything when the poller
// is not currently accepting new work.
func (p *Poller) ProbeNow(executor *models.Executor) bool {
ctx, ok := p.acquire()
if !ok {
return false
}
go func() {
defer p.wg.Done()
p.probeAndPersist(ctx, executor)
}()
return true
}
func (p *Poller) acquireProbeSlot(ctx context.Context) bool {
select {
case p.probeSem <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
func (p *Poller) releaseProbeSlot() {
<-p.probeSem
}
// acquire registers one unit of in-flight work on wg, gated by mu so it can
// never race a concurrent Stop's wg.Wait(). It refuses when the poller isn't
// started or a Stop is already underway.
func (p *Poller) acquire() (context.Context, bool) {
p.mu.Lock()
defer p.mu.Unlock()
if !p.started || p.stopping {
return nil, false
}
// loop already owns a WaitGroup reference, so this positive Add cannot
// race a Stop waiting for the counter to reach zero.
// Safe without passMu: dispatchPass is called only from loop, which holds
// its own WaitGroup slot, so Stop cannot observe zero before this Add.
p.wg.Add(1)
return p.ctx, true
}
func (p *Poller) loop(ctx context.Context) {
defer p.wg.Done()
ticker := time.NewTicker(time.Duration(p.intervalSeconds) * time.Second)
defer ticker.Stop()
p.dispatchPass(ctx)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
p.dispatchPass(ctx)
}
}
}
// dispatchPass starts one pass if none is currently running. An overlapping
// dispatch is dropped and counted rather than queued, so a slow pass never
// builds up a backlog of pending passes. The pass itself runs on its own
// WaitGroup unit so Stop drains it even after loop itself has returned.
func (p *Poller) dispatchPass(ctx context.Context) {
p.passMu.Lock()
if p.passRunning {
p.passMu.Unlock()
passSkippedTotal.Add(1)
return
}
p.passRunning = true
p.passMu.Unlock()
p.wg.Add(1)
go func() {
defer p.wg.Done()
defer func() {
p.passMu.Lock()
p.passRunning = false
p.passMu.Unlock()
}()
p.runPass(ctx)
}()
}
// runPass lists every eligible executor and probes each with bounded
// concurrency. A listing failure abandons the pass without writing anything.
// Cancellation stops dispatching new probes and waits only for the ones
// already started to finish or self-cancel.
func (p *Poller) runPass(ctx context.Context) {
start := time.Now()
executors, err := p.store.repo.ListSSHExecutorsForReachability(ctx)
if err != nil {
p.log.Warn("executor ssh reachability: list failed", zap.Error(err))
return
}
var wg sync.WaitGroup
for _, executor := range executors {
if !p.acquireProbeSlot(ctx) {
wg.Wait()
return
}
wg.Add(1)
go func(executor *models.Executor) {
defer wg.Done()
defer p.releaseProbeSlot()
p.probeAndPersistWithSlot(ctx, executor)
}(executor)
}
wg.Wait()
probeDurationMsLastRun.Set(time.Since(start).Milliseconds())
}
// probeAndPersist runs one probe and records it, unless the probe was
// cancelled — a cancelled outcome is not an observation about the host and
// must never be written (see agentruntime.SSHProbeOutcome.Cancelled).
func (p *Poller) probeAndPersist(ctx context.Context, executor *models.Executor) {
if !p.acquireProbeSlot(ctx) {
return
}
defer p.releaseProbeSlot()
p.probeAndPersistWithSlot(ctx, executor)
}
func (p *Poller) probeAndPersistWithSlot(ctx context.Context, executor *models.Executor) {
outcome := p.probe(ctx, executor)
if outcome.Cancelled {
probeDiscardedTotal.Add(1)
return
}
p.observeAndPublish(ctx, executor, outcome)
}
// observeAndPublish records outcome through store.Observe and, when the
// write actually changed the stored state or reason, announces it through
// the publisher — the one propagation path every probe primitive
// (scheduled pass, ProbeNow, ProbeAndWait) shares.
func (p *Poller) observeAndPublish(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome) observeResult {
result := p.store.Observe(ctx, executor, outcome, time.Now().UTC())
if result.Changed && p.publisher != nil {
p.publisher.PublishChanged(ctx, result.After, p.EffectiveIntervalSeconds())
}
return result
}
@@ -0,0 +1,70 @@
package reachability
import (
"context"
"testing"
"time"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// A scheduled-pass probe (ProbeNow's own code path, probeAndPersist) must
// publish exactly like the immediate-probe route on a real change, and stay
// silent on a repeat that changes nothing — the "one propagation path" rule
// from the system design applies regardless of which primitive ran the
// probe.
func TestPollerProbeNow_PublishesOnlyOnChange(t *testing.T) {
eventBus := bus.NewMemoryEventBus(logger.Default())
published := make(chan RecordDTO, 4)
_, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
published <- ev.Data.(RecordDTO)
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
repo := newFakeRepository()
p := New(repo, 60, logger.Default())
p.SetPublisher(NewPublisher(eventBus))
p.Start(context.Background())
defer p.Stop()
executor := sshExecutor("exec-publish")
// First success: unknown -> reachable is a change.
p.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
}
if !p.ProbeNow(executor) {
t.Fatal("ProbeNow refused")
}
select {
case dto := <-published:
if dto.ExecutorID != executor.ID || dto.State != "reachable" {
t.Fatalf("dto = %+v, want reachable exec-publish", dto)
}
case <-time.After(time.Second):
t.Fatal("no event published for the first, change-carrying probe")
}
// Second success, same outcome: reachable -> reachable is not a change.
if !p.ProbeNow(executor) {
t.Fatal("ProbeNow refused")
}
// Stop drains the second ProbeNow's probeAndPersist goroutine (it is
// registered on the same WaitGroup) before we check for the absence of a
// publish — a happens-before barrier instead of a raw sleep.
p.Stop()
select {
case dto := <-published:
t.Fatalf("unexpected event published for a repeat outcome: %+v", dto)
default:
}
}
@@ -0,0 +1,445 @@
package reachability
import (
"context"
"errors"
"sort"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
)
// fakeRepository is an in-memory Repository double for poller tests: it
// isolates poller mechanics (scheduling, concurrency, lifecycle) from the
// SQL-backed hysteresis contract, which store_test.go already covers
// end-to-end against a real database.
type fakeRepository struct {
mu sync.Mutex
executors []*models.Executor
records map[string]*models.ExecutorReachability
listErr error
upsertErr error
upsertCalls int32
}
// errRepoRefused simulates a repository-level write failure — a test double
// for the last-write-wins WHERE clause silently refusing a write, without
// needing a real SQLite race to reproduce it.
var errRepoRefused = errors.New("write refused")
func newFakeRepository(executors ...*models.Executor) *fakeRepository {
return &fakeRepository{executors: executors, records: map[string]*models.ExecutorReachability{}}
}
func (f *fakeRepository) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.listErr != nil {
return nil, f.listErr
}
out := append([]*models.Executor(nil), f.executors...)
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out, nil
}
func (f *fakeRepository) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
f.mu.Lock()
defer f.mu.Unlock()
rec, ok := f.records[executorID]
if !ok {
return nil, models.ErrExecutorReachabilityNotFound
}
cp := *rec
return &cp, nil
}
func (f *fakeRepository) UpsertExecutorReachability(_ context.Context, obs models.ExecutorReachabilityObservation) error {
f.mu.Lock()
defer f.mu.Unlock()
atomic.AddInt32(&f.upsertCalls, 1)
if f.upsertErr != nil {
return f.upsertErr
}
state := obs.InitialState
failures := obs.InitialFailures
if existing, ok := f.records[obs.ExecutorID]; ok {
if obs.Reason == "" {
failures = 0
state = models.ExecutorReachabilityStateReachable
} else {
failures = existing.ConsecutiveFailures + 1
state = existing.State
if isStickyReason(obs.Reason) || failures >= obs.FailureThreshold {
state = models.ExecutorReachabilityStateUnreachable
}
}
}
f.records[obs.ExecutorID] = &models.ExecutorReachability{
ExecutorID: obs.ExecutorID, State: state, Reason: obs.Reason, Message: obs.Message,
ConsecutiveFailures: failures, Host: obs.Host, CheckedAt: timePtr(obs.CheckedAt), UpdatedAt: time.Now().UTC(),
}
return nil
}
func (f *fakeRepository) ResetExecutorReachability(_ context.Context, executorID, host string, _ time.Time) error {
f.mu.Lock()
defer f.mu.Unlock()
f.records[executorID] = &models.ExecutorReachability{ExecutorID: executorID, State: models.ExecutorReachabilityStateUnknown, Host: host, UpdatedAt: time.Now().UTC()}
return nil
}
func (f *fakeRepository) upsertCount() int32 { return atomic.LoadInt32(&f.upsertCalls) }
func timePtr(t time.Time) *time.Time { return &t }
func sshExecutor(id string) *models.Executor {
return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive, Config: map[string]string{"ssh_host": id}}
}
// concurrencyProbe blocks on a gate until released, tracking the maximum
// number of simultaneous in-flight calls it observed.
type concurrencyProbe struct {
mu sync.Mutex
inFlight int
maxSeen int
release chan struct{}
}
func newConcurrencyProbe() *concurrencyProbe {
return &concurrencyProbe{release: make(chan struct{})}
}
func (c *concurrencyProbe) probe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
c.mu.Lock()
c.inFlight++
if c.inFlight > c.maxSeen {
c.maxSeen = c.inFlight
}
c.mu.Unlock()
select {
case <-c.release:
case <-ctx.Done():
c.mu.Lock()
c.inFlight--
c.mu.Unlock()
return agentruntime.SSHProbeOutcome{Host: executor.ID, Cancelled: true}
}
c.mu.Lock()
c.inFlight--
c.mu.Unlock()
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
func TestPollerStart_RunsImmediatePass(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"), sshExecutor("b"))
p := New(repo, 60, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
defer p.Stop()
synctest.Wait()
if got := repo.upsertCount(); got != 2 {
t.Fatalf("upsert calls = %d, want 2 (one immediate pass over two executors)", got)
}
})
}
func TestPollerStart_IsIdempotent(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 60, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
p.Start(ctx) // second call must be a no-op
synctest.Wait()
p.Stop()
if got := repo.upsertCount(); got != 1 {
t.Fatalf("upsert calls = %d, want exactly 1 (a second Start must not spawn a parallel loop)", got)
}
})
}
func TestPollerStop_BeforeStart_IsNoOp(t *testing.T) {
p := New(newFakeRepository(), 60, logger.Default())
p.Stop() // must not block or panic
}
func TestPollerZeroInterval_DisablesScheduledPass(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 0, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
defer p.Stop()
synctest.Wait()
// Advance well past what would have been several ticks at the
// default interval, to prove no pass ever runs.
time.Sleep(10 * time.Minute)
synctest.Wait()
if got := repo.upsertCount(); got != 0 {
t.Fatalf("upsert calls = %d, want 0 (interval=0 must run no scheduled pass)", got)
}
})
}
func TestPollerZeroInterval_ProbeNowStillWorks(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 0, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
defer p.Stop()
synctest.Wait()
if ok := p.ProbeNow(sshExecutor("a")); !ok {
t.Fatal("ProbeNow returned false while the poller is started")
}
synctest.Wait()
if got := repo.upsertCount(); got != 1 {
t.Fatalf("upsert calls = %d, want 1 from the off-cycle probe", got)
}
})
}
func TestPollerPass_NeverExceedsPassConcurrency(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var executors []*models.Executor
for i := 0; i < passConcurrency+3; i++ {
executors = append(executors, sshExecutor(string(rune('a'+i))))
}
repo := newFakeRepository(executors...)
p := New(repo, 60, logger.Default())
cp := newConcurrencyProbe()
p.probe = cp.probe
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
// Let every worker reach the gate.
synctest.Wait()
close(cp.release)
p.Stop()
cp.mu.Lock()
defer cp.mu.Unlock()
if cp.maxSeen > passConcurrency {
t.Fatalf("max simultaneous probes = %d, want at most passConcurrency (%d)", cp.maxSeen, passConcurrency)
}
if cp.maxSeen != passConcurrency {
t.Fatalf("max simultaneous probes = %d, want exactly passConcurrency (%d) since more executors than slots were queued", cp.maxSeen, passConcurrency)
}
})
}
func TestPollerProbeEntrypointsShareConcurrencyLimit(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var executors []*models.Executor
for i := 0; i < passConcurrency; i++ {
executors = append(executors, sshExecutor(string(rune('a'+i))))
}
repo := newFakeRepository(executors...)
p := New(repo, 60, logger.Default())
cp := newConcurrencyProbe()
p.probe = cp.probe
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
synctest.Wait()
if ok := p.ProbeNow(sshExecutor("manual")); !ok {
t.Fatal("ProbeNow returned false while the poller was running")
}
synctest.Wait()
cp.mu.Lock()
maxSeen := cp.maxSeen
cp.mu.Unlock()
close(cp.release)
p.Stop()
if maxSeen > passConcurrency {
t.Fatalf("max simultaneous probes = %d, want at most %d across scheduled and manual probes", maxSeen, passConcurrency)
}
})
}
func TestPollerConcurrentStopWaitsForTheFirstStop(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 0, logger.Default())
release := make(chan struct{})
var releaseOnce sync.Once
var inFlight atomic.Int32
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
inFlight.Add(1)
defer inFlight.Add(-1)
<-release
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.ID}
}
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
p.Start(context.Background())
if ok := p.ProbeNow(sshExecutor("stop-race")); !ok {
t.Fatal("ProbeNow returned false while the poller was running")
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if inFlight.Load() == 1 {
break
}
time.Sleep(time.Millisecond)
}
firstDone := make(chan struct{})
go func() { p.Stop(); close(firstDone) }()
for time.Now().Before(deadline) {
p.mu.Lock()
stopping := p.stopping
p.mu.Unlock()
if stopping {
break
}
time.Sleep(time.Millisecond)
}
secondDone := make(chan struct{})
go func() { p.Stop(); close(secondDone) }()
select {
case <-secondDone:
t.Fatal("concurrent Stop returned before the first Stop drained the probe")
case <-time.After(20 * time.Millisecond):
}
releaseOnce.Do(func() { close(release) })
select {
case <-firstDone:
case <-time.After(time.Second):
t.Fatal("first Stop did not finish")
}
select {
case <-secondDone:
case <-time.After(time.Second):
t.Fatal("concurrent Stop did not finish after the first Stop")
}
}
func TestPollerPass_OverlappingTickIsSkippedNotQueued(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 60, logger.Default())
cp := newConcurrencyProbe()
p.probe = cp.probe
before := passSkippedTotal.Value()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
defer func() {
close(cp.release)
p.Stop()
}()
// The immediate pass is now blocked on the probe gate. Advance past
// a full tick while it is still running.
synctest.Wait()
time.Sleep(61 * time.Second)
synctest.Wait()
if got := passSkippedTotal.Value() - before; got != 1 {
t.Fatalf("pass_skipped_total delta = %d, want 1 (the overlapping tick must be dropped, not queued)", got)
}
})
}
func TestPollerStop_DiscardsCancelledProbeResult(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 60, logger.Default())
cp := newConcurrencyProbe()
p.probe = cp.probe
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
// The immediate pass is now blocked on the probe gate (never
// released), so Stop must cancel it in flight.
synctest.Wait()
p.Stop()
if got := repo.upsertCount(); got != 0 {
t.Fatalf("upsert calls = %d, want 0 — a probe cancelled by Stop must never be written", got)
}
})
}
func TestPollerProbeNow_RefusedAfterStop(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
p := New(repo, 60, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
p.Stop()
if ok := p.ProbeNow(sshExecutor("a")); ok {
t.Fatal("ProbeNow returned true after Stop; a stopped poller must refuse new work")
}
}
func TestPollerListFailure_AbandonsPassWithoutWriting(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
repo := newFakeRepository(sshExecutor("a"))
repo.listErr = context.DeadlineExceeded
p := New(repo, 60, logger.Default())
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Start(ctx)
defer p.Stop()
synctest.Wait()
if got := repo.upsertCount(); got != 0 {
t.Fatalf("upsert calls = %d, want 0 when listing executors fails", got)
}
})
}
@@ -0,0 +1,22 @@
package reachability
import (
"context"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/task/models"
)
// reachabilityProber is the production seam over the runtime's SSH
// reachability prober. Tests substitute probeFunc directly, so this stays
// unexported and unmocked.
var reachabilityProber = agentruntime.NewSSHReachabilityProber()
// defaultProbe resolves executor's SSH target from its persisted config and
// probes it. A resolution failure (missing host, invalid port, unresolvable
// alias) never dials — it is reported as
// agentruntime.SSHReachabilityReasonConfig directly, the same reason a
// target with no pinned fingerprint uses.
func defaultProbe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return reachabilityProber.Probe(ctx, executor.Config, probeTimeout)
}
@@ -0,0 +1,102 @@
package reachability
import (
"strings"
"github.com/kandev/kandev/internal/task/models"
)
// ProbeResult is what ProbeAndWait returns: the durable record when available,
// or the locally projected observation when persistence failed, plus whether
// that specific write actually landed.
type ProbeResult struct {
Record *models.ExecutorReachability
Persisted bool
}
// probeAndWaitOutcome is the value singleflight.Group shares across every
// coalesced caller for one key. It is deliberately independent of any
// caller's ProbeResult — ProbeAndWait copies out of it before returning, so
// no caller can mutate what another caller already received.
type probeAndWaitOutcome struct {
ran bool
persisted bool
record *models.ExecutorReachability
}
// ProbeAndWait runs one probe for executor synchronously and returns its
// persisted record, coalescing overlapping callers for the same executor id
// into a single dial via singleflight — the primitive behind the immediate-
// probe HTTP route. Unlike ProbeNow, this does not accept (or run on) any
// caller-supplied context: like every other off-cycle probe it runs on the
// poller's own context/WaitGroup, registered through acquire(), so a
// disconnecting HTTP caller neither cancels it nor affects a caller still
// waiting on the same coalesced result, and Stop still drains it.
//
// ran is false only when the poller itself refused new work (not started, or
// stopping); it does not report whether the probe outcome was a success —
// see ProbeResult.Persisted for whether the result was actually stored.
func (p *Poller) ProbeAndWait(executor *models.Executor) (ProbeResult, bool) {
v, _, _ := p.inflight.Do(probeKey(executor), func() (any, error) {
return p.probeAndWaitOnce(executor), nil
})
outcome := v.(probeAndWaitOutcome)
if !outcome.ran {
return ProbeResult{}, false
}
var record *models.ExecutorReachability
if outcome.record != nil {
cp := *outcome.record
record = &cp
}
return ProbeResult{Record: record, Persisted: outcome.persisted}, true
}
// probeKey includes the connection identity, not only the executor ID. A
// save can change the host or its pinned key while an older probe is still in
// flight; those observations must not share a singleflight result.
func probeKey(executor *models.Executor) string {
if executor == nil {
return "<nil>"
}
values := make([]string, 0, len(connectionConfigKeys)+1)
values = append(values, executor.ID)
for _, key := range connectionConfigKeys {
values = append(values, key+"="+executor.Config[key])
}
return strings.Join(values, "\x00")
}
// probeAndWaitOnce is the singleflight leader's body: acquire a WaitGroup
// slot exactly like every other off-cycle unit of work, probe, and persist.
func (p *Poller) probeAndWaitOnce(executor *models.Executor) probeAndWaitOutcome {
ctx, ok := p.acquire()
if !ok {
return probeAndWaitOutcome{}
}
defer p.wg.Done()
if !p.acquireProbeSlot(ctx) {
return probeAndWaitOutcome{ran: true}
}
defer p.releaseProbeSlot()
outcome := p.probe(ctx, executor)
if outcome.Cancelled {
probeDiscardedTotal.Add(1)
return probeAndWaitOutcome{ran: true}
}
result := p.observeAndPublish(ctx, executor, outcome)
record := result.After
if record == nil {
record = result.Observed
}
return probeAndWaitOutcome{ran: true, persisted: result.Applied, record: record}
}
// EffectiveIntervalSeconds returns the clamped interval this poller is
// running with (0 when scheduled probing is disabled), for a caller (the
// reachability DTO) that needs to report probing_enabled/
// probe_interval_seconds without duplicating ClampInterval's rules.
func (p *Poller) EffectiveIntervalSeconds() int {
return p.intervalSeconds
}
@@ -0,0 +1,232 @@
package reachability
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
"time"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
)
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
//
// Two overlapping immediate-probe callers for the same executor must
// coalesce into exactly one dial and receive one identical record each — the
// acceptance criterion this task adds ProbeAndWait to satisfy. The blocking
// probeFunc plus a short settle delay before releasing mirrors
// golang.org/x/sync/singleflight's own TestDoDupSuppress: there is no faster
// deterministic signal for "the second caller has entered Do" without
// instrumenting production code for the test.
func TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 60, logger.Default())
p.Start(context.Background())
defer p.Stop()
var dialCount int32
started := make(chan struct{})
release := make(chan struct{})
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
if atomic.AddInt32(&dialCount, 1) == 1 {
close(started)
}
<-release
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
executor := sshExecutor("exec-coalesce")
type call struct {
result ProbeResult
ran bool
}
results := make(chan call, 2)
go func() {
result, ran := p.ProbeAndWait(executor)
results <- call{result, ran}
}()
<-started
go func() {
result, ran := p.ProbeAndWait(executor)
results <- call{result, ran}
}()
time.Sleep(100 * time.Millisecond)
close(release)
first := <-results
second := <-results
if got := atomic.LoadInt32(&dialCount); got != 1 {
t.Fatalf("dial count = %d, want exactly 1 (coalesced)", got)
}
if !first.ran || !second.ran {
t.Fatalf("ran = %v, %v, want both true", first.ran, second.ran)
}
firstJSON, _ := json.Marshal(first.result)
secondJSON, _ := json.Marshal(second.result)
if string(firstJSON) != string(secondJSON) {
t.Fatalf("responses differ:\n first: %s\n second: %s", firstJSON, secondJSON)
}
if !first.result.Persisted {
t.Fatalf("Persisted = false, want true — the probe outcome was a plain success")
}
}
func TestPollerProbeAndWaitDoesNotCoalesceDifferentConnectionConfigs(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 0, logger.Default())
p.Start(context.Background())
defer p.Stop()
var dialCount int32
started := make(chan struct{}, 2)
release := make(chan struct{})
var releaseOnce sync.Once
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
atomic.AddInt32(&dialCount, 1)
started <- struct{}{}
<-release
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
first := sshExecutor("same-id")
first.Config["ssh_host"] = "host-a"
second := sshExecutor("same-id")
second.Config["ssh_host"] = "host-b"
results := make(chan ProbeResult, 2)
go func() { result, _ := p.ProbeAndWait(first); results <- result }()
go func() { result, _ := p.ProbeAndWait(second); results <- result }()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("first probe did not start")
}
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("second connection config incorrectly coalesced with the first")
}
releaseOnce.Do(func() { close(release) })
<-results
<-results
if got := atomic.LoadInt32(&dialCount); got != 2 {
t.Fatalf("dial count = %d, want 2 for distinct connection configs", got)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
//
// The singleflight group shares one result across coalesced callers. Each
// caller must get its own copy of the record, not a shared pointer, so one
// caller mutating its response cannot corrupt what the other already
// received (the risk this task's work order names explicitly).
func TestPollerProbeAndWait_ReturnsIndependentRecordCopies(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 60, logger.Default())
p.Start(context.Background())
defer p.Stop()
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
executor := sshExecutor("exec-copy")
result, ran := p.ProbeAndWait(executor)
if !ran {
t.Fatalf("ran = false, want true")
}
if result.Record == nil {
t.Fatalf("Record = nil, want the stored record")
}
originalHost := result.Record.Host
result.Record.Host = "mutated-in-caller"
second, ran := p.ProbeAndWait(executor)
if !ran {
t.Fatalf("second call: ran = false, want true")
}
if second.Record.Host == "mutated-in-caller" {
t.Fatalf("second caller observed the first caller's mutation")
}
if second.Record.Host != originalHost {
t.Fatalf("second.Record.Host = %q, want %q", second.Record.Host, originalHost)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.28
//
// The immediate probe must work while the scheduled poller is disabled
// (interval 0) — ProbeAndWait must not depend on the loop ever having run.
func TestPollerProbeAndWait_WorksWhilePollerDisabled(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 0, logger.Default())
p.Start(context.Background())
defer p.Stop()
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
result, ran := p.ProbeAndWait(sshExecutor("exec-disabled"))
if !ran {
t.Fatalf("ran = false, want true even though the scheduled poller is disabled")
}
if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("result = %+v, want a persisted reachable record", result)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
//
// A probe that ran but was not stored (the repository refused the write)
// still reports ran=true with Persisted=false, never an error — the route
// this backs answers 200 either way.
func TestPollerProbeAndWait_WriteRefusedReportsPersistedFalse(t *testing.T) {
repo := newFakeRepository()
repo.upsertErr = errRepoRefused
p := New(repo, 60, logger.Default())
p.Start(context.Background())
defer p.Stop()
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
result, ran := p.ProbeAndWait(sshExecutor("exec-refused"))
if !ran {
t.Fatalf("ran = false, want true")
}
if result.Persisted {
t.Fatalf("Persisted = true, want false — the repository write failed")
}
if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("Record = %+v, want the actual unsaved reachable observation", result.Record)
}
}
// TestPollerProbeAndWait_RefusedAfterStop mirrors
// TestPollerProbeNow_RefusedAfterStop: once Stop has run, ProbeAndWait must
// not dial at all.
func TestPollerProbeAndWait_RefusedAfterStop(t *testing.T) {
repo := newFakeRepository()
p := New(repo, 60, logger.Default())
p.Start(context.Background())
p.Stop()
dialed := false
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
dialed = true
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
}
_, ran := p.ProbeAndWait(sshExecutor("exec-stopped"))
if ran {
t.Fatalf("ran = true, want false — the poller was stopped")
}
if dialed {
t.Fatalf("probe was dialed after Stop")
}
}
@@ -0,0 +1,105 @@
package reachability
import (
"context"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
// RecordDTO is the wire shape every reachability HTTP route and the change
// event share, so a client's WS handler and its GET/POST response handler
// stay structurally identical. Reason is always present, even empty — the
// surface's contract is that a reachable/unknown state can carry a non-empty
// reason (a below-threshold failure), so an omitted field would be
// ambiguous with "definitely no reason".
type RecordDTO struct {
ExecutorID string `json:"executor_id"`
State string `json:"state"`
Reason string `json:"reason"`
Message string `json:"message,omitempty"`
ConsecutiveFailures int `json:"consecutive_failures"`
Host string `json:"host,omitempty"`
CheckedAt *time.Time `json:"checked_at"`
LastSuccessAt *time.Time `json:"last_success_at"`
UpdatedAt *time.Time `json:"updated_at"`
ProbingEnabled bool `json:"probing_enabled"`
ProbeIntervalSeconds int `json:"probe_interval_seconds"`
Persisted bool `json:"persisted"`
}
// BuildRecordDTO projects record into the wire shape. A nil record (an
// eligible executor never probed) synthesizes the "unknown" placeholder with
// every timestamp null, per AC-EXECUTORS-SSH-REACHABILITY-002.5 — this is
// never a 404, so every caller building a GET response for an eligible
// executor can call this unconditionally.
func BuildRecordDTO(executorID string, record *models.ExecutorReachability, effectiveIntervalSeconds int, persisted bool) RecordDTO {
dto := RecordDTO{
ExecutorID: executorID,
State: string(models.ExecutorReachabilityStateUnknown),
ProbingEnabled: effectiveIntervalSeconds > 0,
ProbeIntervalSeconds: effectiveIntervalSeconds,
Persisted: persisted,
}
if record == nil {
return dto
}
dto.State = string(record.State)
dto.Reason = string(record.Reason)
dto.Message = record.Message
dto.ConsecutiveFailures = record.ConsecutiveFailures
dto.Host = record.Host
dto.CheckedAt = record.CheckedAt
dto.LastSuccessAt = record.LastSuccessAt
if !record.UpdatedAt.IsZero() {
updatedAt := record.UpdatedAt
dto.UpdatedAt = &updatedAt
}
return dto
}
// Publisher publishes events.ExecutorReachabilityChanged. Constructed once
// at startup wiring and shared by the poller (probe results) and the save
// observer (configuration resets) — one propagation path for both triggers,
// per the system design.
type Publisher struct {
eventBus bus.EventBus
log *logger.Logger
}
// NewPublisher builds a Publisher. A nil eventBus is accepted so a caller
// that hasn't wired the event bus yet (tests, a degraded boot path) can
// still construct one; PublishChanged becomes a no-op. An optional logger
// records publication failures with the affected executor id.
func NewPublisher(eventBus bus.EventBus, logs ...*logger.Logger) *Publisher {
var log *logger.Logger
if len(logs) > 0 {
log = logs[0]
}
return &Publisher{eventBus: eventBus, log: log}
}
// PublishChanged publishes record's current shape on
// events.ExecutorReachabilityChanged. Safe to call on a nil *Publisher, with
// a nil record, or with a Publisher built from a nil event bus — every
// combination is a no-op rather than a panic, so callers on the probe/reset
// hot paths don't need a defensive check of their own.
func (p *Publisher) PublishChanged(ctx context.Context, record *models.ExecutorReachability, effectiveIntervalSeconds int) {
if p == nil || p.eventBus == nil || record == nil {
return
}
dto := BuildRecordDTO(record.ExecutorID, record, effectiveIntervalSeconds, true)
event := bus.NewEvent(events.ExecutorReachabilityChanged, "executor-reachability", dto)
if err := p.eventBus.Publish(ctx, events.ExecutorReachabilityChanged, event); err != nil {
publishFailedTotal.Add(1)
if p.log != nil {
p.log.Warn("executor ssh reachability: publish failed",
zap.String("executor_id", record.ExecutorID), zap.Error(err))
}
}
}
@@ -0,0 +1,158 @@
package reachability
import (
"context"
"errors"
"testing"
"time"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
type failingPublishEventBus struct {
*bus.MemoryEventBus
err error
}
func (b *failingPublishEventBus) Publish(context.Context, string, *bus.Event) error {
return b.err
}
func testEventBusLogger() *bus.MemoryEventBus {
return bus.NewMemoryEventBus(logger.Default())
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// PublishChanged must publish on events.ExecutorReachabilityChanged with a
// payload carrying the record plus the derived probing fields a client needs
// (probing_enabled/probe_interval_seconds) without a second fetch.
func TestPublisherPublishChanged_PublishesRecordDTO(t *testing.T) {
eventBus := testEventBusLogger()
var captured *bus.Event
_, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
captured = ev
return nil
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
pub := NewPublisher(eventBus)
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
record := &models.ExecutorReachability{
ExecutorID: "exec-1",
State: models.ExecutorReachabilityStateUnreachable,
Reason: models.ExecutorReachabilityReasonNetwork,
Message: "connection refused",
Host: "10.0.0.1",
CheckedAt: &checkedAt,
}
pub.PublishChanged(context.Background(), record, 60)
if captured == nil {
t.Fatal("no event was published")
}
dto, ok := captured.Data.(RecordDTO)
if !ok {
t.Fatalf("event data type = %T, want RecordDTO", captured.Data)
}
if dto.ExecutorID != "exec-1" || dto.State != "unreachable" || dto.Reason != "network" {
t.Fatalf("dto = %+v, want executor-1/unreachable/network", dto)
}
if !dto.ProbingEnabled || dto.ProbeIntervalSeconds != 60 {
t.Fatalf("dto probing fields = %+v, want enabled/60", dto)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
//
// probing_enabled must be false exactly when the effective interval is 0 —
// the client's only way to distinguish "not probed yet" from "probing off".
func TestPublisherPublishChanged_ProbingDisabledWhenIntervalIsZero(t *testing.T) {
eventBus := testEventBusLogger()
var captured *bus.Event
_, _ = eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
captured = ev
return nil
})
pub := NewPublisher(eventBus)
pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 0)
dto := captured.Data.(RecordDTO)
if dto.ProbingEnabled {
t.Fatalf("ProbingEnabled = true, want false when the effective interval is 0")
}
}
// A nil publisher, nil record, or nil bus must never panic — PublishChanged
// is called from probe/save-reset paths that don't want a defensive nil
// check at every call site.
func TestPublisherPublishChanged_NilSafe(t *testing.T) {
var nilPub *Publisher
nilPub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
pub := NewPublisher(testEventBusLogger())
pub.PublishChanged(context.Background(), nil, 60)
pub2 := NewPublisher(nil)
pub2.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
}
func TestPublisherPublishChangedCountsPublishFailures(t *testing.T) {
before := publishFailedTotal.Value()
pub := NewPublisher(&failingPublishEventBus{
MemoryEventBus: testEventBusLogger(),
err: errors.New("event bus unavailable"),
})
pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
if got := publishFailedTotal.Value(); got != before+1 {
t.Fatalf("publishFailedTotal = %d, want %d", got, before+1)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14, AC-EXECUTORS-SSH-REACHABILITY-002.5
//
// BuildRecordDTO is the single place the HTTP routes and the event publisher
// share for projecting a stored (or absent) record into the wire shape —
// including the "no record yet" unknown placeholder with every timestamp
// null, never omitted.
func TestBuildRecordDTO_SynthesizesUnknownShapeWhenRecordIsNil(t *testing.T) {
dto := BuildRecordDTO("exec-1", nil, 60, true)
if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Reason != "" {
t.Fatalf("dto = %+v, want a synthesized unknown/empty-reason shape", dto)
}
if dto.CheckedAt != nil || dto.LastSuccessAt != nil || dto.UpdatedAt != nil {
t.Fatalf("dto = %+v, want every timestamp nil", dto)
}
if dto.ConsecutiveFailures != 0 {
t.Fatalf("ConsecutiveFailures = %d, want 0", dto.ConsecutiveFailures)
}
}
func TestBuildRecordDTO_ProjectsStoredRecord(t *testing.T) {
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
record := &models.ExecutorReachability{
ExecutorID: "exec-1",
State: models.ExecutorReachabilityStateReachable,
ConsecutiveFailures: 0,
Host: "10.0.0.1",
CheckedAt: &checkedAt,
LastSuccessAt: &checkedAt,
UpdatedAt: checkedAt,
}
dto := BuildRecordDTO("exec-1", record, 60, true)
if dto.State != "reachable" || dto.Host != "10.0.0.1" {
t.Fatalf("dto = %+v, want the stored record projected", dto)
}
if dto.CheckedAt == nil || !dto.CheckedAt.Equal(checkedAt) {
t.Fatalf("CheckedAt = %v, want %v", dto.CheckedAt, checkedAt)
}
if !dto.Persisted {
t.Fatalf("Persisted = false, want true")
}
}
@@ -0,0 +1,126 @@
package reachability
import (
"context"
"errors"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/task/models"
)
// connectionConfigKeys names the exact SSH connection-configuration fields —
// host, port, user, identity source, optional ProxyJump, and pinned host
// fingerprint — that a save-detection comparison must diff. Any other Config
// key (or a non-config field) changing is not connection-relevant and must
// not trigger a reset.
var connectionConfigKeys = []string{
"ssh_host",
"ssh_host_alias",
"ssh_port",
"ssh_user",
"ssh_identity_source",
"ssh_identity_file",
"ssh_proxy_jump",
"ssh_host_fingerprint",
}
// SaveObserver implements service.ExecutorSaveObserver: it resets the stored
// reachability record and dispatches an off-cycle probe whenever an active
// SSH executor's connection configuration changes. Declared in
// internal/task/service (this package structurally satisfies it there
// without an import back to it).
type SaveObserver struct {
poller *Poller
}
// NewSaveObserver builds a SaveObserver over poller. poller supplies the
// repository (for the reset and the pre-reset read), the publisher (for the
// change event), and ProbeNow (for the post-reset off-cycle probe) — the same
// primitives task 03/04 already exercise elsewhere in this package.
func NewSaveObserver(poller *Poller) *SaveObserver {
return &SaveObserver{poller: poller}
}
// OnExecutorSaved resets and re-probes after when it is an active SSH
// executor whose connection configuration changed. before is nil on create,
// which is always treated as a change — there is no prior state to compare.
func (o *SaveObserver) OnExecutorSaved(ctx context.Context, before, after *models.Executor) {
if !eligibleForReachability(after) {
return
}
if !connectionConfigChanged(before, after) {
return
}
o.resetAndProbe(ctx, after)
}
func eligibleForReachability(executor *models.Executor) bool {
return executor != nil &&
executor.Type == models.ExecutorTypeSSH &&
executor.Status == models.ExecutorStatusActive
}
func connectionConfigChanged(before, after *models.Executor) bool {
if before == nil {
return true
}
for _, key := range connectionConfigKeys {
if before.Config[key] != after.Config[key] {
return true
}
}
return false
}
// resetAndProbe resets the stored record to the new host, publishes a change
// event when doing so actually changes what a client would see, and dispatches
// an off-cycle probe. The pre-reset record is read first because after the
// reset the "was it worth publishing" comparison is no longer possible — the
// reset itself always leaves the unknown/empty-reason placeholder behind.
func (o *SaveObserver) resetAndProbe(ctx context.Context, executor *models.Executor) {
repo := o.poller.store.repo
previous, readErr := repo.GetExecutorReachability(ctx, executor.ID)
if readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound) {
o.poller.log.Warn("executor ssh reachability: read before reset failed",
zap.String("executor_id", executor.ID), zap.Error(readErr))
}
newHost := executor.Config["ssh_host"]
if err := repo.ResetExecutorReachability(ctx, executor.ID, newHost, executor.UpdatedAt); err != nil {
o.poller.log.Warn("executor ssh reachability: reset failed",
zap.String("executor_id", executor.ID), zap.Error(err))
return
}
if recordWorthPublishing(previous) || (readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound)) {
reset, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
o.poller.log.Warn("executor ssh reachability: read after reset failed",
zap.String("executor_id", executor.ID), zap.Error(err))
reset = &models.ExecutorReachability{
ExecutorID: executor.ID,
State: models.ExecutorReachabilityStateUnknown,
Host: newHost,
UpdatedAt: time.Now().UTC(),
}
}
if o.poller.publisher != nil {
o.poller.publisher.PublishChanged(ctx, reset, o.poller.EffectiveIntervalSeconds())
}
}
o.poller.ProbeNow(executor)
}
// recordWorthPublishing reports whether record differs from the synthesized
// unknown/empty-reason placeholder a client already sees for "no record yet"
// — publishing a reset that lands on that same placeholder would be a no-op
// event.
func recordWorthPublishing(record *models.ExecutorReachability) bool {
if record == nil {
return false
}
return record.State != models.ExecutorReachabilityStateUnknown || record.Reason != ""
}
@@ -0,0 +1,249 @@
package reachability
import (
"context"
"testing"
"time"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
)
// newTestSaveObserver builds a SaveObserver over a started Poller wired to
// repo, with probe replaced by a deterministic success so ProbeNow's
// dispatched probe is observable through repo's upsert count instead of real
// network I/O. Callers must defer poller.Stop().
func newTestSaveObserver(t *testing.T, repo *fakeRepository) (*SaveObserver, *Poller) {
t.Helper()
poller := New(repo, 0, logger.Default())
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
}
poller.Start(context.Background())
return NewSaveObserver(poller), poller
}
func sshExecutorWithConfig(id string, status models.ExecutorStatus, config map[string]string) *models.Executor {
return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: status, Config: config}
}
// waitForUpsertCount blocks until repo has observed at least want upserts, or
// fails the test after a short deadline — a happens-before proof that the
// observer's fire-and-forget ProbeNow dispatch actually ran, without a fixed
// sleep.
func waitForUpsertCount(t *testing.T, repo *fakeRepository, want int32) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if repo.upsertCount() >= want {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("upsertCount = %d, want >= %d", repo.upsertCount(), want)
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2
//
// A host change on an active SSH executor must reset the stored record to
// the new host and dispatch an off-cycle probe — the reset's whole purpose is
// that a stale unreachable/reachable record about the OLD host must never be
// shown against the new one, even for the single tick before the next probe
// lands.
func TestSaveObserverResetsAndProbesOnHostChange(t *testing.T) {
repo := newFakeRepository()
observer, poller := newTestSaveObserver(t, repo)
defer poller.Stop()
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"})
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"})
observer.OnExecutorSaved(context.Background(), before, after)
rec, err := repo.GetExecutorReachability(context.Background(), "exec-1")
if err != nil {
t.Fatalf("GetExecutorReachability: %v", err)
}
if rec.Host != "10.0.0.2" {
t.Fatalf("record.Host = %q, want the new host 10.0.0.2 (reset must run before the probe)", rec.Host)
}
waitForUpsertCount(t, repo, 1)
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// Creating an executor has no prior state to compare against, so it must
// always be treated as a change — before=nil is not "nothing changed".
func TestSaveObserverTreatsNilBeforeAsAlwaysChanged(t *testing.T) {
repo := newFakeRepository()
observer, poller := newTestSaveObserver(t, repo)
defer poller.Stop()
after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"})
observer.OnExecutorSaved(context.Background(), nil, after)
if _, err := repo.GetExecutorReachability(context.Background(), "exec-created"); err != nil {
t.Fatalf("GetExecutorReachability: %v, want a reset record from the create", err)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// A non-SSH executor save must never reset or probe — there is no
// reachability record for it to begin with.
func TestSaveObserverSkipsNonSSHExecutor(t *testing.T) {
repo := newFakeRepository()
observer, poller := newTestSaveObserver(t, repo)
defer poller.Stop()
after := &models.Executor{ID: "exec-local", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
observer.OnExecutorSaved(context.Background(), nil, after)
if _, err := repo.GetExecutorReachability(context.Background(), "exec-local"); err == nil {
t.Fatalf("expected no record for a non-SSH executor")
}
if repo.upsertCount() != 0 {
t.Fatalf("upsertCount = %d, want 0 (no probe should have been dispatched)", repo.upsertCount())
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// A disabled SSH executor must not be reset or probed — probing an executor
// no launch path can use wastes a dial and would misleadingly imply it is
// monitored.
func TestSaveObserverSkipsInactiveSSHExecutor(t *testing.T) {
repo := newFakeRepository()
observer, poller := newTestSaveObserver(t, repo)
defer poller.Stop()
after := sshExecutorWithConfig("exec-disabled", models.ExecutorStatusDisabled, map[string]string{"ssh_host": "10.0.0.5"})
observer.OnExecutorSaved(context.Background(), nil, after)
if _, err := repo.GetExecutorReachability(context.Background(), "exec-disabled"); err == nil {
t.Fatalf("expected no record for a disabled executor")
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// Only the named connection-configuration keys matter. A save that changes
// something else entirely (here: nothing in Config at all differs) must not
// reset an existing record — a reset would wipe legitimate history (state,
// consecutive_failures, last_success_at) for no reason tied to the host's
// actual reachability.
func TestSaveObserverSkipsWhenConnectionConfigUnchanged(t *testing.T) {
repo := newFakeRepository()
observer, poller := newTestSaveObserver(t, repo)
defer poller.Stop()
config := map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"}
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config)
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config)
observer.OnExecutorSaved(context.Background(), before, after)
if _, err := repo.GetExecutorReachability(context.Background(), "exec-1"); err == nil {
t.Fatalf("expected no record written when connection config is unchanged")
}
if repo.upsertCount() != 0 {
t.Fatalf("upsertCount = %d, want 0", repo.upsertCount())
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2, AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// A reset that actually changes what a client would see (the previously
// stored record was not already the unknown/empty-reason placeholder) must
// publish a change event, so a UI watching this executor doesn't have to poll
// to notice the reset.
func TestSaveObserverPublishesChangeEventWhenPriorRecordWasObserved(t *testing.T) {
repo := newFakeRepository()
eventBus := bus.NewMemoryEventBus(logger.Default())
published := make(chan RecordDTO, 4)
if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
published <- ev.Data.(RecordDTO)
return nil
}); err != nil {
t.Fatalf("subscribe: %v", err)
}
poller := New(repo, 0, logger.Default())
poller.SetPublisher(NewPublisher(eventBus))
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
}
poller.Start(context.Background())
defer poller.Stop()
observer := NewSaveObserver(poller)
checkedAt := time.Now().UTC()
repo.records["exec-1"] = &models.ExecutorReachability{
ExecutorID: "exec-1", State: models.ExecutorReachabilityStateReachable,
Host: "10.0.0.1", CheckedAt: &checkedAt, UpdatedAt: checkedAt,
}
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"})
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"})
observer.OnExecutorSaved(context.Background(), before, after)
select {
case dto := <-published:
if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Host != "10.0.0.2" {
t.Fatalf("dto = %+v, want the reset unknown/10.0.0.2 shape", dto)
}
case <-time.After(time.Second):
t.Fatal("no change event published for a reset over a previously observed record")
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// Resetting an executor that was never probed before (no stored record at
// all) is not itself a client-visible change — the client already sees the
// synthesized unknown placeholder for "no record yet" — so the reset step
// must not publish. The dispatched off-cycle probe that follows is a
// separate, legitimate publish opportunity (a real unknown -> reachable
// transition) and is deliberately excluded here via a blocking probe gate:
// OnExecutorSaved's synchronous portion (the reset and its publish decision)
// completes fully before the probe goroutine can even call probe, so
// asserting silence right after OnExecutorSaved returns isolates the reset
// step's own behavior without racing the probe.
func TestSaveObserverDoesNotPublishFromResetWhenNoPriorRecord(t *testing.T) {
repo := newFakeRepository()
eventBus := bus.NewMemoryEventBus(logger.Default())
published := make(chan RecordDTO, 4)
if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
published <- ev.Data.(RecordDTO)
return nil
}); err != nil {
t.Fatalf("subscribe: %v", err)
}
probeGate := make(chan struct{})
poller := New(repo, 0, logger.Default())
poller.SetPublisher(NewPublisher(eventBus))
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
<-probeGate
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
}
poller.Start(context.Background())
defer poller.Stop()
observer := NewSaveObserver(poller)
after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"})
observer.OnExecutorSaved(context.Background(), nil, after)
select {
case dto := <-published:
t.Fatalf("reset step itself must not publish when there was no prior record: %+v", dto)
default:
}
close(probeGate)
waitForUpsertCount(t, repo, 1)
}
@@ -0,0 +1,222 @@
package reachability
import (
"context"
"errors"
"time"
"go.uber.org/zap"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
)
// Repository is the narrow slice of repository.ExecutorRepository this
// package depends on. The poller and the off-cycle-probe primitive share it,
// so a fake test double satisfies both without importing the full
// task/repository interface.
type Repository interface {
ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error)
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error
ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error
}
// observeResult reports what Observe actually did, for the poller's logging
// and metrics and as the change signal a future consumer (the HTTP/event
// surface) reads off the write path rather than re-deriving. Changed is
// broader than StateChanged: the destination state can hold steady
// (unreachable stays unreachable) while the stored reason moves between two
// different failure causes, and that is still worth telling a client about.
type observeResult struct {
Applied bool
StateChanged bool
Changed bool
Previous models.ExecutorReachabilityState
Current models.ExecutorReachabilityState
// After is the full record as written, for a caller (the HTTP immediate-
// probe route, the change-event publisher) that needs more than the bare
// state enum without a second read-back.
After *models.ExecutorReachability
// Observed is the outcome projected locally from the probe, even when the
// repository rejects the write or a read-back fails. It is never published
// as durable state, but lets a manual probe show the actual result.
Observed *models.ExecutorReachability
}
// store owns the write path's hysteresis. The consecutive-failure counter
// and the destination state for an existing row are derived entirely inside
// UpsertExecutorReachability's own SQL (see the Persistence section of the
// design) — store's job is narrower: pick InitialState/InitialFailures for a
// never-before-seen row (the insert-only path), and determine whether the
// write actually landed and whether the resulting state changed by reading
// the record before and after the call.
type store struct {
repo Repository
log *logger.Logger
}
// Observe records one probe outcome for executor. checkedAt is the probe's
// own completion timestamp, supplied by the caller so a test can control it
// precisely (see the design's last-write-wins contract).
func (s *store) Observe(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) observeResult {
before, err := s.repo.GetExecutorReachability(ctx, executor.ID)
if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
s.log.Warn("executor ssh reachability: read before write failed",
zap.String("executor_id", executor.ID), zap.Error(err))
return observeResult{}
}
checkedAt = checkedAt.UTC().Truncate(time.Microsecond)
obs := buildObservation(executor, outcome, checkedAt)
observed := projectObservation(before, obs)
recordProbeOutcome(outcome.Success, string(obs.Reason))
if err := s.repo.UpsertExecutorReachability(ctx, obs); err != nil {
s.log.Warn("executor ssh reachability: write failed",
zap.String("executor_id", executor.ID), zap.Error(err))
return observeResult{Current: observed.State, Observed: observed}
}
after, err := s.repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
s.log.Warn("executor ssh reachability: read-back after write failed",
zap.String("executor_id", executor.ID), zap.Error(err))
readBackFailedTotal.Add(1)
return observeResult{Current: observed.State, Observed: observed}
}
// The upsert's own WHERE (eligibility, last-write-wins on checked_at) may
// have silently dropped this write. Comparing the stored checked_at back
// against the one this observation carried is how we find out — the SQL
// itself never reports it.
if after.CheckedAt == nil || !after.CheckedAt.Equal(checkedAt) {
writeRefusedTotal.Add(1)
s.log.Debug("executor ssh reachability: write refused",
zap.String("executor_id", executor.ID))
return observeResult{Current: after.State, Observed: observed}
}
previous := models.ExecutorReachabilityStateUnknown
var previousReason models.ExecutorReachabilityReason
if before != nil {
previous = before.State
previousReason = before.Reason
}
stateChanged := previous != after.State
if stateChanged {
s.logTransition(executor, previous, after.State, obs.Reason, after.ConsecutiveFailures)
recordStateTransition(string(after.State))
}
changed := stateChanged || previousReason != after.Reason
return observeResult{
Applied: true, StateChanged: stateChanged, Changed: changed,
Previous: previous, Current: after.State, After: after,
Observed: after,
}
}
func projectObservation(before *models.ExecutorReachability, obs models.ExecutorReachabilityObservation) *models.ExecutorReachability {
record := &models.ExecutorReachability{
ExecutorID: obs.ExecutorID,
State: obs.InitialState,
Reason: obs.Reason,
Message: obs.Message,
ConsecutiveFailures: obs.InitialFailures,
Host: obs.Host,
CheckedAt: checkedAtPtr(obs.CheckedAt),
UpdatedAt: time.Now().UTC(),
}
if before == nil {
if obs.Reason == "" {
record.State = models.ExecutorReachabilityStateReachable
record.ConsecutiveFailures = 0
record.LastSuccessAt = checkedAtPtr(obs.CheckedAt)
}
return record
}
record.State = before.State
record.ConsecutiveFailures = before.ConsecutiveFailures
record.LastSuccessAt = before.LastSuccessAt
if obs.Reason == "" {
record.State = models.ExecutorReachabilityStateReachable
record.ConsecutiveFailures = 0
record.LastSuccessAt = checkedAtPtr(obs.CheckedAt)
return record
}
record.ConsecutiveFailures++
if isStickyReason(obs.Reason) || record.ConsecutiveFailures >= obs.FailureThreshold {
record.State = models.ExecutorReachabilityStateUnreachable
}
return record
}
func checkedAtPtr(value time.Time) *time.Time {
copy := value
return &copy
}
// logTransition logs at Warn going into unreachable and Info coming out of
// it, per the design's Observability section. Unchanged results never reach
// here — Observe only calls this when the state actually changed.
func (s *store) logTransition(executor *models.Executor, previous, current models.ExecutorReachabilityState, reason models.ExecutorReachabilityReason, failures int) {
fields := []zap.Field{
zap.String("executor_id", executor.ID),
zap.String("host", executor.Config["ssh_host"]),
zap.String("from", string(previous)),
zap.String("to", string(current)),
zap.String("reason", string(reason)),
zap.Int("consecutive_failures", failures),
}
if current == models.ExecutorReachabilityStateUnreachable {
s.log.Warn("executor ssh reachability state transition", fields...)
return
}
s.log.Info("executor ssh reachability state transition", fields...)
}
// buildObservation projects a probe outcome into the upsert's input. An
// empty Reason means success. InitialState/InitialFailures matter only on
// the insert path (no prior row); every later write derives both in SQL
// from the stored row instead:
// - success: reachable, zero counter.
// - a sticky reason (config, host_key): unreachable immediately, whatever
// the counter — both are deterministic and would repeat, so waiting out
// the threshold buys no confidence.
// - any other failure: the counter starts at 1, which is below
// failureThreshold, so the state stays unknown — a record that doesn't
// exist yet is never promoted by a single below-threshold failure, the
// same rule the SQL enforces for an existing row.
func buildObservation(executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) models.ExecutorReachabilityObservation {
reason := models.ExecutorReachabilityReason(outcome.Reason)
initialState := models.ExecutorReachabilityStateUnknown
initialFailures := 0
switch {
case outcome.Success:
initialState = models.ExecutorReachabilityStateReachable
case isStickyReason(reason):
initialState = models.ExecutorReachabilityStateUnreachable
initialFailures = 1
default:
initialFailures = 1
if initialFailures >= failureThreshold {
initialState = models.ExecutorReachabilityStateUnreachable
}
}
return models.ExecutorReachabilityObservation{
ExecutorID: executor.ID,
SeenUpdatedAt: executor.UpdatedAt,
InitialState: initialState,
InitialFailures: initialFailures,
Reason: reason,
Message: outcome.Message,
Host: outcome.Host,
CheckedAt: checkedAt,
FailureThreshold: failureThreshold,
}
}
func isStickyReason(reason models.ExecutorReachabilityReason) bool {
return reason == models.ExecutorReachabilityReasonConfig || reason == models.ExecutorReachabilityReasonHostKey
}
@@ -0,0 +1,322 @@
package reachability
import (
"context"
"errors"
"path/filepath"
"testing"
"time"
"github.com/jmoiron/sqlx"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository/sqlite"
)
func newStoreTestRepo(t *testing.T) *sqlite.Repository {
t.Helper()
path := filepath.Join(t.TempDir(), "reachability-store.db")
dbConn, err := db.OpenSQLite(path)
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil)
if err != nil {
_ = sqlxDB.Close()
t.Fatalf("new repository: %v", err)
}
t.Cleanup(func() { _ = sqlxDB.Close() })
return repo
}
func newStoreTestExecutor(t *testing.T, repo *sqlite.Repository) *models.Executor {
t.Helper()
executor := &models.Executor{
ID: "exec-" + t.Name(),
Name: "ssh-host",
Type: models.ExecutorTypeSSH,
Status: models.ExecutorStatusActive,
Config: map[string]string{"ssh_host": "10.0.0.1"},
}
if err := repo.CreateExecutor(context.Background(), executor); err != nil {
t.Fatalf("create executor: %v", err)
}
return executor
}
type microsecondReachabilityRepository struct {
*sqlite.Repository
}
func (r *microsecondReachabilityRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
obs.CheckedAt = obs.CheckedAt.UTC().Truncate(time.Microsecond)
return r.Repository.UpsertExecutorReachability(ctx, obs)
}
type failingGetReachabilityRepository struct {
*sqlite.Repository
err error
failAfter int
calls int
}
func (r *failingGetReachabilityRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
r.calls++
if r.calls >= r.failAfter {
return nil, r.err
}
return r.Repository.GetExecutorReachability(ctx, executorID)
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9, AC-EXECUTORS-SSH-REACHABILITY-001.10
//
// Drives a success/fail/fail/success sequence and asserts the exact probe
// index on which each direction flips. A naive flip-on-first-result
// implementation would report unreachable after the very first failure
// (index 2) instead of the second (index 3), and would report a below-
// threshold failure as a state change even though the state stands still.
func TestStoreObserveHysteresisSequence(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
s := &store{repo: repo, log: logger.Default()}
ctx := context.Background()
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
type step struct {
label string
outcome agentruntime.SSHProbeOutcome
wantState models.ExecutorReachabilityState
wantChanged bool
wantFailures int
}
steps := []step{
{
label: "1: first success establishes reachable",
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
wantState: models.ExecutorReachabilityStateReachable,
wantChanged: true,
wantFailures: 0,
},
{
label: "2: below-threshold failure leaves state reachable",
outcome: agentruntime.SSHProbeOutcome{
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
},
wantState: models.ExecutorReachabilityStateReachable,
wantChanged: false,
wantFailures: 1,
},
{
label: "3: failure reaching failureThreshold flips to unreachable",
outcome: agentruntime.SSHProbeOutcome{
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
},
wantState: models.ExecutorReachabilityStateUnreachable,
wantChanged: true,
wantFailures: 2,
},
{
label: "4: a single success flips back to reachable with a zero counter",
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
wantState: models.ExecutorReachabilityStateReachable,
wantChanged: true,
wantFailures: 0,
},
}
for i, st := range steps {
checkedAt := base.Add(time.Duration(i) * time.Second)
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
result := s.Observe(ctx, executor, st.outcome, checkedAt)
if !result.Applied {
t.Fatalf("%s: write was refused, want applied", st.label)
}
if result.Current != st.wantState {
t.Fatalf("%s: state = %q, want %q", st.label, result.Current, st.wantState)
}
if result.StateChanged != st.wantChanged {
t.Fatalf("%s: changed = %v, want %v", st.label, result.StateChanged, st.wantChanged)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("%s: get: %v", st.label, err)
}
if got.ConsecutiveFailures != st.wantFailures {
t.Fatalf("%s: consecutive_failures = %d, want %d", st.label, got.ConsecutiveFailures, st.wantFailures)
}
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9
//
// A single config or host_key failure writes unreachable on that first
// probe, with the counter still below failureThreshold — sticky reasons
// bypass the threshold entirely.
func TestStoreObserveStickyFailureIsUnreachableOnFirstProbe(t *testing.T) {
tests := []struct {
name string
reason agentruntime.SSHReachabilityReason
}{
{"config", agentruntime.SSHReachabilityReasonConfig},
{"host_key", agentruntime.SSHReachabilityReasonHostKey},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
s := &store{repo: repo, log: logger.Default()}
ctx := context.Background()
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
Host: "10.0.0.1", Reason: tt.reason, Message: "boom",
}, time.Now().UTC())
if !result.Applied || result.Current != models.ExecutorReachabilityStateUnreachable {
t.Fatalf("result = %+v, want applied unreachable on the first probe", result)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.ConsecutiveFailures >= failureThreshold {
t.Fatalf("consecutive_failures = %d, want below failureThreshold (%d) — sticky must not need it", got.ConsecutiveFailures, failureThreshold)
}
})
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// The change-publish trigger is broader than StateChanged: a steady
// unreachable executor can still swap failure reasons (network, then
// timeout) between probes without ever leaving the unreachable state. A
// publisher that only watched StateChanged would miss this and never tell a
// client the diagnostic reason on screen just went stale.
func TestStoreObserveReportsChangedOnReasonChangeWithoutStateChange(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
s := &store{repo: repo, log: logger.Default()}
ctx := context.Background()
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
// Two failures reach failureThreshold and flip to unreachable.
for i := 0; i < 2; i++ {
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
}, base.Add(time.Duration(i)*time.Second))
}
// A third failure with a different transient reason keeps the state
// unreachable (already above threshold) but changes the stored reason.
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonTimeout, Message: "handshake timed out",
}, base.Add(2*time.Second))
if result.StateChanged {
t.Fatalf("StateChanged = true, want false — state was already unreachable")
}
if !result.Changed {
t.Fatalf("Changed = false, want true — reason moved from network to timeout")
}
if result.After == nil || result.After.Reason != models.ExecutorReachabilityReasonTimeout {
t.Fatalf("After = %+v, want a record carrying reason %q", result.After, models.ExecutorReachabilityReasonTimeout)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// A probe outcome that reproduces the same state and reason as the stored
// record must not be reported as a change — this is what keeps a steady host
// silent on the event bus.
func TestStoreObserveReportsUnchangedWhenStateAndReasonBothRepeat(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
s := &store{repo: repo, log: logger.Default()}
ctx := context.Background()
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
first := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC())
if !first.Changed {
t.Fatalf("first observation: Changed = false, want true (unknown -> reachable)")
}
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
second := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC().Add(time.Second))
if second.Changed {
t.Fatalf("second observation: Changed = true, want false — same state (reachable) and reason (success)")
}
if second.After == nil {
t.Fatalf("After = nil, want the current stored record")
}
}
func TestStoreObserveNormalizesDatabaseTimestampPrecision(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
s := &store{repo: &microsecondReachabilityRepository{Repository: repo}, log: logger.Default()}
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 123456789, time.UTC)
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
Success: true, Host: "10.0.0.1",
}, checkedAt)
if !result.Applied {
t.Fatalf("result = %+v, want applied despite microsecond storage precision", result)
}
if result.After == nil || result.After.CheckedAt == nil {
t.Fatalf("After = %+v, want a persisted record", result.After)
}
if want := checkedAt.Truncate(time.Microsecond); !result.After.CheckedAt.Equal(want) {
t.Fatalf("CheckedAt = %v, want %v", result.After.CheckedAt, want)
}
}
func TestStoreObserveStopsOnUnexpectedInitialReadError(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
readErr := errors.New("database unavailable")
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 1}
s := &store{repo: wrapped, log: logger.Default()}
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
Success: true, Host: "10.0.0.1",
}, time.Now().UTC())
if result.Applied || result.Observed != nil {
t.Fatalf("result = %+v, want no observation when the initial read fails", result)
}
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
t.Fatalf("stored record error = %v, want no write", err)
}
}
func TestStoreObserveReturnsUnsavedObservationOnReadBackError(t *testing.T) {
repo := newStoreTestRepo(t)
executor := newStoreTestExecutor(t, repo)
readErr := errors.New("read-back unavailable")
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 2}
s := &store{repo: wrapped, log: logger.Default()}
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
Success: true, Host: "10.0.0.1",
}, time.Now().UTC())
if result.Applied || result.Observed == nil || result.Observed.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("result = %+v, want an unsaved reachable observation", result)
}
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); err != nil {
t.Fatalf("stored record lookup: %v, want the write to have landed", err)
}
}
func mustGetUpdatedAt(t *testing.T, repo *sqlite.Repository, id string) time.Time {
t.Helper()
executor, err := repo.GetExecutor(context.Background(), id)
if err != nil {
t.Fatalf("get executor: %v", err)
}
return executor.UpdatedAt
}
@@ -59,6 +59,9 @@ type Hub struct {
pluginConversationService *plugins.Service
conversationSourceReader ConversationSourceReader
conversationEpoch string
// sessionLaunchWarnings keeps the latest launch warning long enough for a
// client that subscribes after the event was published to receive it.
sessionLaunchWarnings map[string][]byte
// clientDisconnectListener releases connection-bound resources after a
// client is removed from the hub. It runs asynchronously so durable cleanup
@@ -100,6 +103,7 @@ func NewHub(dispatcher *ws.Dispatcher, log *logger.Logger) *Hub {
broadcast: make(chan *ws.Message, 256),
dispatcher: dispatcher,
sessionMode: newSessionModeTracker(),
sessionLaunchWarnings: make(map[string][]byte),
logger: log.WithFields(zap.String("component", "ws_hub")),
conversationEpoch: uuid.NewString(),
}
@@ -185,6 +189,7 @@ func (h *Hub) closeAllClients() {
listener := h.clientDisconnectListener
h.taskSubscribers = make(map[string]map[*Client]bool)
h.sessionSubscribers = make(map[string]map[*Client]bool)
h.sessionLaunchWarnings = make(map[string][]byte)
h.runSubscribers = make(map[string]map[*Client]bool)
h.systemMetricsSubscribers = make(map[*Client]bool)
h.sessionMode.focusByClient = make(map[string]map[*Client]bool)
@@ -684,6 +689,9 @@ func (h *Hub) SubscribeToSession(client *Client, sessionID string) bool {
zap.String("session_id", sessionID))
h.recomputeSessionMode(sessionID)
if !wasSubscribed {
h.replaySessionLaunchWarning(client, sessionID)
}
return !wasSubscribed
}
@@ -1,6 +1,7 @@
package websocket
import (
"encoding/json"
"testing"
"github.com/kandev/kandev/internal/auth/authn"
@@ -47,6 +48,45 @@ func TestBroadcastToSessionReachesFocusedAndSubscribedClients(t *testing.T) {
}
}
func TestSessionLaunchWarningReplaysToLateSubscriber(t *testing.T) {
h := newTestHub(t)
warning, err := ws.NewNotification(ws.ActionSessionLaunchWarning, map[string]any{
"session_id": "sess-1",
"host": "10.0.0.5",
})
if err != nil {
t.Fatalf("notification: %v", err)
}
h.rememberAndBroadcastSessionLaunchWarning("sess-1", warning)
late := newTestClient("late")
registerTestClient(h, late)
if !h.SubscribeToSession(late, "sess-1") {
t.Fatal("late subscriber should create a new membership")
}
select {
case data := <-late.send:
var got ws.Message
if err := json.Unmarshal(data, &got); err != nil {
t.Fatalf("decode replay: %v", err)
}
if got.Action != ws.ActionSessionLaunchWarning {
t.Fatalf("replayed action = %q, want %q", got.Action, ws.ActionSessionLaunchWarning)
}
default:
t.Fatal("late subscriber did not receive the cached launch warning")
}
h.clearSessionLaunchWarning("sess-1")
newClient := newTestClient("new")
registerTestClient(h, newClient)
h.SubscribeToSession(newClient, "sess-1")
if clientReceived(newClient) {
t.Fatal("cleared launch warning was replayed")
}
}
func TestBroadcastToSessionDeduplicatesFocusedSubscription(t *testing.T) {
h := newTestHub(t)
c := newTestClient("c1")
@@ -0,0 +1,50 @@
package websocket
import (
"encoding/json"
ws "github.com/kandev/kandev/pkg/websocket"
)
// rememberAndBroadcastSessionLaunchWarning stores the latest warning before
// the live fan-out. This ordering closes the subscribe-versus-publish race:
// a client either receives the live notification or replays the cached frame
// when it joins later.
func (h *Hub) rememberAndBroadcastSessionLaunchWarning(sessionID string, message *ws.Message) {
if sessionID == "" || message == nil {
return
}
data, err := json.Marshal(message)
if err != nil {
return
}
h.mu.Lock()
if h.sessionLaunchWarnings == nil {
h.sessionLaunchWarnings = make(map[string][]byte)
}
h.sessionLaunchWarnings[sessionID] = append([]byte(nil), data...)
h.mu.Unlock()
h.BroadcastToSession(sessionID, message)
}
func (h *Hub) clearSessionLaunchWarning(sessionID string) {
if sessionID == "" {
return
}
h.mu.Lock()
delete(h.sessionLaunchWarnings, sessionID)
h.mu.Unlock()
}
func (h *Hub) replaySessionLaunchWarning(client *Client, sessionID string) {
if client == nil || sessionID == "" {
return
}
h.mu.RLock()
data := append([]byte(nil), h.sessionLaunchWarnings[sessionID]...)
h.mu.RUnlock()
if len(data) == 0 {
return
}
client.sendNotificationFrame(newOutboundNotification(data, ws.ActionSessionLaunchWarning))
}
@@ -53,6 +53,7 @@ func RegisterSessionStreamNotifications(ctx context.Context, eventBus bus.EventB
b.subscribe(eventBus, events.BuildSessionModelsWildcardSubject(), ws.ActionSessionModelsUpdated)
b.subscribe(eventBus, events.BuildSessionModelFallbackWildcardSubject(), ws.ActionSessionModelFallback)
b.subscribe(eventBus, events.BuildSessionModelSelectionWarningWildcardSubject(), ws.ActionSessionModelSelectionWarning)
b.subscribe(eventBus, events.BuildSessionLaunchWarningWildcardSubject(), ws.ActionSessionLaunchWarning)
b.subscribe(eventBus, events.BuildSessionMCPStatusWildcardSubject(), ws.ActionSessionMCPStatusUpdated)
b.subscribe(eventBus, events.BuildSessionInfoWildcardSubject(), ws.ActionSessionInfoUpdated)
b.subscribe(eventBus, events.BuildSessionTodosWildcardSubject(), ws.ActionSessionTodosUpdated)
@@ -95,6 +96,10 @@ func (b *SessionStreamBroadcaster) subscribe(eventBus bus.EventBus, subject, act
b.logger.Error("failed to build websocket notification", zap.String("action", action), zap.Error(err))
return nil
}
if action == ws.ActionSessionLaunchWarning {
b.hub.rememberAndBroadcastSessionLaunchWarning(sessionID, msg)
return nil
}
b.hub.BroadcastToSession(sessionID, msg)
return nil
})
@@ -236,3 +236,78 @@ func TestSessionStreamBroadcaster_BroadcastsFallbackNotification(t *testing.T) {
}
}
}
// TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject verifies the
// session stream broadcaster subscribes the session.launch.warning wildcard
// subject task 05 publishes on. Without this the event never leaves the bus.
func TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject(t *testing.T) {
log := testLogger()
eventBus := &subscriptionRecordingEventBus{MemoryEventBus: bus.NewMemoryEventBus(log)}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
hub := NewHub(nil, log)
b := RegisterSessionStreamNotifications(ctx, eventBus, hub, log)
_ = b
for _, subject := range eventBus.subjects {
if subject == events.BuildSessionLaunchWarningWildcardSubject() {
return
}
}
t.Fatalf("session stream broadcaster must subscribe the launch warning wildcard subject %q",
events.BuildSessionLaunchWarningWildcardSubject())
}
// TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification verifies a
// published session.launch.warning event is routed to the session's
// subscribers as the session.launch.warning action.
func TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification(t *testing.T) {
hub := newAccessTestHub(t)
eventBus := bus.NewMemoryEventBus(testLogger())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = RegisterSessionStreamNotifications(ctx, eventBus, hub, testLogger())
client := registerAccessClient(t, hub, "client-launch-warning", authn.Identity{UserID: "user-1", Role: authn.RoleMember})
raw, err := json.Marshal(map[string]interface{}{"session_id": "session-warn-1"})
if err != nil {
t.Fatalf("marshal subscribe payload: %v", err)
}
client.handleSessionSubscribe(&ws.Message{
ID: "1", Type: ws.MessageTypeRequest, Action: ws.ActionSessionSubscribe, Payload: raw,
})
sessionID := "session-warn-1"
_ = eventBus.Publish(ctx, events.BuildSessionLaunchWarningSubject(sessionID), bus.NewEvent(
events.SessionLaunchWarning,
"test",
&lifecycle.LaunchWarningEventPayload{
TaskID: "task-1",
SessionID: sessionID,
ExecutorID: "executor-1",
Host: "10.0.0.5",
State: "unreachable",
Reason: "timeout",
Timestamp: time.Now().UTC().Format(time.RFC3339),
},
))
timeout := time.After(2 * time.Second)
for {
select {
case raw := <-client.send:
var msg ws.Message
if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning {
return
}
case raw := <-client.controlSend:
var msg ws.Message
if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning {
return
}
case <-timeout:
t.Fatalf("client did not receive %s", ws.ActionSessionLaunchWarning)
}
}
}
@@ -75,6 +75,7 @@ func RegisterTaskNotifications(ctx context.Context, eventBus bus.EventBus, hub *
b.subscribe(eventBus, events.ExecutorCreated, ws.ActionExecutorCreated)
b.subscribe(eventBus, events.ExecutorUpdated, ws.ActionExecutorUpdated)
b.subscribe(eventBus, events.ExecutorDeleted, ws.ActionExecutorDeleted)
b.subscribe(eventBus, events.ExecutorReachabilityChanged, ws.ActionExecutorReachabilityChanged)
b.subscribe(eventBus, events.ExecutorProfileCreated, ws.ActionExecutorProfileCreated)
b.subscribe(eventBus, events.ExecutorProfileUpdated, ws.ActionExecutorProfileUpdated)
b.subscribe(eventBus, events.ExecutorProfileDeleted, ws.ActionExecutorProfileDeleted)
@@ -290,6 +291,9 @@ func (b *TaskEventBroadcaster) routeBroadcast(
return nil
}
case ws.ActionSessionStateChanged:
if sessionID != "" && extractStringField(data, "new_state") == string(models.TaskSessionStateRunning) {
b.hub.clearSessionLaunchWarning(sessionID)
}
// Broadcast beyond the session subscribers so the sidebar task
// switcher can track state changes for all tasks — but scoped to
// the owning workspace's user when auth is enabled.
@@ -304,6 +308,7 @@ func (b *TaskEventBroadcaster) routeBroadcast(
}
case ws.ActionSessionRemoved:
if sessionID != "" {
b.hub.clearSessionLaunchWarning(sessionID)
if _, hasReceipt := conversationReceiptFromData(data); hasReceipt {
b.hub.BroadcastConversationMutation(data)
}
@@ -0,0 +1,36 @@
package websocket
import (
"context"
"testing"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
ws "github.com/kandev/kandev/pkg/websocket"
)
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
//
// events.ExecutorReachabilityChanged must reach WS clients as
// ws.ActionExecutorReachabilityChanged. Executor events carry no
// workspace_id, so — like ExecutorUpdated before it — this rides the
// broadcaster's trailing default case (global broadcast) rather than a
// dedicated routeBroadcast switch entry.
func TestTaskEventBroadcaster_BroadcastsExecutorReachabilityChanged(t *testing.T) {
log := testLogger()
eventBus := bus.NewMemoryEventBus(log)
hub := NewHub(nil, log)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = RegisterTaskNotifications(ctx, eventBus, hub, log)
payload := map[string]interface{}{"executor_id": "exec-1", "state": "unreachable", "reason": "network"}
_ = eventBus.Publish(ctx, events.ExecutorReachabilityChanged, bus.NewEvent(
events.ExecutorReachabilityChanged, "test", payload,
))
msg := <-hub.broadcast
if msg.Action != ws.ActionExecutorReachabilityChanged {
t.Fatalf("broadcast action = %q, want %q", msg.Action, ws.ActionExecutorReachabilityChanged)
}
}
@@ -226,7 +226,7 @@ func TestTaskEventBroadcaster_NoDuplicateSubscriptions(t *testing.T) {
//
// Update this number when adding or removing event subscriptions in
// RegisterTaskNotifications — it is intentionally exact.
const wantSubscriptions = 75
const wantSubscriptions = 76
if got := len(b.subscriptions); got != wantSubscriptions {
t.Errorf("RegisterTaskNotifications created %d subscriptions, want %d — "+
"did an event get subscribed twice?", got, wantSubscriptions)
@@ -299,13 +299,13 @@ func (s *Sweeper) sweepRuns(ctx context.Context, q queryer, cfg TableSettings, n
}}, satelliteResults{}
}
return SweptTableResult{TableSweepResult: TableSweepResult{
Deleted: result.RunsDeleted,
Backlog: eligible > int64(batchLimit),
}}, satelliteResults{
RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted},
RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted},
RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted},
}
Deleted: result.RunsDeleted,
Backlog: eligible > int64(batchLimit),
}}, satelliteResults{
RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted},
RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted},
RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted},
}
}
// retentionCutoff turns a table's configured window into the instant a
+30 -43
View File
@@ -25,7 +25,10 @@ import (
ws "github.com/kandev/kandev/pkg/websocket"
)
const errorJSONKey = "error"
const (
errorJSONKey = "error"
errExecutorIDRequired = "executor id required"
)
// ExecutorRunningLister is the narrow repository slice we need to surface
// active SSH sessions. We list all running executors and resolve each row's
@@ -54,11 +57,13 @@ type AgentLister interface {
// Handler exposes /api/v1/ssh routes used by the settings UI.
type Handler struct {
repo ExecutorRunningLister
executorFetcher ExecutorFetcher
agents AgentLister
resolver *lifecycle.AgentctlResolver
logger *logger.Logger
repo ExecutorRunningLister
executorFetcher ExecutorFetcher
agents AgentLister
resolver *lifecycle.AgentctlResolver
logger *logger.Logger
reachabilityRepo ReachabilityLister
reachabilityPoller ReachabilityProber
}
// NewHandler builds an SSH HTTP/WS handler.
@@ -68,13 +73,17 @@ func NewHandler(
agents AgentLister,
resolver *lifecycle.AgentctlResolver,
log *logger.Logger,
reachabilityRepo ReachabilityLister,
reachabilityPoller ReachabilityProber,
) *Handler {
return &Handler{
repo: repo,
executorFetcher: executorFetcher,
agents: agents,
resolver: resolver,
logger: log.WithFields(zap.String("component", "ssh-handler")),
repo: repo,
executorFetcher: executorFetcher,
agents: agents,
resolver: resolver,
logger: log.WithFields(zap.String("component", "ssh-handler")),
reachabilityRepo: reachabilityRepo,
reachabilityPoller: reachabilityPoller,
}
}
@@ -87,12 +96,14 @@ func RegisterRoutes(
registry *registry.Registry,
resolver *lifecycle.AgentctlResolver,
log *logger.Logger,
reachabilityRepo ReachabilityLister,
reachabilityPoller ReachabilityProber,
) {
var agentLister AgentLister
if registry != nil {
agentLister = registry
}
h := NewHandler(repo, executorFetcher, agentLister, resolver, log)
h := NewHandler(repo, executorFetcher, agentLister, resolver, log, reachabilityRepo, reachabilityPoller)
h.registerHTTP(router)
h.registerWS(dispatcher)
}
@@ -103,6 +114,9 @@ func (h *Handler) registerHTTP(router *gin.Engine) {
api.GET("/executors/:id/sessions", h.httpListSessions)
api.POST("/executors/:id/probe-agents", h.httpProbeAgents)
api.POST("/executors/:id/probe-shells", h.httpProbeShells)
api.GET("/reachability", h.httpListReachability)
api.GET("/executors/:id/reachability", h.httpGetReachability)
api.POST("/executors/:id/reachability/probe", h.httpProbeReachability)
}
func (h *Handler) registerWS(dispatcher *ws.Dispatcher) {
@@ -179,7 +193,7 @@ func (h *Handler) httpTest(c *gin.Context) {
func (h *Handler) httpListSessions(c *gin.Context) {
id := strings.TrimSpace(c.Param("id"))
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
return
}
rows, err := h.listSessions(c.Request.Context(), id)
@@ -232,7 +246,7 @@ type ProbeAgentsRequest struct {
func (h *Handler) httpProbeAgents(c *gin.Context) {
id := strings.TrimSpace(c.Param("id"))
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
return
}
if h.executorFetcher == nil || h.agents == nil {
@@ -332,7 +346,7 @@ type ProbeShellsResponse struct {
func (h *Handler) httpProbeShells(c *gin.Context) {
id := strings.TrimSpace(c.Param("id"))
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
return
}
if h.executorFetcher == nil {
@@ -421,40 +435,13 @@ func (h *Handler) resolveSSHTarget(ctx context.Context, executorID string) (*lif
if executor == nil || executor.Type != models.ExecutorTypeSSH {
return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID)
}
target, err := sshTargetFromExecutorConfig(executor.Config)
target, err := lifecycle.SSHTargetFromExecutorConfig(executor.Config)
if err != nil {
return nil, http.StatusBadRequest, err
}
return target, http.StatusOK, nil
}
// sshTargetFromExecutorConfig projects an executor.Config into the
// SSHConnConfig the dialer expects. Keeps the handler decoupled from
// lifecycle's internal metadata-vs-config representation.
func sshTargetFromExecutorConfig(cfg map[string]string) (*lifecycle.SSHTarget, error) {
if cfg == nil {
return nil, fmt.Errorf("ssh executor has no config")
}
port := 0
if p := strings.TrimSpace(cfg["ssh_port"]); p != "" {
n, err := strconv.Atoi(p)
if err != nil || n < 1 || n > 65535 {
return nil, fmt.Errorf("invalid ssh_port %q", p)
}
port = n
}
return lifecycle.ResolveSSHTarget(lifecycle.SSHConnConfig{
HostAlias: cfg["ssh_host_alias"],
Host: cfg["ssh_host"],
Port: port,
User: cfg["ssh_user"],
IdentitySource: lifecycle.SSHIdentitySource(cfg["ssh_identity_source"]),
IdentityFile: cfg["ssh_identity_file"],
ProxyJump: cfg["ssh_proxy_jump"],
PinnedFingerprint: cfg["ssh_host_fingerprint"],
})
}
// --- WS handlers ---
func (h *Handler) wsTest(ctx context.Context, msg *ws.Message) (*ws.Message, error) {
@@ -0,0 +1,228 @@
package ssh
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
"github.com/kandev/kandev/internal/task/models"
)
// reachabilityConfigValidator is the production seam over the runtime's SSH
// reachability prober, used only to check whether an executor's config
// resolves to a dialable target — no probe is actually run here.
var reachabilityConfigValidator = agentruntime.NewSSHReachabilityProber()
// ReachabilityLister is the narrow repository slice the reachability GET
// routes need: every stored record in one query (the list route) or a
// single record (the single-executor route). Executor lookup itself reuses
// ExecutorFetcher, already wired on Handler for the existing SSH routes.
type ReachabilityLister interface {
ListExecutors(ctx context.Context) ([]*models.Executor, error)
ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error)
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
}
// ReachabilityProber is the narrow slice of *reachability.Poller the
// immediate-probe route needs. Declared here (not imported as the concrete
// type) so a handler test can substitute a fake without spinning up a real
// poller when it only needs to exercise coalescing or 404/400/409
// short-circuits.
type ReachabilityProber interface {
ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool)
EffectiveIntervalSeconds() int
}
// --- HTTP handlers ---
func (h *Handler) httpListReachability(c *gin.Context) {
dtos, status, err := h.listReachability(c.Request.Context())
if err != nil {
h.writeReachabilityError(c, status, err)
return
}
c.JSON(status, dtos)
}
func (h *Handler) httpGetReachability(c *gin.Context) {
id := strings.TrimSpace(c.Param("id"))
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
return
}
dto, status, err := h.getReachability(c.Request.Context(), id)
if err != nil {
h.writeReachabilityError(c, status, err)
return
}
c.JSON(status, dto)
}
func (h *Handler) httpProbeReachability(c *gin.Context) {
id := strings.TrimSpace(c.Param("id"))
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
return
}
dto, status, err := h.probeReachability(c.Request.Context(), id)
if err != nil {
h.writeReachabilityError(c, status, err)
return
}
c.JSON(status, dto)
}
func (h *Handler) writeReachabilityError(c *gin.Context, status int, err error) {
message := "reachability request failed"
if status >= http.StatusBadRequest && status < http.StatusInternalServerError {
message = err.Error()
} else if h.logger != nil {
h.logger.Error("reachability request failed", zap.Int("status", status), zap.Error(err))
}
c.JSON(status, gin.H{errorJSONKey: message})
}
// --- Testable cores ---
// listReachability returns one DTO per SSH executor that is not
// soft-deleted, ordered by executor_id ascending — the same total order the
// poller's own pass uses. An executor whose status is not active is still
// included, carrying its retained record.
func (h *Handler) listReachability(ctx context.Context) ([]reachabilitypkg.RecordDTO, int, error) {
if h.reachabilityRepo == nil {
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired")
}
executors, err := h.reachabilityRepo.ListExecutors(ctx)
if err != nil {
return nil, http.StatusInternalServerError, fmt.Errorf("list executors: %w", err)
}
records, err := h.reachabilityRepo.ListExecutorReachability(ctx)
if err != nil {
return nil, http.StatusInternalServerError, fmt.Errorf("list executor reachability: %w", err)
}
byExecutor := make(map[string]*models.ExecutorReachability, len(records))
for _, record := range records {
byExecutor[record.ExecutorID] = record
}
sshExecutors := filterSSHExecutors(executors)
sort.Slice(sshExecutors, func(i, j int) bool { return sshExecutors[i].ID < sshExecutors[j].ID })
interval := h.reachabilityIntervalSeconds()
dtos := make([]reachabilitypkg.RecordDTO, 0, len(sshExecutors))
for _, executor := range sshExecutors {
dtos = append(dtos, buildReachabilityDTOFromRecord(executor, byExecutor[executor.ID], interval))
}
return dtos, http.StatusOK, nil
}
// getReachability returns the single record for executorID. Never 404s for
// a missing record — only for a missing or soft-deleted executor — per
// AC-EXECUTORS-SSH-REACHABILITY-002.5.
func (h *Handler) getReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) {
executor, status, err := h.loadSSHExecutor(ctx, executorID)
if err != nil {
return nil, status, err
}
if h.reachabilityRepo == nil {
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired")
}
record, err := h.reachabilityRepo.GetExecutorReachability(ctx, executorID)
if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
return nil, http.StatusInternalServerError, fmt.Errorf("get executor reachability: %w", err)
}
if errors.Is(err, models.ErrExecutorReachabilityNotFound) {
record = nil
}
dto := buildReachabilityDTOFromRecord(executor, record, h.reachabilityIntervalSeconds())
return &dto, http.StatusOK, nil
}
// probeReachability runs (or joins an in-flight) synchronous probe for
// executorID and returns its persisted record. Coalesces concurrent callers
// through reachabilityPoller.ProbeAndWait's own singleflight group.
func (h *Handler) probeReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) {
executor, status, err := h.loadSSHExecutor(ctx, executorID)
if err != nil {
return nil, status, err
}
if executor.Status != models.ExecutorStatusActive {
return nil, http.StatusConflict, fmt.Errorf("executor %q is not active", executorID)
}
if h.reachabilityPoller == nil {
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability probing not wired")
}
result, ok := h.reachabilityPoller.ProbeAndWait(executor)
if !ok {
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability poller is not accepting probes")
}
dto := reachabilitypkg.BuildRecordDTO(executorID, result.Record, h.reachabilityPoller.EffectiveIntervalSeconds(), result.Persisted)
return &dto, http.StatusOK, nil
}
// loadSSHExecutor enforces the 404/400 split the reachability routes need:
// 404 for a nonexistent or soft-deleted id, 400 only when the executor
// exists but isn't type ssh. Deliberately does not attempt config
// resolution (unlike resolveSSHTarget) — an unresolvable ssh config is a 200
// with reason config, decided by the caller, never a 400 here.
func (h *Handler) loadSSHExecutor(ctx context.Context, executorID string) (*models.Executor, int, error) {
executor, err := h.executorFetcher.GetExecutor(ctx, executorID)
if err != nil {
if errors.Is(err, models.ErrExecutorNotFound) {
return nil, http.StatusNotFound, fmt.Errorf("executor %q not found: %w", executorID, err)
}
return nil, http.StatusInternalServerError, fmt.Errorf("look up executor %q: %w", executorID, err)
}
if executor.Type != models.ExecutorTypeSSH {
return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID)
}
return executor, http.StatusOK, nil
}
func (h *Handler) reachabilityIntervalSeconds() int {
if h.reachabilityPoller == nil {
return 0
}
return h.reachabilityPoller.EffectiveIntervalSeconds()
}
func filterSSHExecutors(executors []*models.Executor) []*models.Executor {
out := make([]*models.Executor, 0, len(executors))
for _, executor := range executors {
if executor != nil && executor.Type == models.ExecutorTypeSSH {
out = append(out, executor)
}
}
return out
}
// buildReachabilityDTOFromRecord projects executor/record into the wire
// shape without any I/O. record nil means "no record yet" — that synthesizes
// the unknown placeholder unless the executor's own config can't resolve a
// target, in which case AC-EXECUTORS-SSH-REACHABILITY-002.5's sibling rule
// applies: 200 with reason config, the same outcome a real probe would reach
// on its very first attempt, rather than a misleading "not probed yet".
func buildReachabilityDTOFromRecord(executor *models.Executor, record *models.ExecutorReachability, intervalSeconds int) reachabilitypkg.RecordDTO {
if record != nil {
return reachabilitypkg.BuildRecordDTO(executor.ID, record, intervalSeconds, true)
}
if err := reachabilityConfigValidator.ValidateConfig(executor.Config); err != nil {
synthetic := &models.ExecutorReachability{
ExecutorID: executor.ID,
State: models.ExecutorReachabilityStateUnreachable,
Reason: models.ExecutorReachabilityReasonConfig,
Message: agentruntime.SanitizeSSHReachabilityMessage(err),
Host: executor.Config["ssh_host"],
}
return reachabilitypkg.BuildRecordDTO(executor.ID, synthetic, intervalSeconds, false)
}
return reachabilitypkg.BuildRecordDTO(executor.ID, nil, intervalSeconds, false)
}
@@ -0,0 +1,362 @@
package ssh
import (
"context"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
"golang.org/x/sync/singleflight"
"github.com/kandev/kandev/internal/common/logger"
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
"github.com/kandev/kandev/internal/task/models"
)
func testHandlerLogger() *logger.Logger {
return logger.Default()
}
type fakeReachabilityExecutorFetcher struct {
executors map[string]*models.Executor
}
func (f *fakeReachabilityExecutorFetcher) GetExecutor(_ context.Context, id string) (*models.Executor, error) {
executor, ok := f.executors[id]
if !ok {
return nil, models.ErrExecutorNotFound
}
return executor, nil
}
type fakeReachabilityLister struct {
executors []*models.Executor
records []*models.ExecutorReachability
byID map[string]*models.ExecutorReachability
}
func (f *fakeReachabilityLister) ListExecutors(_ context.Context) ([]*models.Executor, error) {
return f.executors, nil
}
func (f *fakeReachabilityLister) ListExecutorReachability(_ context.Context) ([]*models.ExecutorReachability, error) {
return f.records, nil
}
func (f *fakeReachabilityLister) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
if record, ok := f.byID[executorID]; ok {
return record, nil
}
return nil, models.ErrExecutorReachabilityNotFound
}
func sshExecutor(id string, status models.ExecutorStatus, config map[string]string) *models.Executor {
return &models.Executor{ID: id, Type: models.ExecutorTypeSSH, Status: status, Config: config}
}
func resolvableSSHConfig() map[string]string {
return map[string]string{
"ssh_host": "10.0.0.1",
"ssh_user": "deploy",
"ssh_host_fingerprint": "SHA256:test",
}
}
// fakeReachabilityProber models the exact guarantee *reachability.Poller
// makes for concurrent callers of the same executor id: it coalesces
// through the same golang.org/x/sync/singleflight primitive the production
// Poller uses, so this test proves the handler's own plumbing (always
// routing through the one shared prober instance) rather than re-deriving
// singleflight's own coalescing guarantee, which is already covered by
// TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor.
type fakeReachabilityProber struct {
group singleflight.Group
dialCount int32
callsEntered int32
gate chan struct{}
result reachabilitypkg.ProbeResult
interval int
}
func (f *fakeReachabilityProber) ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool) {
atomic.AddInt32(&f.callsEntered, 1)
v, _, _ := f.group.Do(executor.ID, func() (any, error) {
atomic.AddInt32(&f.dialCount, 1)
<-f.gate
return f.result, nil
})
return v.(reachabilitypkg.ProbeResult), true
}
func (f *fakeReachabilityProber) EffectiveIntervalSeconds() int { return f.interval }
func waitForCallsEntered(t *testing.T, prober *fakeReachabilityProber, want int32) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if atomic.LoadInt32(&prober.callsEntered) >= want {
return
}
time.Sleep(time.Millisecond)
}
t.Fatalf("callsEntered did not reach %d within 1s (got %d)", want, atomic.LoadInt32(&prober.callsEntered))
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002 (coalescing)
//
// Two overlapping probe requests for the same executor must share one dial
// and receive one identical record. Written first as a RED test against a
// handler that has not yet been implemented, per the task-04 work order.
func TestProbeReachability_CoalescesConcurrentProbes(t *testing.T) {
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
updatedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
prober := &fakeReachabilityProber{
gate: make(chan struct{}),
result: reachabilitypkg.ProbeResult{
Record: &models.ExecutorReachability{
ExecutorID: "exec-1",
State: models.ExecutorReachabilityStateReachable,
UpdatedAt: updatedAt,
},
Persisted: true,
},
interval: 30,
}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober)
const callers = 2
dtos := make([]*reachabilitypkg.RecordDTO, callers)
statuses := make([]int, callers)
errs := make([]error, callers)
var wg sync.WaitGroup
for i := 0; i < callers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
dto, status, err := h.probeReachability(context.Background(), "exec-1")
dtos[i] = dto
statuses[i] = status
errs[i] = err
}(i)
}
waitForCallsEntered(t, prober, callers)
close(prober.gate)
wg.Wait()
if got := atomic.LoadInt32(&prober.dialCount); got != 1 {
t.Fatalf("dialCount = %d, want 1 (concurrent probes must coalesce into one dial)", got)
}
for i := 0; i < callers; i++ {
if errs[i] != nil {
t.Fatalf("caller %d: unexpected error: %v", i, errs[i])
}
if statuses[i] != 200 {
t.Fatalf("caller %d: status = %d, want 200", i, statuses[i])
}
}
if !reflect.DeepEqual(dtos[0], dtos[1]) {
t.Fatalf("responses differ: %+v vs %+v", dtos[0], dtos[1])
}
if dtos[0].State != string(models.ExecutorReachabilityStateReachable) {
t.Fatalf("state = %q, want %q", dtos[0].State, models.ExecutorReachabilityStateReachable)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
func TestGetReachability_NoRecordAndResolvableConfigReturnsUnknown(t *testing.T) {
executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig())
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
lister := &fakeReachabilityLister{}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
dto, status, err := h.getReachability(context.Background(), "exec-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != 200 {
t.Fatalf("status = %d, want 200", status)
}
if dto.State != string(models.ExecutorReachabilityStateUnknown) {
t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateUnknown)
}
if dto.Reason != "" {
t.Fatalf("reason = %q, want empty", dto.Reason)
}
if dto.Persisted {
t.Fatal("persisted = true for a synthesized never-probed record")
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
//
// An ssh executor whose config cannot resolve a target (no ssh_host here)
// must answer 200 with reason "config", never 400 — resolveSSHTarget's own
// 400-on-unresolvable-config mapping does not apply to the reachability
// routes.
func TestGetReachability_NoRecordAndUnresolvableConfigReturnsReasonConfig(t *testing.T) {
executor := sshExecutor("exec-1", models.ExecutorStatusActive, map[string]string{})
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
lister := &fakeReachabilityLister{}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
dto, status, err := h.getReachability(context.Background(), "exec-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != 200 {
t.Fatalf("status = %d, want 200", status)
}
if dto.Reason != string(models.ExecutorReachabilityReasonConfig) {
t.Fatalf("reason = %q, want %q", dto.Reason, models.ExecutorReachabilityReasonConfig)
}
if dto.Persisted {
t.Fatal("persisted = true for a synthesized record that was never written")
}
}
func TestGetReachability_ReturnsStoredRecordVerbatim(t *testing.T) {
executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig())
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
stored := &models.ExecutorReachability{
ExecutorID: "exec-1",
State: models.ExecutorReachabilityStateReachable,
Host: "10.0.0.1",
}
lister := &fakeReachabilityLister{byID: map[string]*models.ExecutorReachability{"exec-1": stored}}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
dto, status, err := h.getReachability(context.Background(), "exec-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != 200 {
t.Fatalf("status = %d, want 200", status)
}
if !dto.Persisted {
t.Fatal("persisted = false for an actually-stored record")
}
if dto.State != string(models.ExecutorReachabilityStateReachable) {
t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateReachable)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
func TestGetReachability_NonexistentExecutorReturns404(t *testing.T) {
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil)
_, status, err := h.getReachability(context.Background(), "missing")
if err == nil {
t.Fatal("expected an error for a nonexistent executor")
}
if status != 404 {
t.Fatalf("status = %d, want 404", status)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
//
// A wrong-type executor id is a 400, distinct from the config-unresolvable
// case (which is a 200 with reason config).
func TestGetReachability_WrongTypeExecutorReturns400(t *testing.T) {
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil)
_, status, err := h.getReachability(context.Background(), "exec-1")
if err == nil {
t.Fatal("expected an error for a non-ssh executor")
}
if status != 400 {
t.Fatalf("status = %d, want 400", status)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.4
//
// The list route orders by executor_id ascending, includes a deactivated
// ssh executor's retained record, and omits non-ssh executors entirely.
func TestListReachability_OrdersFiltersAndIncludesDeactivated(t *testing.T) {
sshA := sshExecutor("ssh-b", models.ExecutorStatusActive, resolvableSSHConfig())
sshB := sshExecutor("ssh-a", models.ExecutorStatusDisabled, resolvableSSHConfig())
nonSSH := &models.Executor{ID: "local-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
fetcher := &fakeReachabilityExecutorFetcher{}
lister := &fakeReachabilityLister{
executors: []*models.Executor{sshA, sshB, nonSSH},
records: []*models.ExecutorReachability{
{ExecutorID: "ssh-a", State: models.ExecutorReachabilityStateUnreachable, Reason: models.ExecutorReachabilityReasonTimeout},
},
}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
dtos, status, err := h.listReachability(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status != 200 {
t.Fatalf("status = %d, want 200", status)
}
if len(dtos) != 2 {
t.Fatalf("len(dtos) = %d, want 2 (non-ssh executor must be omitted)", len(dtos))
}
if dtos[0].ExecutorID != "ssh-a" || dtos[1].ExecutorID != "ssh-b" {
t.Fatalf("order = [%s, %s], want [ssh-a, ssh-b] (executor_id ascending)", dtos[0].ExecutorID, dtos[1].ExecutorID)
}
if dtos[0].State != string(models.ExecutorReachabilityStateUnreachable) {
t.Fatalf("deactivated executor's record state = %q, want %q (retained record)", dtos[0].State, models.ExecutorReachabilityStateUnreachable)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.6
//
// A POST against a deactivated ssh executor answers 409 without probing.
func TestProbeReachability_InactiveExecutorReturns409WithoutProbing(t *testing.T) {
executor := sshExecutor("exec-1", models.ExecutorStatusDisabled, resolvableSSHConfig())
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
prober := &fakeReachabilityProber{gate: make(chan struct{})}
close(prober.gate) // would return immediately if ever called
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober)
_, status, err := h.probeReachability(context.Background(), "exec-1")
if err == nil {
t.Fatal("expected an error for a probe against an inactive executor")
}
if status != 409 {
t.Fatalf("status = %d, want 409", status)
}
if got := atomic.LoadInt32(&prober.callsEntered); got != 0 {
t.Fatalf("callsEntered = %d, want 0 (must not probe an inactive executor)", got)
}
}
func TestProbeReachability_NonexistentExecutorReturns404(t *testing.T) {
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{})
_, status, err := h.probeReachability(context.Background(), "missing")
if err == nil {
t.Fatal("expected an error for a nonexistent executor")
}
if status != 404 {
t.Fatalf("status = %d, want 404", status)
}
}
func TestProbeReachability_WrongTypeExecutorReturns400(t *testing.T) {
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{})
_, status, err := h.probeReachability(context.Background(), "exec-1")
if err == nil {
t.Fatal("expected an error for a non-ssh executor")
}
if status != 400 {
t.Fatalf("status = %d, want 400", status)
}
}
@@ -607,6 +607,24 @@ func (m *mockRepository) UpdateExecutorRunningStatus(ctx context.Context, sessio
func (m *mockRepository) RepairExecutorRunningDead(ctx context.Context, sessionID string) error {
return nil
}
func (m *mockRepository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) {
return nil, nil
}
func (m *mockRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
return nil, models.ErrExecutorReachabilityNotFound
}
func (m *mockRepository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) {
return nil, nil
}
func (m *mockRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
return nil
}
func (m *mockRepository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error {
return nil
}
func (m *mockRepository) DeleteExecutorReachability(ctx context.Context, executorID string) error {
return nil
}
func (m *mockRepository) CreateEnvironment(ctx context.Context, environment *models.Environment) error {
return nil
}
@@ -0,0 +1,69 @@
package models
import (
"errors"
"time"
)
// ErrExecutorReachabilityNotFound is returned when no reachability record
// exists for the requested executor.
var ErrExecutorReachabilityNotFound = errors.New("executor reachability record not found")
// ExecutorReachabilityState is the coarse status shown to operators.
type ExecutorReachabilityState string
const (
ExecutorReachabilityStateUnknown ExecutorReachabilityState = "unknown"
ExecutorReachabilityStateReachable ExecutorReachabilityState = "reachable"
ExecutorReachabilityStateUnreachable ExecutorReachabilityState = "unreachable"
)
// ExecutorReachabilityReason mirrors lifecycle.SSHReachabilityReason's closed
// set as plain strings — this package sits below the agent runtime tier and
// must not import it. An empty reason means the observation was a success.
type ExecutorReachabilityReason string
const (
ExecutorReachabilityReasonConfig ExecutorReachabilityReason = "config"
ExecutorReachabilityReasonTimeout ExecutorReachabilityReason = "timeout"
ExecutorReachabilityReasonHostKey ExecutorReachabilityReason = "host_key"
ExecutorReachabilityReasonAuth ExecutorReachabilityReason = "auth"
ExecutorReachabilityReasonNetwork ExecutorReachabilityReason = "network"
ExecutorReachabilityReasonUnknown ExecutorReachabilityReason = "unknown"
)
// ExecutorReachability is the single stored row for one SSH executor's
// observed reachability. It lives in its own table, never on Executor
// itself — Executor.Config is user-authored input and Executor.Status is a
// user-controlled switch, and neither should share a row with observed
// state.
type ExecutorReachability struct {
ExecutorID string `json:"executor_id"`
State ExecutorReachabilityState `json:"state"`
Reason ExecutorReachabilityReason `json:"reason"`
Message string `json:"message"`
ConsecutiveFailures int `json:"consecutive_failures"`
Host string `json:"host"`
CheckedAt *time.Time `json:"checked_at,omitempty"`
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
// ExecutorReachabilityObservation is what a single probe (or a launch's own
// dial attempt) reports to UpsertExecutorReachability. The derived counter
// and state for an existing row live entirely in the upsert's own SQL — this
// struct carries only the observation and the parameters the SQL needs to
// derive from it. InitialState and InitialFailures are used only when no row
// exists yet for the executor; every later write derives both from the
// stored row instead.
type ExecutorReachabilityObservation struct {
ExecutorID string
SeenUpdatedAt time.Time
InitialState ExecutorReachabilityState
InitialFailures int
Reason ExecutorReachabilityReason
Message string
Host string
CheckedAt time.Time
FailureThreshold int
}
@@ -706,6 +706,33 @@ type ExecutorRepository interface {
// the resume-safety invariant instead of deleting a resumable row.
// Returns models.ErrExecutorRunningNotFound if no row exists for the session.
RepairExecutorRunningDead(ctx context.Context, sessionID string) error
// ListSSHExecutorsForReachability returns every eligible SSH executor
// (type=ssh, not soft-deleted, status=active) ordered ascending by id —
// the poller's per-pass work list.
ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error)
// GetExecutorReachability returns the stored record for one executor.
// Returns models.ErrExecutorReachabilityNotFound if none exists.
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
// ListExecutorReachability returns every stored reachability record.
ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error)
// UpsertExecutorReachability records a single probe (or launch dial)
// observation. The consecutive-failure counter and derived state are
// computed by the statement itself from the row's own prior values, and
// a write is discarded when obs.CheckedAt is not strictly later than the
// stored checked_at — see the system design's Persistence section.
UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error
// ResetExecutorReachability invalidates the stored record after a
// connection-configuration save when seenUpdatedAt still matches the
// executor row. The version guard prevents a delayed save callback from
// resetting a newer configuration. State becomes unknown, the counter and
// reason/message clear, host is set to the newly saved value, and both
// timestamps become NULL. A no-op (zero rows affected) when the executor
// is not an active SSH executor or the version is stale.
ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error
// DeleteExecutorReachability removes the stored record. DeleteExecutor
// calls this in the same transaction as the soft delete.
DeleteExecutorReachability(ctx context.Context, executorID string) error
}
// EnvironmentRepository handles environment CRUD.
@@ -423,6 +423,24 @@ func (r *Repository) runMigrations(ctx context.Context) error {
// allocated.
_ = r.migrate.Apply("workflow_step_entries.marker_positions", `ALTER TABLE workflow_step_entries ADD COLUMN marker_positions TEXT NOT NULL DEFAULT ''`)
// One row per SSH executor holding observed reachability — deliberately
// not columns on executors, which holds user-authored config and a
// user-controlled status switch. No foreign key: deletion is explicit
// (DeleteExecutor deletes the row in the same transaction as the soft
// delete), not a cascade.
_ = r.migrate.Apply("executor_reachability.table", `
CREATE TABLE IF NOT EXISTS executor_reachability (
executor_id TEXT PRIMARY KEY,
state TEXT NOT NULL DEFAULT 'unknown',
reason TEXT NOT NULL DEFAULT '',
message TEXT NOT NULL DEFAULT '',
consecutive_failures INTEGER NOT NULL DEFAULT 0,
host TEXT NOT NULL DEFAULT '',
checked_at TIMESTAMP,
last_success_at TIMESTAMP,
updated_at TIMESTAMP NOT NULL
)`)
// Checked last so a failure on any required migration above --
// including this file's own marker_positions column -- fails startup
// instead of leaving a schema that allocateStepEntryIfPending can't write to.
@@ -94,7 +94,13 @@ func (r *Repository) UpdateExecutor(ctx context.Context, executor *models.Execut
func (r *Repository) DeleteExecutor(ctx context.Context, id string) error {
now := time.Now().UTC()
result, err := r.db.ExecContext(ctx, r.db.Rebind(`
tx, err := r.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("begin executor delete: %w", err)
}
defer func() { _ = tx.Rollback() }()
result, err := tx.ExecContext(ctx, tx.Rebind(`
UPDATE executors SET deleted_at = ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL
`), now, now, id)
if err != nil {
@@ -104,9 +110,42 @@ func (r *Repository) DeleteExecutor(ctx context.Context, id string) error {
if rows == 0 {
return fmt.Errorf("executor not found: %s", id)
}
if _, err := tx.ExecContext(ctx, tx.Rebind(`
DELETE FROM executor_reachability WHERE executor_id = ?
`), id); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit executor delete: %w", err)
}
return nil
}
// scanExecutorRow scans one row of the standard executors column list
// (id, name, type, status, is_system, resumable, config, created_at,
// updated_at, deleted_at) into a models.Executor. Shared by every query that
// selects that exact column set.
func scanExecutorRow(scanner interface{ Scan(...any) error }) (*models.Executor, error) {
executor := &models.Executor{}
var configJSON string
var isSystem int
var resumable int
if err := scanner.Scan(
&executor.ID, &executor.Name, &executor.Type, &executor.Status,
&isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt,
); err != nil {
return nil, err
}
executor.IsSystem = isSystem == 1
executor.Resumable = resumable == 1
if configJSON != "" && configJSON != "{}" {
if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil {
return nil, fmt.Errorf("failed to deserialize executor config: %w", err)
}
}
return executor, nil
}
func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, error) {
rows, err := r.ro.QueryContext(ctx, `
SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at
@@ -119,23 +158,10 @@ func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, err
var result []*models.Executor
for rows.Next() {
executor := &models.Executor{}
var configJSON string
var isSystem int
var resumable int
if err := rows.Scan(
&executor.ID, &executor.Name, &executor.Type, &executor.Status,
&isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt,
); err != nil {
executor, err := scanExecutorRow(rows)
if err != nil {
return nil, err
}
executor.IsSystem = isSystem == 1
executor.Resumable = resumable == 1
if configJSON != "" && configJSON != "{}" {
if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil {
return nil, fmt.Errorf("failed to deserialize executor config: %w", err)
}
}
result = append(result, executor)
}
return result, rows.Err()
@@ -0,0 +1,183 @@
package sqlite
import (
"context"
"database/sql"
"time"
"github.com/kandev/kandev/internal/task/models"
)
const executorReachabilityColumns = `executor_id, state, reason, message, consecutive_failures, host, checked_at, last_success_at, updated_at`
func scanExecutorReachability(scanner interface{ Scan(...any) error }) (*models.ExecutorReachability, error) {
r := &models.ExecutorReachability{}
var state, reason string
var checkedAt, lastSuccessAt sql.NullTime
if err := scanner.Scan(
&r.ExecutorID, &state, &reason, &r.Message, &r.ConsecutiveFailures, &r.Host,
&checkedAt, &lastSuccessAt, &r.UpdatedAt,
); err != nil {
return nil, err
}
r.State = models.ExecutorReachabilityState(state)
r.Reason = models.ExecutorReachabilityReason(reason)
if checkedAt.Valid {
v := checkedAt.Time
r.CheckedAt = &v
}
if lastSuccessAt.Valid {
v := lastSuccessAt.Time
r.LastSuccessAt = &v
}
return r, nil
}
// ListSSHExecutorsForReachability returns every eligible SSH executor
// (type=ssh, not soft-deleted, status=active) ordered ascending by id, the
// primary key, never reused, so the order is total and needs no tiebreak.
func (r *Repository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) {
rows, err := r.ro.QueryContext(ctx, `
SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at
FROM executors
WHERE type = 'ssh' AND deleted_at IS NULL AND status = 'active'
ORDER BY id ASC
`)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var result []*models.Executor
for rows.Next() {
executor, err := scanExecutorRow(rows)
if err != nil {
return nil, err
}
result = append(result, executor)
}
return result, rows.Err()
}
// GetExecutorReachability returns the stored record for one executor.
// Returns models.ErrExecutorReachabilityNotFound if none exists.
func (r *Repository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
row := r.ro.QueryRowContext(ctx, r.ro.Rebind(`
SELECT `+executorReachabilityColumns+` FROM executor_reachability WHERE executor_id = ?
`), executorID)
rec, err := scanExecutorReachability(row)
if err != nil {
if err == sql.ErrNoRows {
return nil, models.ErrExecutorReachabilityNotFound
}
return nil, err
}
return rec, nil
}
// ListExecutorReachability returns every stored reachability record.
func (r *Repository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) {
rows, err := r.ro.QueryContext(ctx, `
SELECT `+executorReachabilityColumns+` FROM executor_reachability ORDER BY executor_id ASC
`)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var result []*models.ExecutorReachability
for rows.Next() {
rec, err := scanExecutorReachability(rows)
if err != nil {
return nil, err
}
result = append(result, rec)
}
return result, rows.Err()
}
// UpsertExecutorReachability records a single probe (or launch dial)
// observation. The write is a single statement: the consecutive-failure
// counter and the derived state for an existing row are computed by the
// statement itself from the row's own prior values, so no read-modify-write
// in Go can lose a streak to an interleaving. A write whose CheckedAt is not
// strictly later than the stored checked_at is discarded (last-write-wins),
// and the row is only touched at all when the executor is still the active
// SSH executor obs.SeenUpdatedAt was captured against — see the system
// design's Persistence section for the full contract.
func (r *Repository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
now := time.Now().UTC()
checkedAt := obs.CheckedAt.UTC()
var lastSuccessAt any
if obs.Reason == "" {
lastSuccessAt = checkedAt
}
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
INSERT INTO executor_reachability (
executor_id, state, reason, message, consecutive_failures, host,
checked_at, last_success_at, updated_at
)
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?
WHERE EXISTS (SELECT 1 FROM executors e
WHERE e.id = ? AND e.deleted_at IS NULL
AND e.status = 'active' AND e.updated_at = ?)
ON CONFLICT (executor_id) DO UPDATE SET
consecutive_failures = CASE WHEN excluded.reason = '' THEN 0
ELSE executor_reachability.consecutive_failures + 1 END,
state = CASE WHEN excluded.reason = '' THEN 'reachable'
WHEN excluded.reason IN ('config','host_key') THEN 'unreachable'
WHEN executor_reachability.consecutive_failures + 1 >= ?
THEN 'unreachable'
ELSE executor_reachability.state END,
reason = excluded.reason, message = excluded.message, host = excluded.host,
checked_at = excluded.checked_at,
last_success_at = CASE WHEN excluded.reason = '' THEN excluded.checked_at
ELSE executor_reachability.last_success_at END,
updated_at = excluded.updated_at
WHERE executor_reachability.checked_at IS NULL
OR excluded.checked_at > executor_reachability.checked_at
`),
obs.ExecutorID, string(obs.InitialState), string(obs.Reason), obs.Message, obs.InitialFailures, obs.Host,
checkedAt, lastSuccessAt, now,
obs.ExecutorID, obs.SeenUpdatedAt.UTC(),
obs.FailureThreshold,
)
return err
}
// ResetExecutorReachability invalidates the stored record after a
// connection-configuration save. It is its own statement, not a branch of
// the upsert above: the upsert's state CASE has no unknown branch and its
// WHERE admits only a strictly later checked_at, which a reset does not
// carry. The executor updated_at predicate couples the reset to the save
// version, so a delayed callback cannot reset a newer configuration. A
// no-op when the executor isn't an active SSH executor or the version is
// stale: AC-EXECUTORS-SSH-REACHABILITY-001.17 keeps a disabled executor's
// record untouched, so eligibility wins over the reset trigger.
func (r *Repository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error {
now := time.Now().UTC()
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
INSERT INTO executor_reachability (
executor_id, state, reason, message, consecutive_failures,
checked_at, host, last_success_at, updated_at
)
SELECT ?, 'unknown', '', '', 0, NULL, ?, NULL, ?
WHERE EXISTS (SELECT 1 FROM executors e
WHERE e.id = ? AND e.type = 'ssh'
AND e.deleted_at IS NULL AND e.status = 'active'
AND e.updated_at = ?)
ON CONFLICT (executor_id) DO UPDATE SET
state = 'unknown', reason = '', message = '', consecutive_failures = 0,
host = excluded.host, checked_at = NULL, last_success_at = NULL,
updated_at = excluded.updated_at
`), executorID, host, now, executorID, seenUpdatedAt.UTC())
return err
}
// DeleteExecutorReachability removes the stored record for one executor.
func (r *Repository) DeleteExecutorReachability(ctx context.Context, executorID string) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
DELETE FROM executor_reachability WHERE executor_id = ?
`), executorID)
return err
}
@@ -0,0 +1,59 @@
package sqlite
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/testutil"
)
// TestPostgresExecutorReachabilityCheckedAtOrdering guards the SQLite-only
// risk the design calls out: TIMESTAMP takes NUMERIC affinity on SQLite and
// compares the driver's text encoding lexically, so a whole-second value can
// sort after a later sub-second one unless writes are normalized to UTC and
// bind time.Time rather than a pre-formatted string. Postgres compares
// temporally and is unaffected either way, so this test alone would pass
// even with the bug — it exists to prove the fix doesn't regress on
// Postgres, not to catch the bug (that's the SQLite test's job).
func TestPostgresExecutorReachabilityCheckedAtOrdering(t *testing.T) {
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
repo, err := NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("init postgres schema: %v", err)
}
ctx := context.Background()
executor := &models.Executor{ID: "exec-reachability-pg", Name: "ssh-host", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
if err := repo.CreateExecutor(ctx, executor); err != nil {
t.Fatalf("create executor: %v", err)
}
older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC)
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert newer: %v", err)
}
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1,
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert older: %v", err)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("record = %+v, want the newer write to survive", got)
}
}
@@ -0,0 +1,332 @@
package sqlite
import (
"context"
"path/filepath"
"testing"
"time"
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/task/models"
)
func newReachabilityTestRepo(t *testing.T) (*Repository, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "reachability.db")
dbConn, err := db.OpenSQLite(path)
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
repo, err := NewWithDB(sqlxDB, sqlxDB, nil)
if err != nil {
_ = sqlxDB.Close()
t.Fatalf("new repository: %v", err)
}
t.Cleanup(func() { _ = sqlxDB.Close() })
return repo, path
}
// createSSHExecutor inserts an active SSH executor and returns it, so
// UpsertExecutorReachability's eligibility EXISTS clause is satisfied.
func createSSHExecutor(t *testing.T, repo *Repository) *models.Executor {
t.Helper()
executor := &models.Executor{
ID: "exec-" + t.Name(),
Name: "ssh-host",
Type: models.ExecutorTypeSSH,
Status: models.ExecutorStatusActive,
}
if err := repo.CreateExecutor(context.Background(), executor); err != nil {
t.Fatalf("create executor: %v", err)
}
return executor
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — last-write-wins on
// checked_at regardless of commit order, including a whole-second value
// racing a sub-second one.
func TestUpsertExecutorReachabilityNewerCheckedAtWinsRegardlessOfOrder(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC) // same whole second, later fraction
// Issue the newer write first, the older write second — the newer must survive.
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert newer: %v", err)
}
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateUnreachable,
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert older: %v", err)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("record = %+v, want the newer (reachable, 10.0.0.2) write to survive", got)
}
if got.CheckedAt == nil || !got.CheckedAt.Equal(newer) {
t.Fatalf("CheckedAt = %v, want %v", got.CheckedAt, newer)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — an equal checked_at leaves
// the row unchanged.
func TestUpsertExecutorReachabilityEqualCheckedAtIsDiscarded(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
at := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
obs := models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
}
if err := repo.UpsertExecutorReachability(ctx, obs); err != nil {
t.Fatalf("first upsert: %v", err)
}
obs.Host = "10.0.0.9" // a second write at the SAME checked_at, different host
if err := repo.UpsertExecutorReachability(ctx, obs); err != nil {
t.Fatalf("second upsert: %v", err)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Host != "10.0.0.1" {
t.Fatalf("Host = %q, want the first write's host unchanged (equal checked_at must be discarded)", got.Host)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — a record survives a
// repository re-open against the same database file, byte-identical
// including both timestamps.
func TestExecutorReachabilityRoundTripAcrossReopen(t *testing.T) {
repo, path := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
at := time.Date(2026, 3, 4, 5, 6, 7, 123_000_000, time.UTC)
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
before, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get before reopen: %v", err)
}
dbConn, err := db.OpenSQLite(path)
if err != nil {
t.Fatalf("reopen sqlite: %v", err)
}
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
t.Cleanup(func() { _ = sqlxDB.Close() })
reopened, err := NewWithDB(sqlxDB, sqlxDB, nil)
if err != nil {
t.Fatalf("reopen repository: %v", err)
}
after, err := reopened.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get after reopen: %v", err)
}
if after.Host != before.Host || after.State != before.State {
t.Fatalf("record changed across reopen: before=%+v after=%+v", before, after)
}
if after.CheckedAt == nil || before.CheckedAt == nil || !after.CheckedAt.Equal(*before.CheckedAt) {
t.Fatalf("CheckedAt changed across reopen: before=%v after=%v", before.CheckedAt, after.CheckedAt)
}
if after.LastSuccessAt == nil || before.LastSuccessAt == nil || !after.LastSuccessAt.Equal(*before.LastSuccessAt) {
t.Fatalf("LastSuccessAt changed across reopen: before=%v after=%v", before.LastSuccessAt, after.LastSuccessAt)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.21
func TestDeleteExecutorRemovesReachabilityRecord(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
if err := repo.DeleteExecutor(ctx, executor.ID); err != nil {
t.Fatalf("delete executor: %v", err)
}
if _, err := repo.GetExecutorReachability(ctx, executor.ID); err != models.ErrExecutorReachabilityNotFound {
t.Fatalf("err = %v, want ErrExecutorReachabilityNotFound", err)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.22
func TestResetExecutorReachabilityClearsTheRecordButAdvancesUpdatedAt(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
at := time.Now().UTC()
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1,
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 1,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
before, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get before reset: %v", err)
}
if before.ConsecutiveFailures == 0 || before.CheckedAt == nil {
t.Fatalf("precondition: expected a populated failure record, got %+v", before)
}
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil {
t.Fatalf("reset: %v", err)
}
after, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get after reset: %v", err)
}
if after.State != models.ExecutorReachabilityStateUnknown {
t.Fatalf("State = %q, want unknown", after.State)
}
if after.ConsecutiveFailures != 0 || after.Reason != "" || after.Message != "" {
t.Fatalf("record not cleared: %+v", after)
}
if after.Host != "10.0.0.9" {
t.Fatalf("Host = %q, want the newly saved host", after.Host)
}
if after.CheckedAt != nil || after.LastSuccessAt != nil {
t.Fatalf("timestamps not cleared: checked_at=%v last_success_at=%v", after.CheckedAt, after.LastSuccessAt)
}
if !after.UpdatedAt.After(before.UpdatedAt) {
t.Fatalf("UpdatedAt did not advance: before=%v after=%v", before.UpdatedAt, after.UpdatedAt)
}
}
func TestResetExecutorReachabilityIgnoresStaleExecutorVersion(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
staleVersion := executor.UpdatedAt
executor.Config = map[string]string{"ssh_host": "10.0.0.2"}
if err := repo.UpdateExecutor(ctx, executor); err != nil {
t.Fatalf("update executor: %v", err)
}
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.2", staleVersion); err != nil {
t.Fatalf("reset: %v", err)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.State != models.ExecutorReachabilityStateReachable || got.Host != "10.0.0.1" {
t.Fatalf("record = %+v, want the newer executor version to remain untouched", got)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.17, 001.22
func TestResetExecutorReachabilityIsANoOpForANonActiveExecutor(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
executor := createSSHExecutor(t, repo)
at := time.Now().UTC()
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
InitialState: models.ExecutorReachabilityStateReachable,
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
}); err != nil {
t.Fatalf("upsert: %v", err)
}
executor.Status = models.ExecutorStatusDisabled
if err := repo.UpdateExecutor(ctx, executor); err != nil {
t.Fatalf("disable executor: %v", err)
}
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil {
t.Fatalf("reset: %v", err)
}
got, err := repo.GetExecutorReachability(ctx, executor.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Host != "10.0.0.1" || got.State != models.ExecutorReachabilityStateReachable {
t.Fatalf("record = %+v, want it untouched by a reset on a disabled executor", got)
}
}
func TestListSSHExecutorsForReachabilityOrdersByID(t *testing.T) {
repo, _ := newReachabilityTestRepo(t)
ctx := context.Background()
mustCreate := func(id string, execType models.ExecutorType, status models.ExecutorStatus) {
t.Helper()
if err := repo.CreateExecutor(ctx, &models.Executor{ID: id, Name: id, Type: execType, Status: status}); err != nil {
t.Fatalf("create %s: %v", id, err)
}
}
mustCreate("b-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive)
mustCreate("a-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive)
mustCreate("c-ssh-disabled", models.ExecutorTypeSSH, models.ExecutorStatusDisabled)
mustCreate("d-docker", models.ExecutorTypeLocalDocker, models.ExecutorStatusActive)
deleted := &models.Executor{ID: "e-ssh-deleted", Name: "e-ssh-deleted", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
if err := repo.CreateExecutor(ctx, deleted); err != nil {
t.Fatalf("create deleted: %v", err)
}
if err := repo.DeleteExecutor(ctx, deleted.ID); err != nil {
t.Fatalf("delete: %v", err)
}
got, err := repo.ListSSHExecutorsForReachability(ctx)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(got) != 2 || got[0].ID != "a-ssh" || got[1].ID != "b-ssh" {
ids := make([]string, len(got))
for i, e := range got {
ids[i] = e.ID
}
t.Fatalf("ids = %v, want [a-ssh b-ssh]", ids)
}
}
@@ -280,6 +280,17 @@ type WorkflowStepCreator interface {
CreateStepsFromTemplate(ctx context.Context, workflowID, templateID string) error
}
// ExecutorSaveObserver is notified after an executor create or update
// commits, with before the pre-update snapshot (nil on create) and after the
// saved executor. CreateExecutor and UpdateExecutor are the only call sites
// that can form this before/after comparison — before must be captured prior
// to any in-place mutation of the loaded executor. Implementations decide for
// themselves whether the save is worth acting on (e.g. only SSH executors
// whose connection configuration changed).
type ExecutorSaveObserver interface {
OnExecutorSaved(ctx context.Context, before, after *models.Executor)
}
// WorkspaceBootstrapper owns the atomic persistence of a standard Kanban
// workspace and its initial workflow state.
type WorkspaceBootstrapper interface {
@@ -501,6 +512,7 @@ type Service struct {
providerProber ProviderDefaultBranchProber
gitArchiveCapture GitArchiveCapture
workflowStepCreator WorkflowStepCreator
executorSaveObserver ExecutorSaveObserver
workspaceBootstrapper WorkspaceBootstrapper
workflowStepGetter WorkflowStepGetter
workflowMovePreflight WorkflowMovePreflight
@@ -930,6 +942,13 @@ func (s *Service) SetWorkflowStepCreator(creator WorkflowStepCreator) {
s.workflowStepCreator = creator
}
// SetExecutorSaveObserver wires the observer notified after every executor
// create/update commits. Optional — a Service with no observer wired saves
// executors exactly as before.
func (s *Service) SetExecutorSaveObserver(observer ExecutorSaveObserver) {
s.executorSaveObserver = observer
}
func (s *Service) SetWorkspaceBootstrapper(bootstrapper WorkspaceBootstrapper) {
s.workspaceBootstrapper = bootstrapper
}
@@ -1806,9 +1806,19 @@ func (s *Service) CreateExecutor(ctx context.Context, req *CreateExecutorRequest
return nil, err
}
s.publishExecutorEvent(ctx, events.ExecutorCreated, executor)
s.notifyExecutorSaved(ctx, nil, executor)
return executor, nil
}
// notifyExecutorSaved forwards a committed executor create/update to the
// wired ExecutorSaveObserver, if any. before is nil on create.
func (s *Service) notifyExecutorSaved(ctx context.Context, before, after *models.Executor) {
if s.executorSaveObserver == nil {
return
}
s.executorSaveObserver.OnExecutorSaved(ctx, before, after)
}
func (s *Service) GetExecutor(ctx context.Context, id string) (*models.Executor, error) {
return s.executors.GetExecutor(ctx, id)
}
@@ -1842,12 +1852,14 @@ func (s *Service) UpdateExecutor(ctx context.Context, id string, req *UpdateExec
return nil, ErrActiveTaskSessions
}
}
before := *executor
applyExecutorUpdates(executor, req)
executor.UpdatedAt = time.Now().UTC()
if err := s.executors.UpdateExecutor(ctx, executor); err != nil {
return nil, err
}
s.publishExecutorEvent(ctx, events.ExecutorUpdated, executor)
s.notifyExecutorSaved(ctx, &before, executor)
return executor, nil
}
@@ -0,0 +1,190 @@
package service
import (
"context"
"testing"
"github.com/kandev/kandev/internal/task/models"
)
// fakeExecutorSaveObserver records every OnExecutorSaved call it receives, so
// a test can assert the exact before/after pair the service handed it without
// depending on the reachability package's own behavior.
type fakeExecutorSaveObserver struct {
calls []executorSaveCall
}
type executorSaveCall struct {
before *models.Executor
after *models.Executor
}
func (f *fakeExecutorSaveObserver) OnExecutorSaved(_ context.Context, before, after *models.Executor) {
f.calls = append(f.calls, executorSaveCall{before: before, after: after})
}
func sshExecutorCreateRequest(name string, config map[string]string) *CreateExecutorRequest {
return &CreateExecutorRequest{
Name: name, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive,
Resumable: true, Config: config,
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// Creating an SSH executor always counts as a connection-configuration
// change — there is no prior state to compare against — so the observer must
// see before=nil paired with the newly created executor.
func TestCreateExecutorNotifiesSaveObserverWithNilBefore(t *testing.T) {
svc, _, _ := createTestService(t)
observer := &fakeExecutorSaveObserver{}
svc.SetExecutorSaveObserver(observer)
ctx := context.Background()
executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{
"ssh_host": "10.0.0.5", "ssh_user": "deploy",
}))
if err != nil {
t.Fatalf("CreateExecutor: %v", err)
}
if len(observer.calls) != 1 {
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
}
call := observer.calls[0]
if call.before != nil {
t.Fatalf("before = %+v, want nil on create", call.before)
}
if call.after == nil || call.after.ID != executor.ID {
t.Fatalf("after = %+v, want the created executor", call.after)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// The service notifies the observer on every executor create, regardless of
// type — deciding whether a save is SSH-reachability-relevant is the
// observer implementation's job (internal/executors/reachability.SaveObserver),
// not the service's. This keeps the service layer decoupled from SSH-specific
// eligibility rules, the same way publishExecutorEvent fires unconditionally
// and lets subscribers filter.
func TestCreateExecutorNotifiesSaveObserverForNonSSHType(t *testing.T) {
svc, _, _ := createTestService(t)
observer := &fakeExecutorSaveObserver{}
svc.SetExecutorSaveObserver(observer)
ctx := context.Background()
executor, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{
Name: "Local runner", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive, Resumable: true,
})
if err != nil {
t.Fatalf("CreateExecutor: %v", err)
}
if len(observer.calls) != 1 {
t.Fatalf("observer calls = %d, want 1 (gating is the observer's job, not the service's)", len(observer.calls))
}
if observer.calls[0].after == nil || observer.calls[0].after.ID != executor.ID {
t.Fatalf("after = %+v, want the created executor", observer.calls[0].after)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1, AC-EXECUTORS-SSH-REACHABILITY-003.2
//
// Updating an SSH executor's host must notify the observer with the literal
// old-host/new-host pair — captured strictly before applyExecutorUpdates
// mutates the executor in place, per the work order's own risk callout. A
// self-comparison (before and after pointing at the same, already-mutated
// struct) would silently make this test pass with either host string, so it
// asserts both values explicitly.
func TestUpdateExecutorNotifiesSaveObserverOnHostChange(t *testing.T) {
svc, _, _ := createTestService(t)
executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH)
ctx := context.Background()
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"},
}); err != nil {
t.Fatalf("seed UpdateExecutor: %v", err)
}
observer := &fakeExecutorSaveObserver{}
svc.SetExecutorSaveObserver(observer)
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
Config: map[string]string{"ssh_host": "10.0.0.2", "ssh_user": "deploy"},
}); err != nil {
t.Fatalf("UpdateExecutor: %v", err)
}
if len(observer.calls) != 1 {
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
}
call := observer.calls[0]
if call.before == nil || call.before.Config["ssh_host"] != "10.0.0.1" {
t.Fatalf("before host = %+v, want 10.0.0.1", call.before)
}
if call.after == nil || call.after.Config["ssh_host"] != "10.0.0.2" {
t.Fatalf("after host = %+v, want 10.0.0.2", call.after)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// A name-only update still notifies the observer (the service does not
// pre-filter by which fields changed) — the observer's own
// connectionConfigChanged comparison is what decides a name-only save isn't
// reachability-relevant, and that decision is covered in the reachability
// package's own save_observer_test.go, not here. This test only pins that the
// service keeps calling with the correct before/after regardless of which
// fields the caller supplied.
func TestUpdateExecutorNotifiesSaveObserverForNameOnlyChange(t *testing.T) {
svc, _, _ := createTestService(t)
executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH)
ctx := context.Background()
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"},
}); err != nil {
t.Fatalf("seed UpdateExecutor: %v", err)
}
observer := &fakeExecutorSaveObserver{}
svc.SetExecutorSaveObserver(observer)
newName := "Renamed SSH box"
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{Name: &newName}); err != nil {
t.Fatalf("UpdateExecutor: %v", err)
}
if len(observer.calls) != 1 {
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
}
if observer.calls[0].before.Config["ssh_host"] != "10.0.0.1" || observer.calls[0].after.Config["ssh_host"] != "10.0.0.1" {
t.Fatalf("host should be unchanged across a name-only update: %+v", observer.calls[0])
}
if observer.calls[0].after.Name != newName {
t.Fatalf("after.Name = %q, want %q", observer.calls[0].after.Name, newName)
}
}
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
//
// A nil observer (the default in every test not exercising reachability) must
// never be dereferenced by CreateExecutor/UpdateExecutor.
func TestExecutorSaveObserverNilSafe(t *testing.T) {
svc, _, _ := createTestService(t)
ctx := context.Background()
executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{
"ssh_host": "10.0.0.5",
}))
if err != nil {
t.Fatalf("CreateExecutor: %v", err)
}
newHost := "10.0.0.9"
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
Config: map[string]string{"ssh_host": newHost},
}); err != nil {
t.Fatalf("UpdateExecutor: %v", err)
}
}
+2
View File
@@ -271,6 +271,7 @@ const (
ActionSessionModelsUpdated = "session.models_updated"
ActionSessionModelFallback = "session.model_fallback"
ActionSessionModelSelectionWarning = "session.model_selection_warning"
ActionSessionLaunchWarning = "session.launch.warning"
ActionSessionMCPStatusUpdated = "session.mcp_status_updated"
ActionSessionInfoUpdated = "session.info_updated"
ActionSessionSetMode = "session.set_mode"
@@ -298,6 +299,7 @@ const (
ActionExecutorCreated = "executor.created"
ActionExecutorUpdated = "executor.updated"
ActionExecutorDeleted = "executor.deleted"
ActionExecutorReachabilityChanged = "executor.reachability.changed"
ActionEnvironmentCreated = "environment.created"
ActionEnvironmentUpdated = "environment.updated"
ActionEnvironmentDeleted = "environment.deleted"
@@ -13,6 +13,7 @@ import { fetchExecutor, listExecutors, updateExecutor } from "@/lib/api/domains/
import { SSHConnectionCard } from "@/components/settings/ssh-connection-card";
import type { SSHExecutorConfig } from "@/components/settings/ssh-connection-card";
import { SSHSessionsCard } from "@/components/settings/ssh-sessions-card";
import { SSHReachabilityCard } from "@/components/settings/ssh-reachability-card";
import { listSSHSessions } from "@/lib/api/domains/ssh-api";
import { getExecutorLabel } from "@/lib/executor-icons";
import {
@@ -135,6 +136,7 @@ function SSHExecutorView({
coordinatedSaveId={`ssh-executor:${executor.id}`}
runningSessionCount={sessionCount}
/>
<SSHReachabilityCard executorId={executor.id} />
<SSHSessionsCard executorId={executor.id} />
</div>
);
@@ -0,0 +1,213 @@
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { StateProvider, useAppStoreApi } from "@/components/state-provider";
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
const getSSHExecutorReachability = vi.fn();
const probeSSHExecutorReachability = vi.fn();
vi.mock("@/lib/api/domains/ssh-api", () => ({
getSSHExecutorReachability: (...args: unknown[]) => getSSHExecutorReachability(...args),
probeSSHExecutorReachability: (...args: unknown[]) => probeSSHExecutorReachability(...args),
}));
import { SSHReachabilityCard, isReachabilityStale } from "./ssh-reachability-card";
const EXECUTOR_ID = "executor-1";
const BASE_TIMESTAMP = "2026-09-17T00:00:00.000Z";
const STATE_TESTID = "ssh-reachability-state";
function record(overrides: Partial<SSHReachabilityRecord> = {}): SSHReachabilityRecord {
return {
executor_id: EXECUTOR_ID,
state: "reachable",
reason: "",
consecutive_failures: 0,
host: "10.0.0.5",
checked_at: BASE_TIMESTAMP,
last_success_at: BASE_TIMESTAMP,
updated_at: BASE_TIMESTAMP,
probing_enabled: true,
probe_interval_seconds: 60,
persisted: true,
...overrides,
};
}
function renderCard(executorId = EXECUTOR_ID) {
return render(
<StateProvider>
<SSHReachabilityCard executorId={executorId} />
</StateProvider>,
);
}
function PushReachability({ record: pushed }: { record: SSHReachabilityRecord }) {
const storeApi = useAppStoreApi();
return (
<button type="button" onClick={() => storeApi.getState().setSSHReachability(pushed)}>
push
</button>
);
}
function renderCardWithPush(pushed: SSHReachabilityRecord) {
return render(
<StateProvider>
<SSHReachabilityCard executorId={EXECUTOR_ID} />
<PushReachability record={pushed} />
</StateProvider>,
);
}
async function waitForState(text: string) {
await waitFor(() => expect(screen.getByTestId(STATE_TESTID).textContent).toBe(text));
}
beforeEach(() => {
getSSHExecutorReachability.mockReset();
probeSSHExecutorReachability.mockReset();
});
afterEach(() => {
cleanup();
vi.restoreAllMocks();
});
describe("isReachabilityStale", () => {
it("is false when there is no probing cadence or no completed probe", () => {
expect(isReachabilityStale(null, 60, true, Date.now())).toBe(false);
expect(isReachabilityStale(BASE_TIMESTAMP, 60, false, Date.now())).toBe(false);
});
it("is false within three intervals and true just past it", () => {
const checkedMs = Date.parse(BASE_TIMESTAMP);
expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000)).toBe(false);
expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000 + 1)).toBe(true);
});
});
describe("SSHReachabilityCard rendering", () => {
it("renders state, host, failure count and age for a reachable record", async () => {
getSSHExecutorReachability.mockResolvedValue(record());
renderCard();
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
expect(screen.getByTestId("ssh-reachability-host").textContent).toContain("10.0.0.5");
expect(screen.queryByTestId("ssh-reachability-reason")).toBeNull();
});
it("renders reason and message for an unreachable record", async () => {
getSSHExecutorReachability.mockResolvedValue(
record({
state: "unreachable",
reason: "timeout",
message: "dial tcp: i/o timeout",
consecutive_failures: 3,
}),
);
renderCard();
await waitFor(() => expect(screen.getByTestId("ssh-reachability-reason")).toBeTruthy());
expect(screen.getByTestId("ssh-reachability-message").textContent).toBe(
"dial tcp: i/o timeout",
);
expect(screen.getByTestId("ssh-reachability-failures").textContent).toContain("3");
});
it("marks a record older than three intervals as stale", async () => {
const old = new Date(Date.now() - 4 * 60_000).toISOString();
getSSHExecutorReachability.mockResolvedValue(
record({ probe_interval_seconds: 60, checked_at: old }),
);
renderCard();
await waitFor(() => expect(screen.getByTestId("ssh-reachability-stale")).toBeTruthy());
});
it("does not mark a fresh record as stale", async () => {
const fresh = new Date().toISOString();
getSSHExecutorReachability.mockResolvedValue(
record({
probe_interval_seconds: 60,
checked_at: fresh,
last_success_at: fresh,
updated_at: fresh,
}),
);
renderCard();
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
expect(screen.queryByTestId("ssh-reachability-stale")).toBeNull();
});
});
describe("SSHReachabilityCard live updates", () => {
it("applies a pushed executor.reachability.changed record live, without a reload", async () => {
getSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" }));
renderCardWithPush(record({ state: "unreachable", reason: "auth", message: "boom" }));
await waitForState("Reachable");
fireEvent.click(screen.getByRole("button", { name: "push" }));
await waitForState("Unreachable");
expect(getSSHExecutorReachability).toHaveBeenCalledTimes(1);
});
it("replaces a stored record with a reset (null checked_at, newer updated_at)", async () => {
getSSHExecutorReachability.mockResolvedValue(
record({ state: "unreachable", reason: "timeout", updated_at: BASE_TIMESTAMP }),
);
renderCardWithPush(
record({
state: "unknown",
reason: "",
checked_at: null,
last_success_at: null,
updated_at: "2026-09-17T00:01:00.000Z",
}),
);
await waitForState("Unreachable");
fireEvent.click(screen.getByRole("button", { name: "push" }));
await waitForState("Unknown");
});
});
describe("SSHReachabilityCard refresh cadence", () => {
it("says periodic probing is off and starts no refresh timer when probing_enabled is false", async () => {
const setIntervalSpy = vi.spyOn(window, "setInterval");
getSSHExecutorReachability.mockResolvedValue(
record({ probing_enabled: false, probe_interval_seconds: 0 }),
);
renderCard();
await waitFor(() => expect(screen.getByTestId("ssh-reachability-probing-off")).toBeTruthy());
// Exclude testing-library's own internal waitFor polling interval (50ms);
// this asserts the card itself never armed a refresh timer.
const ownIntervalCalls = setIntervalSpy.mock.calls.filter((call) => call[1] !== 50);
expect(ownIntervalCalls).toHaveLength(0);
expect(screen.queryByTestId(STATE_TESTID)?.textContent).not.toBe("Unreachable");
});
it("starts a refresh timer at the configured probe interval when probing is enabled", async () => {
const setIntervalSpy = vi.spyOn(window, "setInterval");
getSSHExecutorReachability.mockResolvedValue(record({ probe_interval_seconds: 45 }));
renderCard();
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 45_000);
});
});
describe("SSHReachabilityCard errors and probe-now", () => {
it("reports reachability as not known, never unreachable, when the initial load fails", async () => {
getSSHExecutorReachability.mockRejectedValue(new Error("network error"));
renderCard();
await waitFor(() => expect(screen.getByTestId("ssh-reachability-not-known")).toBeTruthy());
expect(screen.queryByTestId(STATE_TESTID)).toBeNull();
});
it("runs an immediate probe and applies the result", async () => {
getSSHExecutorReachability.mockResolvedValue(
record({ state: "unreachable", reason: "network" }),
);
probeSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" }));
renderCard();
await waitForState("Unreachable");
fireEvent.click(screen.getByTestId("ssh-reachability-probe-now"));
await waitForState("Reachable");
expect(probeSSHExecutorReachability).toHaveBeenCalledWith(EXECUTOR_ID);
});
});
@@ -0,0 +1,198 @@
"use client";
import { useTranslation } from "react-i18next";
import { Badge } from "@kandev/ui/badge";
import { Button } from "@kandev/ui/button";
import { CardContent } from "@kandev/ui/card";
import { IconLoader2 } from "@tabler/icons-react";
import type { SSHReachabilityRecord, SSHReachabilityReason } from "@/lib/types/http-ssh";
import { formatRelative } from "@/lib/i18n/formats";
import { useSSHReachability } from "@/hooks/domains/settings/use-ssh-reachability";
import { SettingsCard } from "@/components/settings/settings-card";
import { SettingsCardHeader } from "@/components/settings/settings-card-header";
import { settingsActionClassName } from "@/components/settings/settings-control";
export interface SSHReachabilityCardProps {
executorId: string;
}
const STALE_INTERVAL_MULTIPLIER = 3;
/**
* A record older than three probe intervals is stale rather than current.
* Disabled probing has no cadence to measure against, and a never-probed
* record has nothing to be stale relative to, so both read as not-stale.
*/
export function isReachabilityStale(
checkedAt: string | null,
probeIntervalSeconds: number,
probingEnabled: boolean,
now: number = Date.now(),
): boolean {
if (!probingEnabled || !checkedAt || probeIntervalSeconds <= 0) return false;
const checkedMs = Date.parse(checkedAt);
if (Number.isNaN(checkedMs)) return false;
return now - checkedMs > probeIntervalSeconds * 1000 * STALE_INTERVAL_MULTIPLIER;
}
export function reachabilityReasonKey(reason: SSHReachabilityReason): string | null {
switch (reason) {
case "config":
return "executors:sshReachabilityReasonConfig";
case "timeout":
return "executors:sshReachabilityReasonTimeout";
case "host_key":
return "executors:sshReachabilityReasonHostKey";
case "auth":
return "executors:sshReachabilityReasonAuth";
case "network":
return "executors:sshReachabilityReasonNetwork";
case "unknown":
return "executors:sshReachabilityReasonUnknown";
default:
return null;
}
}
export function reachabilityStateKey(state: SSHReachabilityRecord["state"]): string {
switch (state) {
case "reachable":
return "executors:sshReachabilityStateReachable";
case "unreachable":
return "executors:sshReachabilityStateUnreachable";
default:
return "executors:sshReachabilityStateUnknown";
}
}
/**
* Renders the most recent SSH reachability record for one executor: state,
* probed host, failure reason/message, consecutive-failure count, and the
* age of the last completed and last successful probe. Never fetches
* directly into local-only state — every fetch result is written through the
* store so a live executor.reachability.changed push and this card's own
* refresh share one reconciliation path.
*/
export function SSHReachabilityCard({ executorId }: SSHReachabilityCardProps) {
const { t } = useTranslation();
const { record, loadError, probing, probeNow, now } = useSSHReachability(executorId);
return (
<SettingsCard data-testid="ssh-reachability-card">
<SettingsCardHeader
title={t("executors:sshReachabilityTitle")}
description={t("executors:sshReachabilityDescription")}
actions={
<Button
variant="outline"
size="sm"
onClick={() => void probeNow()}
disabled={probing}
data-testid="ssh-reachability-probe-now"
className={settingsActionClassName("cursor-pointer")}
>
{probing ? <IconLoader2 className="mr-1.5 h-4 w-4 animate-spin" /> : null}
{t("executors:sshReachabilityProbeNow")}
</Button>
}
/>
<CardContent>
<ReachabilityBody record={record} loadError={loadError} now={now} />
</CardContent>
</SettingsCard>
);
}
function ReachabilityBody({
record,
loadError,
now,
}: {
record: SSHReachabilityRecord | undefined;
loadError: boolean;
now: number;
}) {
const { t } = useTranslation();
if (!record) {
if (loadError) {
return (
<p data-testid="ssh-reachability-not-known" className="text-sm text-muted-foreground">
{t("executors:sshReachabilityLoadFailed")}
</p>
);
}
return null;
}
const stale = isReachabilityStale(
record.checked_at,
record.probe_interval_seconds,
record.probing_enabled,
now,
);
const reasonKey = record.reason ? reachabilityReasonKey(record.reason) : null;
return (
<div className="space-y-2 text-sm">
<div className="flex items-center gap-2">
<StateBadge state={record.state} />
{stale ? (
<Badge
data-testid="ssh-reachability-stale"
variant="outline"
className="border-amber-500/30 bg-amber-500/10 text-amber-700"
>
{t("executors:sshReachabilityStale")}
</Badge>
) : null}
</div>
{record.host ? (
<p data-testid="ssh-reachability-host">
{t("executors:sshReachabilityHost", { host: record.host })}
</p>
) : null}
{record.state === "unreachable" && reasonKey ? (
<p data-testid="ssh-reachability-reason">{t(reasonKey)}</p>
) : null}
{record.state === "unreachable" && record.message ? (
<p data-testid="ssh-reachability-message" className="text-muted-foreground">
{record.message}
</p>
) : null}
<p data-testid="ssh-reachability-failures">
{t("executors:sshReachabilityConsecutiveFailures", { count: record.consecutive_failures })}
</p>
<p data-testid="ssh-reachability-age">
{record.checked_at
? t("executors:sshReachabilityAgeLastProbe", { age: formatRelative(record.checked_at) })
: t("executors:sshReachabilityNeverProbed")}
</p>
<p data-testid="ssh-reachability-last-success">
{record.last_success_at
? t("executors:sshReachabilityAgeLastSuccess", {
age: formatRelative(record.last_success_at),
})
: t("executors:sshReachabilityNoSuccess")}
</p>
{!record.probing_enabled ? (
<p data-testid="ssh-reachability-probing-off" className="text-muted-foreground">
{t("executors:sshReachabilityProbingOff")}
</p>
) : null}
</div>
);
}
const STATE_BADGE_CLASSNAMES: Record<SSHReachabilityRecord["state"], string> = {
reachable: "border-green-500/30 bg-green-500/10 text-green-700",
unreachable: "border-red-500/30 bg-red-500/10 text-red-700",
unknown: "border-muted-foreground/30 bg-muted text-muted-foreground",
};
function StateBadge({ state }: { state: SSHReachabilityRecord["state"] }) {
const { t } = useTranslation();
const variantClass = STATE_BADGE_CLASSNAMES[state];
return (
<Badge data-testid="ssh-reachability-state" variant="outline" className={variantClass}>
{t(reachabilityStateKey(state))}
</Badge>
);
}
@@ -7,3 +7,5 @@ export { SSHConnectionCard } from "./ssh-connection-card";
export type { SSHConnectionCardProps, SSHExecutorConfig } from "./ssh-connection-card";
export { SSHSessionsCard } from "./ssh-sessions-card";
export type { SSHSessionsCardProps } from "./ssh-sessions-card";
export { SSHReachabilityCard } from "./ssh-reachability-card";
export type { SSHReachabilityCardProps } from "./ssh-reachability-card";
@@ -0,0 +1,77 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { cleanup, render, screen } from "@testing-library/react";
import { StateProvider } from "@/components/state-provider";
import type { LaunchWarningEntry } from "@/lib/state/slices/session-runtime/types";
import { LaunchWarning } from "./launch-warning";
const SESSION_ID = "session-1";
function makeEntry(overrides: Partial<LaunchWarningEntry> = {}): LaunchWarningEntry {
return {
executorId: "executor-1",
host: "10.0.0.5",
state: "unreachable",
reason: "timeout",
...overrides,
};
}
afterEach(() => {
cleanup();
vi.useRealTimers();
});
function renderWithEntry(entry?: LaunchWarningEntry) {
return render(
<StateProvider
initialState={{
launchWarning: { bySessionId: entry ? { [SESSION_ID]: entry } : {} },
}}
>
<LaunchWarning sessionId={SESSION_ID} />
</StateProvider>,
);
}
describe("LaunchWarning", () => {
it("renders nothing when no launch warning was published for this session", () => {
const { container } = renderWithEntry(undefined);
expect(container.textContent).toBe("");
});
it("names the probed host from the event payload", () => {
renderWithEntry(makeEntry());
expect(screen.getByTestId("launch-warning").textContent).toContain("10.0.0.5");
});
it("shows the age of the last successful probe", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-06-11T00:05:00.000Z"));
try {
renderWithEntry(makeEntry({ lastSuccessAt: "2026-06-11T00:00:00.000Z" }));
expect(screen.getByTestId("launch-warning-last-success").textContent).toMatch(/5m/);
} finally {
vi.useRealTimers();
}
});
it("states plainly that no successful probe was ever recorded, not a zero age", () => {
renderWithEntry(makeEntry({ lastSuccessAt: undefined }));
const text = screen.getByTestId("launch-warning-last-success").textContent ?? "";
expect(text).not.toMatch(/0[ms]/);
expect(text.length).toBeGreaterThan(0);
});
it.each(["config", "timeout", "host_key", "auth", "network", "unknown"])(
"renders a reason sentence for %s without throwing",
(reason) => {
renderWithEntry(makeEntry({ reason }));
expect(screen.getByTestId("launch-warning-reason").textContent?.length).toBeGreaterThan(0);
},
);
it("never renders any confirmation control — the launch is never blocked", () => {
renderWithEntry(makeEntry());
expect(screen.queryByRole("button")).toBeNull();
});
});
@@ -0,0 +1,67 @@
"use client";
import { useTranslation } from "react-i18next";
import { IconAlertTriangle } from "@tabler/icons-react";
import { useAppStore } from "@/components/state-provider";
import { formatRelative } from "@/lib/i18n/formats";
type LaunchWarningProps = {
sessionId: string;
};
function reasonKey(reason: string): string {
switch (reason) {
case "config":
return "task:launchWarningReasonConfig";
case "timeout":
return "task:launchWarningReasonTimeout";
case "host_key":
return "task:launchWarningReasonHostKey";
case "auth":
return "task:launchWarningReasonAuth";
case "network":
return "task:launchWarningReasonNetwork";
default:
return "task:launchWarningReasonUnknown";
}
}
/**
* Renders the session.launch.warning event verbatim at the point the user
* initiated the launch — never re-derives reachability from the settings
* store slice, and never blocks or confirms the launch it describes.
*/
export function LaunchWarning({ sessionId }: LaunchWarningProps) {
const { t } = useTranslation();
const entry = useAppStore((state) => state.launchWarning.bySessionId[sessionId]);
if (!entry) return null;
return (
<div
className="flex min-w-0 gap-3 border-b border-border/50 py-3"
data-testid="launch-warning"
role="status"
>
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-md bg-amber-500/10 text-amber-600 dark:text-amber-400">
<IconAlertTriangle className="h-4 w-4" aria-hidden="true" />
</div>
<div className="min-w-0 flex-1 space-y-1 text-sm">
<p className="font-medium">{t("task:launchWarningTitle")}</p>
{entry.host ? (
<p className="break-words text-muted-foreground">
{t("task:launchWarningHost", { host: entry.host })}
</p>
) : null}
<p data-testid="launch-warning-last-success" className="text-muted-foreground">
{entry.lastSuccessAt
? t("task:launchWarningLastSuccess", { age: formatRelative(entry.lastSuccessAt) })
: t("task:launchWarningNeverSucceeded")}
</p>
<p data-testid="launch-warning-reason" className="text-muted-foreground">
{t(reasonKey(entry.reason))}
</p>
</div>
</div>
);
}
@@ -44,6 +44,7 @@ const appStoreState = {
},
agentProfiles: { items: [] },
messages: { bySession: { [SESSION_ID]: [priorTranscriptMessage] } },
launchWarning: { bySessionId: {} },
};
const panelState = {
+62 -48
View File
@@ -54,6 +54,7 @@ import { routePanelMouseDown } from "./chat/route-panel-mouse-down";
import { useTranslation } from "react-i18next";
import { loadMessageWindowAround } from "@/hooks/domains/session/load-message-window";
import { LaunchWarning } from "./launch-warning";
import { useTaskLaunchErrorContext } from "./task-launch-error-context";
import { useTaskStatusSummary } from "@/hooks/domains/task/use-task-status-summary";
import { TaskMarkdownFileLinkProvider } from "@/components/shared/task-markdown-file-link-provider";
@@ -1253,54 +1254,67 @@ export const TaskChatPanel = memo(function TaskChatPanel({
{!hideLaunchQueueStatus && <LaunchQueueStatus queue={launchStatusSummary?.launch_queue} />}
{!hideWipQueueStatus && <WipQueueStatus taskId={summaryTaskId} />}
<ParkedSessionNote visible={hasWorkflowParkingMarker(session?.metadata)} />
<PanelBody padding={false} scroll={false} className="relative overflow-hidden">
<TaskMarkdownFileLinkProvider
taskId={taskId}
sessionId={resolvedSessionId}
worktreePath={getSessionWorkspacePath(session)}
onOpenFile={onOpenFile}
>
<MessageList
ref={messageListRef}
items={groupedItems}
messages={allMessages}
footerActionMessages={footerActionMessages}
permissionsByToolCallId={permissionsByToolCallId}
childrenByParentToolCallId={childrenByParentToolCallId}
taskId={taskId ?? undefined}
sessionId={resolvedSessionId}
messagesLoading={messagesLoading}
historyRefreshPending={historyRefreshPending}
historyStatus={historyStatus}
historyError={historyError}
onRetryHistory={retryHistory}
isWorking={isWorking}
sessionState={session?.state}
worktreePath={getSessionWorkspacePath(session)}
onOpenFile={onOpenFile}
dividerBeforeItemKey={dividerBeforeItemKey}
lastPromptMessageId={lastPromptMessageId}
onLastPromptEdgeChange={setLastPromptEdge}
firstMessageId={firstMessageId}
onFirstMessageHiddenChange={setIsFirstMessageHidden}
anchoredBarHeight={showAnchoredBar && lastPromptMessage ? anchoredBarHeight : 0}
isVisible={transcriptIsVisible}
launchErrorOwned={launchErrorOwned}
launchErrorStamp={launchErrorOwned ? taskLaunchError?.stamp : undefined}
launchErrorOccurredAt={launchErrorOwned ? taskLaunchError?.occurred_at : undefined}
stickyPromptBar={
showAnchoredBar && lastPromptMessage ? (
<AnchoredLastPromptBar
promptText={lastPromptMessage.content}
isVisible={anchoredBarVisible}
onScrollUp={scrollToLastPrompt}
showScrollToLastPrompt={showScrollToLastPrompt}
onHeightChange={setAnchoredBarHeight}
/>
) : undefined
}
/>
</TaskMarkdownFileLinkProvider>
<PanelBody
padding={false}
scroll={false}
className="relative flex min-h-0 flex-col overflow-hidden"
>
<div className="flex min-h-0 flex-1 flex-col">
{resolvedSessionId ? (
<div className="shrink-0 px-2">
<LaunchWarning sessionId={resolvedSessionId} />
</div>
) : null}
<div className="min-h-0 flex-1">
<TaskMarkdownFileLinkProvider
taskId={taskId}
sessionId={resolvedSessionId}
worktreePath={getSessionWorkspacePath(session)}
onOpenFile={onOpenFile}
>
<MessageList
ref={messageListRef}
items={groupedItems}
messages={allMessages}
footerActionMessages={footerActionMessages}
permissionsByToolCallId={permissionsByToolCallId}
childrenByParentToolCallId={childrenByParentToolCallId}
taskId={taskId ?? undefined}
sessionId={resolvedSessionId}
messagesLoading={messagesLoading}
historyRefreshPending={historyRefreshPending}
historyStatus={historyStatus}
historyError={historyError}
onRetryHistory={retryHistory}
isWorking={isWorking}
sessionState={session?.state}
worktreePath={getSessionWorkspacePath(session)}
onOpenFile={onOpenFile}
dividerBeforeItemKey={dividerBeforeItemKey}
lastPromptMessageId={lastPromptMessageId}
onLastPromptEdgeChange={setLastPromptEdge}
firstMessageId={firstMessageId}
onFirstMessageHiddenChange={setIsFirstMessageHidden}
anchoredBarHeight={showAnchoredBar && lastPromptMessage ? anchoredBarHeight : 0}
isVisible={transcriptIsVisible}
launchErrorOwned={launchErrorOwned}
launchErrorStamp={launchErrorOwned ? taskLaunchError?.stamp : undefined}
launchErrorOccurredAt={launchErrorOwned ? taskLaunchError?.occurred_at : undefined}
stickyPromptBar={
showAnchoredBar && lastPromptMessage ? (
<AnchoredLastPromptBar
promptText={lastPromptMessage.content}
isVisible={anchoredBarVisible}
onScrollUp={scrollToLastPrompt}
showScrollToLastPrompt={showScrollToLastPrompt}
onHeightChange={setAnchoredBarHeight}
/>
) : undefined
}
/>
</TaskMarkdownFileLinkProvider>
</div>
</div>
{isJumpLoading && (
<div
data-testid="transcript-jump-loading"
+32
View File
@@ -195,4 +195,36 @@ export class SSHSettingsPage {
sessionRow(sessionId: string): Locator {
return this.page.getByTestId(`ssh-session-row-${sessionId}`);
}
// --- reachability card ---
get reachabilityCard(): Locator {
return this.page.getByTestId("ssh-reachability-card");
}
get reachabilityState(): Locator {
return this.page.getByTestId("ssh-reachability-state");
}
get reachabilityHost(): Locator {
return this.page.getByTestId("ssh-reachability-host");
}
get reachabilityReason(): Locator {
return this.page.getByTestId("ssh-reachability-reason");
}
get reachabilityProbeNowButton(): Locator {
return this.page.getByTestId("ssh-reachability-probe-now");
}
/**
* Click "Probe now" and wait for the synchronous probe route to return.
* The button re-enabling is the completion signal — the click handler
* awaits the request before clearing its own disabled state.
*/
async probeNow(): Promise<void> {
await this.reachabilityProbeNowButton.click();
await expect(this.reachabilityProbeNowButton).toBeEnabled({ timeout: 30_000 });
}
}
+100
View File
@@ -0,0 +1,100 @@
import { test, expect } from "../../fixtures/ssh-test-base";
import { dropTrafficToPort22, restoreTraffic } from "../../helpers/ssh";
import { SSHSettingsPage } from "../../pages/SSHSettingsPage";
/**
* SSH reachability: settings-panel state transitions and non-gating launch
* behavior against a real sshd target.
*
* Drives the failure threshold through repeated "Probe now" clicks rather
* than the scheduled poller's wall-clock interval — `httpProbeReachability`
* runs the exact same hysteresis path (`store.Observe`) an automatic pass
* would, so this proves the same threshold logic without making the spec's
* runtime depend on a configured interval.
*
* Covers AC-EXECUTORS-SSH-REACHABILITY-001.9, 001.11, 002.1, 003.1, 003.3.
*/
test.describe("ssh executor — reachability", () => {
test("reflects reachable, then unreachable after the failure threshold, then clears on a single success", async ({
testPage,
seedData,
}) => {
test.setTimeout(150_000);
const page = new SSHSettingsPage(testPage);
await page.gotoExisting(seedData.sshExecutorId);
// A fresh probe against the live target starts reachable.
await page.probeNow();
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host);
dropTrafficToPort22(seedData.sshTarget);
try {
// First failed probe: below the two-probe failure threshold, so the
// panel must not have flipped yet.
await page.probeNow();
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
// Second consecutive failure crosses the threshold.
await page.probeNow();
await expect(page.reachabilityState).toHaveText(/^unreachable$/i);
await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host);
await expect(page.reachabilityReason).toBeVisible();
} finally {
restoreTraffic(seedData.sshTarget);
}
// A single successful probe clears an unreachable state immediately;
// no second confirming probe is required.
await page.probeNow();
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
});
test("a launch started while the host is unreachable is attempted and fails naming the host, not refused", async ({
apiClient,
seedData,
}) => {
test.setTimeout(240_000);
dropTrafficToPort22(seedData.sshTarget);
try {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Reachability: launch while unreachable",
seedData.agentProfileId,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
executor_profile_id: seedData.sshExecutorProfileId,
},
);
// The task and its session are created rather than refused — the
// launch is attempted and only fails once the real connection attempt
// gives up. `dropTrafficToPort22` drops inbound packets on the
// container's own port 22, but Docker's own port-publishing plumbing
// (docker-proxy on some hosts) can still complete the client's TCP
// handshake before its own upstream dial to the container stalls, so
// the failure surfaces only once that stalls out — well past the
// client-side `sshDialTimeout` (30s) alone. Give this real, variable
// network path a wide budget rather than one tuned to a fast local
// failure (e.g. a missing remote binary).
let errMsg = "";
await expect
.poll(
async () => {
const { sessions } = await apiClient.listTaskSessions(task.id);
const session = sessions[0];
errMsg = session?.error_message ?? "";
return session?.state ?? "";
},
{ timeout: 200_000, message: "session reaches FAILED against the unreachable host" },
)
.toBe("FAILED");
expect(errMsg).toContain(seedData.sshTarget.host);
} finally {
restoreTraffic(seedData.sshTarget);
}
});
});
@@ -0,0 +1,82 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useAppStore, useAppStoreApi } from "@/components/state-provider";
import {
getSSHExecutorReachability,
probeSSHExecutorReachability,
} from "@/lib/api/domains/ssh-api";
/**
* Owns the settings-domain fetch, refresh, and immediate-probe lifecycle for
* one SSH executor. The store remains the source of truth for the record, so
* WebSocket updates and HTTP responses use the same reconciliation path.
*/
export function useSSHReachability(executorId: string) {
const record = useAppStore((state) => state.sshReachability.byExecutorId[executorId]);
const storeApi = useAppStoreApi();
const [loadError, setLoadError] = useState(false);
const [probing, setProbing] = useState(false);
const [now, setNow] = useState(() => Date.now());
const seqRef = useRef(0);
const load = useCallback(async () => {
const seq = ++seqRef.current;
try {
const response = await getSSHExecutorReachability(executorId);
if (seq !== seqRef.current) return;
setLoadError(false);
storeApi.getState().setSSHReachability(response);
} catch {
if (seq !== seqRef.current) return;
setLoadError(true);
}
}, [executorId, storeApi]);
useEffect(() => {
seqRef.current = 0;
setLoadError(false);
setNow(Date.now());
void load();
return () => {
seqRef.current = -1;
};
// The executor identity is the lifecycle boundary. `load` is stable for
// that identity and must not restart the request on every store update.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [executorId]);
useEffect(() => {
if (!record || !record.probing_enabled || record.probe_interval_seconds <= 0) return;
const interval = window.setInterval(() => void load(), record.probe_interval_seconds * 1000);
return () => window.clearInterval(interval);
}, [record, load]);
// A failed refresh can leave the record object unchanged. Keep the stale
// badge live in that case instead of waiting for a successful fetch to
// trigger another render.
useEffect(() => {
if (!record?.probing_enabled || !record.checked_at || record.probe_interval_seconds <= 0) {
return;
}
const checkedAt = Date.parse(record.checked_at);
if (Number.isNaN(checkedAt)) return;
const staleAt = checkedAt + record.probe_interval_seconds * 1000 * 3;
const delay = Math.max(0, staleAt - Date.now() + 1);
const timer = window.setTimeout(() => setNow(Date.now()), delay);
return () => window.clearTimeout(timer);
}, [record]);
const probeNow = useCallback(async () => {
setProbing(true);
try {
const response = await probeSSHExecutorReachability(executorId);
setLoadError(false);
storeApi.getState().setSSHReachability(response);
} catch {
setLoadError(true);
} finally {
setProbing(false);
}
}, [executorId, storeApi]);
return { record, loadError, probing, probeNow, now };
}
+97
View File
@@ -0,0 +1,97 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
vi.mock("@/lib/config", () => ({
getBackendConfig: () => ({ apiBaseUrl: "http://api.test" }),
}));
import {
getSSHReachability,
getSSHExecutorReachability,
probeSSHExecutorReachability,
} from "./ssh-api";
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
const BASE = "http://api.test/api/v1/ssh";
type FetchInput = Parameters<typeof fetch>[0];
type FetchInit = Parameters<typeof fetch>[1];
const fetchSpy = vi.fn<(...args: [FetchInput, FetchInit?]) => Promise<Response>>();
beforeEach(() => {
fetchSpy.mockReset();
vi.stubGlobal("fetch", fetchSpy);
});
afterEach(() => {
vi.unstubAllGlobals();
});
function jsonResponse(body: unknown): Response {
return new Response(JSON.stringify(body), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
function lastCall(): { url: string; init: FetchInit | undefined } {
const call = fetchSpy.mock.calls.at(-1);
if (!call) throw new Error("expected fetch to have been called");
return { url: String(call[0]), init: call[1] };
}
function record(overrides: Partial<SSHReachabilityRecord> = {}): SSHReachabilityRecord {
return {
executor_id: "exec-1",
state: "reachable",
reason: "",
consecutive_failures: 0,
host: "build.example",
checked_at: "2026-09-17T00:00:00Z",
last_success_at: "2026-09-17T00:00:00Z",
updated_at: "2026-09-17T00:00:00Z",
probing_enabled: true,
probe_interval_seconds: 60,
persisted: true,
...overrides,
};
}
describe("getSSHReachability", () => {
it("GETs the list route", async () => {
fetchSpy.mockResolvedValueOnce(jsonResponse([record()]));
const result = await getSSHReachability();
expect(result).toEqual([record()]);
const { url, init } = lastCall();
expect(url).toBe(`${BASE}/reachability`);
expect(init?.method ?? "GET").toBe("GET");
});
});
describe("getSSHExecutorReachability", () => {
it("GETs the single-executor route", async () => {
fetchSpy.mockResolvedValueOnce(jsonResponse(record()));
const result = await getSSHExecutorReachability("exec-1");
expect(result).toEqual(record());
const { url } = lastCall();
expect(url).toBe(`${BASE}/executors/exec-1/reachability`);
});
it("encodes the executor id", async () => {
fetchSpy.mockResolvedValueOnce(jsonResponse(record({ executor_id: "exec/1" })));
await getSSHExecutorReachability("exec/1");
const { url } = lastCall();
expect(url).toBe(`${BASE}/executors/exec%2F1/reachability`);
});
});
describe("probeSSHExecutorReachability", () => {
it("POSTs the probe-now route", async () => {
fetchSpy.mockResolvedValueOnce(jsonResponse(record()));
const result = await probeSSHExecutorReachability("exec-1");
expect(result).toEqual(record());
const { url, init } = lastCall();
expect(url).toBe(`${BASE}/executors/exec-1/reachability/probe`);
expect(init?.method).toBe("POST");
});
});
+34
View File
@@ -5,6 +5,7 @@ import type {
SSHSession,
SSHAgentReadinessResponse,
SSHProbeShellsResponse,
SSHReachabilityRecord,
} from "@/lib/types/http-ssh";
export async function testSSHConnection(
@@ -72,3 +73,36 @@ export async function probeSSHShells(
},
);
}
export async function getSSHReachability(
options?: ApiRequestOptions,
): Promise<SSHReachabilityRecord[]> {
return fetchJson<SSHReachabilityRecord[]>("/api/v1/ssh/reachability", options);
}
export async function getSSHExecutorReachability(
executorId: string,
options?: ApiRequestOptions,
): Promise<SSHReachabilityRecord> {
return fetchJson<SSHReachabilityRecord>(
`/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability`,
options,
);
}
export async function probeSSHExecutorReachability(
executorId: string,
options?: ApiRequestOptions,
): Promise<SSHReachabilityRecord> {
return fetchJson<SSHReachabilityRecord>(
`/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability/probe`,
{
...options,
init: {
...(options?.init ?? {}),
method: "POST",
headers: { ...(options?.init?.headers ?? {}), "Content-Type": "application/json" },
},
},
);
}
+9
View File
@@ -19,6 +19,7 @@ import type {
import type { SystemHealthResponse } from "@/lib/types/health";
import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime";
import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use";
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
import type { UISliceActions as UIA } from "./slices/ui/types";
import type * as UISliceTypes from "./slices/ui/types";
import type {
@@ -121,6 +122,7 @@ export type AppState = KanbanSlice & {
sleepInhibition: (typeof defaultSettingsState)["sleepInhibition"];
userSettings: (typeof defaultSettingsState)["userSettings"];
agentProfileRecentUse: (typeof defaultSettingsState)["agentProfileRecentUse"];
sshReachability: (typeof defaultSettingsState)["sshReachability"];
// Session slice
messages: (typeof defaultSessionState)["messages"];
@@ -152,6 +154,7 @@ export type AppState = KanbanSlice & {
sessionMode: (typeof defaultSessionRuntimeState)["sessionMode"];
userShells: (typeof defaultSessionRuntimeState)["userShells"];
prepareProgress: (typeof defaultSessionRuntimeState)["prepareProgress"];
launchWarning: (typeof defaultSessionRuntimeState)["launchWarning"];
sessionTodos: (typeof defaultSessionRuntimeState)["sessionTodos"];
agentCapabilities: (typeof defaultSessionRuntimeState)["agentCapabilities"];
sessionModels: (typeof defaultSessionRuntimeState)["sessionModels"];
@@ -282,6 +285,7 @@ export type AppState = KanbanSlice & {
upsertAgentUpdateJob: (job: AgentUpdateJob) => void;
appendAgentUpdateOutput: (agentName: string, jobId: string, chunk: string) => void;
clearAgentUpdateJob: (agentName: string) => void;
setSSHReachability: (record: SSHReachabilityRecord) => void;
setRepositories: (workspaceId: string, repositories: Repository[]) => void;
upsertRepository: (workspaceId: string, repository: Repository) => void;
setRepositoriesLoading: (workspaceId: string, loading: boolean) => void;
@@ -599,6 +603,11 @@ export type AppState = KanbanSlice & {
) => void;
setSessionPollMode: (sessionId: string, mode: SessionPollMode) => void;
setEmbeddedVscodeSupport: (sessionId: string, supported: boolean) => void;
setLaunchWarning: (
sessionId: string,
entry: import("./slices/session-runtime/types").LaunchWarningEntry,
) => void;
clearLaunchWarning: (sessionId: string) => void;
/* prettier-ignore */ setSidebarActiveView: UIA["setSidebarActiveView"];
createSidebarView: UIA["createSidebarView"];
updateSidebarDraft: UIA["updateSidebarDraft"];
+2
View File
@@ -101,6 +101,7 @@ export const defaultState = {
sessionMode: defaultSessionRuntimeState.sessionMode,
userShells: defaultSessionRuntimeState.userShells,
prepareProgress: defaultSessionRuntimeState.prepareProgress,
launchWarning: defaultSessionRuntimeState.launchWarning,
sessionTodos: defaultSessionRuntimeState.sessionTodos,
agentCapabilities: defaultSessionRuntimeState.agentCapabilities,
sessionModels: defaultSessionRuntimeState.sessionModels,
@@ -515,6 +516,7 @@ export function mergeInitialState(initialState?: HydrationState): DefaultState {
sessionMode: { ...defaultState.sessionMode, ...initialState.sessionMode },
userShells: { ...defaultState.userShells, ...initialState.userShells },
prepareProgress: { ...defaultState.prepareProgress, ...initialState.prepareProgress },
launchWarning: { ...defaultState.launchWarning, ...initialState.launchWarning },
sessionTodos: { ...defaultState.sessionTodos, ...initialState.sessionTodos },
agentCapabilities: { ...defaultState.agentCapabilities, ...initialState.agentCapabilities },
sessionModels: { ...defaultState.sessionModels, ...initialState.sessionModels },
@@ -18,6 +18,20 @@ describe("purgeSessionRuntimeState", () => {
store = makeStore();
});
it("clears a launch warning when a relaunch reaches RUNNING", () => {
const s = store.getState();
s.setLaunchWarning(SESSION_ID, {
executorId: "executor-1",
host: "10.0.0.5",
state: "unreachable",
reason: "timeout",
});
s.clearLaunchWarning(SESSION_ID);
expect(store.getState().launchWarning.bySessionId[SESSION_ID]).toBeUndefined();
});
it("drops per-session maps, process output, and env-scoped buffers", () => {
const s = store.getState();
s.registerSessionEnvironment(SESSION_ID, "env-1");
@@ -39,6 +53,12 @@ describe("purgeSessionRuntimeState", () => {
status: "running",
});
s.appendProcessOutput("proc-1", "process noise");
s.setLaunchWarning("session-1", {
executorId: "executor-1",
host: "10.0.0.5",
state: "unreachable",
reason: "timeout",
});
store.setState((draft) => {
purgeSessionRuntimeState(draft, SESSION_ID);
@@ -52,6 +72,7 @@ describe("purgeSessionRuntimeState", () => {
expect(after.processes.devProcessBySessionId[SESSION_ID]).toBeUndefined();
expect(after.processes.processesById["proc-1"]).toBeUndefined();
expect(after.processes.outputsByProcessId["proc-1"]).toBeUndefined();
expect(after.launchWarning.bySessionId["session-1"]).toBeUndefined();
// env-scoped buffers gone because no other session references env-1.
expect(after.shell.outputs["env-1"]).toBeUndefined();
expect(after.shell.statuses["env-1"]).toBeUndefined();
@@ -50,6 +50,7 @@ function purgePerSessionRuntime(state: SessionRuntimeSliceState, sessionId: stri
delete state.promptUsage.bySessionId[sessionId];
delete state.sessionTodos.bySessionId[sessionId];
delete state.prepareProgress.bySessionId[sessionId];
delete state.launchWarning.bySessionId[sessionId];
delete state.sessionPollMode.bySessionId[sessionId];
delete state.embeddedVscodeSupport.bySessionId[sessionId];
}
@@ -121,6 +122,7 @@ export const defaultSessionRuntimeState: SessionRuntimeSliceState = {
sessionTodos: { bySessionId: {} },
userShells: { byEnvironmentId: {}, dismissedByEnvironmentId: {}, loading: {}, loaded: {} },
prepareProgress: { bySessionId: {} },
launchWarning: { bySessionId: {} },
sessionPollMode: { bySessionId: {} },
embeddedVscodeSupport: { bySessionId: {} },
workspaceFilesRefresh: { bySessionId: {} },
@@ -534,5 +536,13 @@ export const createSessionRuntimeSlice: StateCreator<
set((draft) => {
draft.sessionTodos.bySessionId[sessionId] = entries;
}),
setLaunchWarning: (sessionId, entry) =>
set((draft) => {
draft.launchWarning.bySessionId[sessionId] = entry;
}),
clearLaunchWarning: (sessionId) =>
set((draft) => {
delete draft.launchWarning.bySessionId[sessionId];
}),
...buildUserShellActions(set),
});
@@ -445,6 +445,22 @@ export type EmbeddedVscodeSupportState = {
bySessionId: Record<string, boolean>;
};
/**
* Applied verbatim from a session.launch.warning event — no field is ever
* re-derived from the reachability settings store. See launch-warning.tsx.
*/
export type LaunchWarningEntry = {
executorId: string;
host: string;
state: string;
reason: string;
lastSuccessAt?: string;
};
export type LaunchWarningState = {
bySessionId: Record<string, LaunchWarningEntry>;
};
export type SessionRuntimeSliceState = {
terminal: TerminalState;
shell: ShellState;
@@ -465,6 +481,7 @@ export type SessionRuntimeSliceState = {
sessionTodos: SessionTodosState;
userShells: UserShellsState;
prepareProgress: PrepareProgressState;
launchWarning: LaunchWarningState;
sessionPollMode: SessionPollModeState;
embeddedVscodeSupport: EmbeddedVscodeSupportState;
workspaceFilesRefresh: { bySessionId: Record<string, number> };
@@ -558,6 +575,8 @@ export type SessionRuntimeSliceActions = {
completeWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean;
failWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt, details: string) => boolean;
clearWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean;
setLaunchWarning: (sessionId: string, entry: LaunchWarningEntry) => void;
clearLaunchWarning: (sessionId: string) => void;
};
export type SessionRuntimeSlice = SessionRuntimeSliceState & SessionRuntimeSliceActions;
@@ -366,6 +366,107 @@ describe("setAvailableAgents capability propagation", () => {
});
});
describe("ssh reachability", () => {
const EXEC_ID = "exec-1";
const BASE_TIMESTAMP = "2026-09-17T00:00:00Z";
function reachabilityRecord(
overrides: Partial<import("@/lib/types/http-ssh").SSHReachabilityRecord> = {},
): import("@/lib/types/http-ssh").SSHReachabilityRecord {
return {
executor_id: EXEC_ID,
state: "reachable",
reason: "",
consecutive_failures: 0,
host: "build.example",
checked_at: BASE_TIMESTAMP,
last_success_at: BASE_TIMESTAMP,
updated_at: BASE_TIMESTAMP,
probing_enabled: true,
probe_interval_seconds: 60,
persisted: true,
...overrides,
};
}
it("stores a record keyed by executor id", () => {
const store = makeStore();
store.getState().setSSHReachability(reachabilityRecord());
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reachabilityRecord());
});
it("keeps the record with the later updated_at and discards an older one", () => {
const store = makeStore();
const newer = reachabilityRecord({ state: "unreachable", updated_at: "2026-09-17T01:00:00Z" });
const older = reachabilityRecord({ state: "reachable", updated_at: BASE_TIMESTAMP });
store.getState().setSSHReachability(newer);
store.getState().setSSHReachability(older);
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(newer);
});
it("applies a reset (null checked_at, newer updated_at) over a stored reachable/unreachable record", () => {
// A connection-configuration reset clears checked_at but still advances
// updated_at. Reconciling on checked_at would make this reset compare as
// older than the record it just invalidated and get discarded, leaving a
// stale reachable/unreachable state showing for a host the user just
// re-pointed.
const store = makeStore();
const stale = reachabilityRecord({
state: "unreachable",
checked_at: "2026-09-17T02:00:00Z",
updated_at: "2026-09-17T02:00:00Z",
});
const reset = reachabilityRecord({
state: "unknown",
checked_at: null,
updated_at: "2026-09-17T03:00:00Z",
persisted: false,
});
store.getState().setSSHReachability(stale);
store.getState().setSSHReachability(reset);
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reset);
});
it("discards a null-updated_at placeholder applied after a record with a real timestamp", () => {
// The synthesized never-probed placeholder always loses.
const store = makeStore();
const real = reachabilityRecord({ updated_at: BASE_TIMESTAMP });
const placeholder = reachabilityRecord({ state: "unknown", updated_at: null });
store.getState().setSSHReachability(real);
store.getState().setSSHReachability(placeholder);
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(real);
});
it("applies the first record for an executor even when its updated_at is null", () => {
const store = makeStore();
const placeholder = reachabilityRecord({ state: "unknown", updated_at: null });
store.getState().setSSHReachability(placeholder);
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(placeholder);
});
it("tracks distinct executors independently", () => {
const store = makeStore();
const first = reachabilityRecord({ executor_id: EXEC_ID });
const second = reachabilityRecord({ executor_id: "exec-2", state: "unreachable" });
store.getState().setSSHReachability(first);
store.getState().setSSHReachability(second);
expect(store.getState().sshReachability.byExecutorId).toEqual({
[EXEC_ID]: first,
"exec-2": second,
});
});
});
describe("folder opener discovery", () => {
it("defaults unavailable and preserves capability through editor preference updates", () => {
const store = makeStore();
@@ -38,6 +38,7 @@ export const defaultSettingsState: SettingsSliceState = {
sleepInhibition: { response: null, loaded: false, loading: false, error: false },
userSettings: createDefaultUserSettings(),
agentProfileRecentUse: { records: {}, loaded: false },
sshReachability: { byExecutorId: {} },
};
type ImmerSet = Parameters<
@@ -437,6 +438,29 @@ function createSecretAndSpriteActions(
};
}
// reachabilityUpdatedAtMs parses updated_at to epoch ms for reconciliation.
// A null updated_at (the synthesized never-probed placeholder) sorts as
// "always older" so any real record replaces it.
function reachabilityUpdatedAtMs(updatedAt: string | null): number {
return updatedAt ? Date.parse(updatedAt) : -Infinity;
}
function createSSHReachabilityActions(set: ImmerSet): Pick<SettingsSlice, "setSSHReachability"> {
return {
setSSHReachability: (record) =>
set((draft) => {
const current = draft.sshReachability.byExecutorId[record.executor_id];
if (
current &&
reachabilityUpdatedAtMs(current.updated_at) > reachabilityUpdatedAtMs(record.updated_at)
) {
return;
}
draft.sshReachability.byExecutorId[record.executor_id] = record;
}),
};
}
export const createSettingsSlice: StateCreator<
SettingsSlice,
[["zustand/immer", never]],
@@ -450,4 +474,5 @@ export const createSettingsSlice: StateCreator<
...createInstallJobActions(set),
...createAgentUpdateJobActions(set),
...createSecretAndSpriteActions(set),
...createSSHReachabilityActions(set),
});
@@ -34,6 +34,7 @@ import type {
} from "@/lib/agent-profile-recent-use";
import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use";
import type { TaskColor } from "@/lib/task-colors";
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
export type {
AgentProfileRecentUseRecord,
@@ -44,6 +45,15 @@ export type ExecutorsState = {
items: Executor[];
};
/**
* SSH reachability records keyed by executor id. Populated by a per-card
* fetch (task 06's SSHReachabilityCard) and kept live via the
* executor.reachability.changed WS event.
*/
export type SSHReachabilityStoreState = {
byExecutorId: Record<string, SSHReachabilityRecord>;
};
export type SettingsAgentsState = {
items: Agent[];
};
@@ -520,6 +530,7 @@ export type SettingsSliceState = {
sleepInhibition: SleepInhibitionStoreState;
userSettings: UserSettingsState;
agentProfileRecentUse: AgentProfileRecentUseState;
sshReachability: SSHReachabilityStoreState;
};
export type SettingsSliceActions = {
@@ -568,6 +579,16 @@ export type SettingsSliceActions = {
record: AgentProfileRecentUseRecord,
) => void;
bumpAgentProfilesVersion: () => void;
/**
* Applies a reachability record (a fetch response or a pushed
* executor.reachability.changed event). Reconciles on updated_at, never
* checked_at: a connection-configuration reset clears checked_at (null)
* while still advancing updated_at, so comparing on checked_at would make
* the reset compare as older than the record it just invalidated and get
* discarded. A null updated_at (the synthesized never-probed placeholder)
* always loses to a record that has one.
*/
setSSHReachability: (record: SSHReachabilityRecord) => void;
};
export type SettingsSlice = SettingsSliceState & SettingsSliceActions;
+1
View File
@@ -53,6 +53,7 @@ export function buildStateOverrides(m: DefaultState) {
sessionMode: m.sessionMode,
userShells: m.userShells,
prepareProgress: m.prepareProgress,
launchWarning: m.launchWarning,
sessionTodos: m.sessionTodos,
agentCapabilities: m.agentCapabilities,
sessionModels: m.sessionModels,
+8
View File
@@ -44,6 +44,7 @@ import type {
} from "@/lib/types/github";
import type { TaskMR, TaskMRDeletedEvent, TaskMRAutomationOptions } from "@/lib/types/gitlab";
import type { TaskStatusSummary } from "@/lib/types/task-status-summary";
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
import type { AgentProfileRecentUseApiRecord } from "@/lib/types/http-agent-profile-recent-use";
import type { SystemMetricsSnapshot, StorageAnalysisUpdatedPayload } from "./system";
import type { AgentRuntimeAvailability } from "./agent-runtime";
@@ -52,6 +53,7 @@ import type {
ExecutorProfilePayload,
PrepareProgressPayload,
PrepareCompletedPayload,
LaunchWarningPayload,
EnvironmentPayload,
} from "./executor-payloads";
@@ -358,6 +360,7 @@ export {
type ExecutorProfilePayload,
type PrepareProgressPayload,
type PrepareCompletedPayload,
type LaunchWarningPayload,
type EnvironmentPayload,
} from "./executor-payloads";
@@ -552,6 +555,11 @@ export type BackendMessageMap = SessionBackendMessageMap &
"executor.prepare.completed",
PrepareCompletedPayload
>;
"executor.reachability.changed": BackendMessage<
"executor.reachability.changed",
SSHReachabilityRecord
>;
"session.launch.warning": BackendMessage<"session.launch.warning", LaunchWarningPayload>;
"environment.created": BackendMessage<"environment.created", EnvironmentPayload>;
"environment.updated": BackendMessage<"environment.updated", EnvironmentPayload>;
"environment.deleted": BackendMessage<"environment.deleted", EnvironmentPayload>;
+11
View File
@@ -65,6 +65,17 @@ export type PrepareCompletedPayload = {
timestamp: string;
};
export type LaunchWarningPayload = {
task_id: string;
session_id: string;
executor_id: string;
host: string;
state: string;
reason: string;
last_success_at?: string;
timestamp: string;
};
export type EnvironmentPayload = {
id: string;
name: string;
+29
View File
@@ -70,3 +70,32 @@ export interface SSHProbeShellsResponse {
duration_ms: number;
available: string[];
}
export type SSHReachabilityState = "unknown" | "reachable" | "unreachable";
export type SSHReachabilityReason =
| ""
| "config"
| "timeout"
| "host_key"
| "auth"
| "network"
| "unknown";
/** Mirrors the backend's reachability.RecordDTO wire shape. */
export interface SSHReachabilityRecord {
executor_id: string;
state: SSHReachabilityState;
reason: SSHReachabilityReason;
message?: string;
consecutive_failures: number;
host?: string;
checked_at: string | null;
last_success_at: string | null;
/** Null on the synthesized "never probed" placeholder. */
updated_at: string | null;
probing_enabled: boolean;
/** The effective, clamped probe interval in seconds (0 when disabled). */
probe_interval_seconds: number;
persisted: boolean;
}
@@ -33,6 +33,7 @@ function makeStore(overrides: Record<string, unknown> = {}) {
setActiveSessionAuto: vi.fn(),
setSessionAgentctlStatus: vi.fn(),
setResumeSkipped: vi.fn(),
clearLaunchWarning: vi.fn(),
setSessionFailureNotification: vi.fn(),
setContextWindow: vi.fn(),
clearContextWindow: vi.fn(),
@@ -2079,6 +2080,7 @@ describe("session.state_changed resume-skipped clearing", () => {
it("clears the resume-skipped marker on a RUNNING transition", () => {
handler(makeMessage({ task_id: "t-1", session_id: "s-1", new_state: "RUNNING" }));
expect(store.getState().setResumeSkipped).toHaveBeenCalledWith("s-1", false);
expect(store.getState().clearLaunchWarning).toHaveBeenCalledWith("s-1");
});
it("does NOT clear the marker on a STARTING transition (failed resumes keep the retry button)", () => {
+7 -1
View File
@@ -853,6 +853,12 @@ function handleQueueStatusChangedMessage(
state.setQueueEntries(payload.session_id, entries, meta);
}
function clearResumeAndLaunchWarnings(store: StoreApi<AppState>, sessionId: string) {
const state = store.getState();
state.setResumeSkipped(sessionId, false);
state.clearLaunchWarning?.(sessionId);
}
/** Registers the task-session WebSocket handlers (state, messages, workspace sources, queue). */
// eslint-disable-next-line max-lines-per-function -- session events remain one ordered registry.
export function registerTaskSessionHandlers(store: StoreApi<AppState>): WsHandlers {
@@ -912,7 +918,7 @@ export function registerTaskSessionHandlers(store: StoreApi<AppState>): WsHandle
// it: a failed manual resume emits STARTING before the launch fails,
// and clearing there would drop the Start agent retry affordance.
if (newState === "RUNNING") {
store.getState().setResumeSkipped(sessionId, false);
clearResumeAndLaunchWarnings(store, sessionId);
}
maybeAdoptSessionOnTransition(
+3
View File
@@ -42,5 +42,8 @@ export function registerExecutorsHandlers(store: StoreApi<AppState>): WsHandlers
},
}));
},
"executor.reachability.changed": (message) => {
store.getState().setSSHReachability(message.payload);
},
};
}
@@ -0,0 +1,89 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { StoreApi } from "zustand";
import type { AppState } from "@/lib/state/store";
import type { BackendMessageMap, LaunchWarningPayload } from "@/lib/types/backend";
import { registerSSHLaunchWarningHandlers } from "./ssh-launch-warning";
const SESSION_ID = "session-1";
const TASK_ID = "task-1";
function makeStore() {
const state = {
launchWarning: { bySessionId: {} as Record<string, unknown> },
setLaunchWarning: vi.fn(),
} as unknown as AppState;
return {
getState: () => state,
setState: vi.fn(),
subscribe: vi.fn(),
destroy: vi.fn(),
getInitialState: vi.fn(),
} as unknown as StoreApi<AppState>;
}
function makePayload(overrides: Partial<LaunchWarningPayload> = {}): LaunchWarningPayload {
return {
task_id: TASK_ID,
session_id: SESSION_ID,
executor_id: "executor-1",
host: "10.0.0.5",
state: "unreachable",
reason: "timeout",
last_success_at: "2026-06-10T00:00:00.000Z",
timestamp: "2026-06-11T00:00:00.000Z",
...overrides,
};
}
function makeMessage(payload: LaunchWarningPayload): BackendMessageMap["session.launch.warning"] {
return {
id: "message-1",
type: "notification",
action: "session.launch.warning",
payload,
timestamp: "2026-06-11T00:00:00.000Z",
};
}
describe("session.launch.warning handler", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("stores a normalized entry keyed by session id", () => {
const store = makeStore();
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
handler(makeMessage(makePayload()));
expect(store.getState().setLaunchWarning).toHaveBeenCalledWith(SESSION_ID, {
executorId: "executor-1",
host: "10.0.0.5",
state: "unreachable",
reason: "timeout",
lastSuccessAt: "2026-06-10T00:00:00.000Z",
});
});
it("omits lastSuccessAt when the record has never succeeded", () => {
const store = makeStore();
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
handler(makeMessage(makePayload({ last_success_at: undefined })));
expect(store.getState().setLaunchWarning).toHaveBeenCalledWith(
SESSION_ID,
expect.objectContaining({ lastSuccessAt: undefined }),
);
});
it("ignores payloads without a session id", () => {
const store = makeStore();
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
handler(makeMessage(makePayload({ session_id: "" })));
expect(store.getState().setLaunchWarning).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,20 @@
import type { StoreApi } from "zustand";
import type { AppState } from "@/lib/state/store";
import type { WsHandlers } from "@/lib/ws/handlers/types";
import type { LaunchWarningPayload } from "@/lib/types/backend";
export function registerSSHLaunchWarningHandlers(store: StoreApi<AppState>): WsHandlers {
return {
"session.launch.warning": (message) => {
const payload = message.payload as LaunchWarningPayload;
if (!payload.session_id) return;
store.getState().setLaunchWarning(payload.session_id, {
executorId: payload.executor_id,
host: payload.host,
state: payload.state,
reason: payload.reason,
lastSuccessAt: payload.last_success_at,
});
},
};
}
+2
View File
@@ -20,6 +20,7 @@ import { registerDiffsHandlers } from "@/lib/ws/handlers/diffs";
import { registerExecutorsHandlers } from "@/lib/ws/handlers/executors";
import { registerExecutorProfileHandlers } from "@/lib/ws/handlers/executor-profiles";
import { registerExecutorPrepareHandlers } from "@/lib/ws/handlers/executor-prepare";
import { registerSSHLaunchWarningHandlers } from "@/lib/ws/handlers/ssh-launch-warning";
import { registerGitStatusHandlers } from "@/lib/ws/handlers/git-status";
import { registerKanbanHandlers } from "@/lib/ws/handlers/kanban";
import { registerSystemEventsHandlers } from "@/lib/ws/handlers/system-events";
@@ -59,6 +60,7 @@ export function registerWsHandlers(store: StoreApi<AppState>) {
...registerExecutorsHandlers(store),
...registerExecutorProfileHandlers(store),
...registerExecutorPrepareHandlers(store),
...registerSSHLaunchWarningHandlers(store),
...registerAgentsHandlers(store),
...registerTaskSessionHandlers(store),
...registerAvailableCommandsHandlers(store),
+22
View File
@@ -386,6 +386,28 @@
"sshProbeHint": "Click \"{{button}}\" to check which agents are installed on the remote.",
"sshProxyJumpHint": "Single bastion hop. Chained jumps are not supported.",
"sshProxyJumpLabel": "ProxyJump (optional)",
"sshReachabilityAgeLastProbe": "Last probe: {{age}}",
"sshReachabilityAgeLastSuccess": "Last success: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} consecutive failure",
"sshReachabilityConsecutiveFailures_other": "{{count}} consecutive failures",
"sshReachabilityDescription": "The most recent SSH probe result for this host, checked automatically in the background.",
"sshReachabilityHost": "Probed host: {{host}}",
"sshReachabilityLoadFailed": "Reachability status could not be loaded. It is not known.",
"sshReachabilityNeverProbed": "Not probed yet.",
"sshReachabilityNoSuccess": "No successful probe recorded.",
"sshReachabilityProbeNow": "Probe now",
"sshReachabilityProbingOff": "Periodic probing is off. This shows the last recorded result, not the current state.",
"sshReachabilityReasonAuth": "Authentication failed",
"sshReachabilityReasonConfig": "The connection configuration is invalid",
"sshReachabilityReasonHostKey": "The host key does not match the pinned fingerprint",
"sshReachabilityReasonNetwork": "A network error occurred",
"sshReachabilityReasonTimeout": "The connection attempt timed out",
"sshReachabilityReasonUnknown": "An unknown error occurred",
"sshReachabilityStale": "Stale: no probe within the last three check intervals",
"sshReachabilityStateReachable": "Reachable",
"sshReachabilityStateUnknown": "Unknown",
"sshReachabilityStateUnreachable": "Unreachable",
"sshReachabilityTitle": "Reachability",
"sshReclaimTaskDirDescription": "Every task that runs on {{host}} keeps its checkout under {{path}} after the task ends, so a resumed or unarchived task finds its work where it left it.",
"sshReclaimTaskDirLabel": "Delete a task's directory when the task is archived or deleted",
"sshReclaimTaskDirTitle": "Remote task directories",
+10
View File
@@ -2245,6 +2245,16 @@
"launchErrorSessionPreview": "The task launch needs attention.",
"launchNeedsAttention": "Launch needs attention",
"launchErrorNoChanges": "No task or branch settings were changed.",
"launchWarningTitle": "SSH host may be unreachable",
"launchWarningHost": "Probed host: {{host}}",
"launchWarningLastSuccess": "Last success: {{age}}",
"launchWarningNeverSucceeded": "No successful probe has ever been recorded for this host.",
"launchWarningReasonConfig": "The last attempt failed because the connection configuration is invalid.",
"launchWarningReasonTimeout": "The last attempt timed out.",
"launchWarningReasonHostKey": "The last attempt failed because the host key did not match the pinned fingerprint.",
"launchWarningReasonAuth": "The last attempt failed authentication.",
"launchWarningReasonNetwork": "The last attempt failed with a network error.",
"launchWarningReasonUnknown": "The last attempt failed for an unknown reason.",
"launchPromptPreview": "Launch prompt preview",
"retryWithDefaultBranch": "Retry with default branch",
"pickBaseBranch": "Pick a base branch",
@@ -386,6 +386,28 @@
"sshProbeHint": "Ćĺĩćķ \"{{button}}\" ţō ćĥēćķ ŵĥĩćĥ àĝēńţś àŕē ĩńśţàĺĺēď ōń ţĥē ŕēḿōţē.",
"sshProxyJumpHint": "Śĩńĝĺē ƀàśţĩōń ĥōƥ. Ćĥàĩńēď ĵũḿƥś àŕē ńōţ śũƥƥōŕţēď.",
"sshProxyJumpLabel": "ƤŕōxŷĴũḿƥ (ōƥţĩōńàĺ)",
"sshReachabilityAgeLastProbe": "Ĺàśţ ƥŕōƀē: {{age}}",
"sshReachabilityAgeLastSuccess": "Ĺàśţ śũććēśś: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕē",
"sshReachabilityConsecutiveFailures_other": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕēś",
"sshReachabilityDescription": "Ţĥē ḿōśţ ŕēćēńţ ŚŚĤ ƥŕōƀē ŕēśũĺţ ƒōŕ ţĥĩś ĥōśţ, ćĥēćķēď àũţōḿàţĩćàĺĺŷ ĩń ţĥē ƀàćķĝŕōũńď.",
"sshReachabilityHost": "Ƥŕōƀēď ĥōśţ: {{host}}",
"sshReachabilityLoadFailed": "Ŕēàćĥàƀĩĺĩţŷ śţàţũś ćōũĺď ńōţ ƀē ĺōàďēď. Ĩţ ĩś ńōţ ķńōŵń.",
"sshReachabilityNeverProbed": "Ńōţ ƥŕōƀēď ŷēţ.",
"sshReachabilityNoSuccess": "Ńō śũććēśśƒũĺ ƥŕōƀē ŕēćōŕďēď.",
"sshReachabilityProbeNow": "Ƥŕōƀē ńōŵ",
"sshReachabilityProbingOff": "Ƥēŕĩōďĩć ƥŕōƀĩńĝ ĩś ōƒƒ. Ţĥĩś śĥōŵś ţĥē ĺàśţ ŕēćōŕďēď ŕēśũĺţ, ńōţ ţĥē ćũŕŕēńţ śţàţē.",
"sshReachabilityReasonAuth": "Àũţĥēńţĩćàţĩōń ƒàĩĺēď",
"sshReachabilityReasonConfig": "Ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď",
"sshReachabilityReasonHostKey": "Ţĥē ĥōśţ ķēŷ ďōēś ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ",
"sshReachabilityReasonNetwork": "À ńēţŵōŕķ ēŕŕōŕ ōććũŕŕēď",
"sshReachabilityReasonTimeout": "Ţĥē ćōńńēćţĩōń àţţēḿƥţ ţĩḿēď ōũţ",
"sshReachabilityReasonUnknown": "Àń ũńķńōŵń ēŕŕōŕ ōććũŕŕēď",
"sshReachabilityStale": "Śţàĺē: ńō ƥŕōƀē ŵĩţĥĩń ţĥē ĺàśţ ţĥŕēē ćĥēćķ ĩńţēŕvàĺś",
"sshReachabilityStateReachable": "Ŕēàćĥàƀĺē",
"sshReachabilityStateUnknown": "Ũńķńōŵń",
"sshReachabilityStateUnreachable": "Ũńŕēàćĥàƀĺē",
"sshReachabilityTitle": "Ŕēàćĥàƀĩĺĩţŷ",
"sshReclaimTaskDirDescription": "Ēvēŕŷ ţàśķ ţĥàţ ŕũńś ōń {{host}} ķēēƥś ĩţś ćĥēćķōũţ ũńďēŕ {{path}} àƒţēŕ ţĥē ţàśķ ēńďś, śō à ŕēśũḿēď ōŕ ũńàŕćĥĩvēď ţàśķ ƒĩńďś ĩţś ŵōŕķ ŵĥēŕē ĩţ ĺēƒţ ĩţ.",
"sshReclaimTaskDirLabel": "Ďēĺēţē à ţàśķ'ś ďĩŕēćţōŕŷ ŵĥēń ţĥē ţàśķ ĩś àŕćĥĩvēď ōŕ ďēĺēţēď",
"sshReclaimTaskDirTitle": "Ŕēḿōţē ţàśķ ďĩŕēćţōŕĩēś",
+10
View File
@@ -2245,6 +2245,16 @@
"launchErrorSessionPreview": "Ţĥē ţàśķ ĺàũńćĥ ńēēďś àţţēńţĩōń.",
"launchNeedsAttention": "Ĺàũńćĥ ńēēďś àţţēńţĩōń",
"launchErrorNoChanges": "Ńō ţàśķ ōŕ ƀŕàńćĥ śēţţĩńĝś ŵēŕē ćĥàńĝēď.",
"launchWarningTitle": "ŚŚĤ ĥōśţ ḿàŷ ƀē ũńŕēàćĥàƀĺē",
"launchWarningHost": "Ƥŕōƀēď ĥōśţ: {{host}}",
"launchWarningLastSuccess": "Ĺàśţ śũććēśś: {{age}}",
"launchWarningNeverSucceeded": "Ńō śũććēśśƒũĺ ƥŕōƀē ĥàś ēvēŕ ƀēēń ŕēćōŕďēď ƒōŕ ţĥĩś ĥōśţ.",
"launchWarningReasonConfig": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď.",
"launchWarningReasonTimeout": "Ţĥē ĺàśţ àţţēḿƥţ ţĩḿēď ōũţ.",
"launchWarningReasonHostKey": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ĥōśţ ķēŷ ďĩď ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ.",
"launchWarningReasonAuth": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď àũţĥēńţĩćàţĩōń.",
"launchWarningReasonNetwork": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ŵĩţĥ à ńēţŵōŕķ ēŕŕōŕ.",
"launchWarningReasonUnknown": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƒōŕ àń ũńķńōŵń ŕēàśōń.",
"launchPromptPreview": "Ĺàũńćĥ ƥŕōḿƥţ ƥŕēvĩēŵ",
"retryWithDefaultBranch": "Ŕēţŕŷ ŵĩţĥ ďēƒàũĺţ ƀŕàńćĥ",
"pickBaseBranch": "Ƥĩćķ à ƀàśē ƀŕàńćĥ",
+22
View File
@@ -370,6 +370,28 @@
"sshProbeHint": "Clique em \"{{button}}\" para verificar que agentes estão instalados no anfitrião remoto.",
"sshProxyJumpHint": "Um único salto por bastião. Saltos encadeados não são suportados.",
"sshProxyJumpLabel": "ProxyJump (opcional)",
"sshReachabilityAgeLastProbe": "Última verificação: {{age}}",
"sshReachabilityAgeLastSuccess": "Último sucesso: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} falha consecutiva",
"sshReachabilityConsecutiveFailures_other": "{{count}} falhas consecutivas",
"sshReachabilityDescription": "O resultado mais recente da verificação SSH deste anfitrião, verificado automaticamente em segundo plano.",
"sshReachabilityHost": "Anfitrião verificado: {{host}}",
"sshReachabilityLoadFailed": "Não foi possível carregar o estado de alcançabilidade. Não é conhecido.",
"sshReachabilityNeverProbed": "Ainda não verificado.",
"sshReachabilityNoSuccess": "Nenhuma verificação bem-sucedida registada.",
"sshReachabilityProbeNow": "Verificar agora",
"sshReachabilityProbingOff": "A verificação periódica está desativada. Isto mostra o último resultado registado, não o estado atual.",
"sshReachabilityReasonAuth": "Falha na autenticação",
"sshReachabilityReasonConfig": "A configuração da ligação é inválida",
"sshReachabilityReasonHostKey": "A chave do anfitrião não corresponde à impressão digital fixada",
"sshReachabilityReasonNetwork": "Ocorreu um erro de rede",
"sshReachabilityReasonTimeout": "A tentativa de ligação expirou",
"sshReachabilityReasonUnknown": "Ocorreu um erro desconhecido",
"sshReachabilityStale": "Desatualizado: sem verificação nos últimos três intervalos",
"sshReachabilityStateReachable": "Alcançável",
"sshReachabilityStateUnknown": "Desconhecido",
"sshReachabilityStateUnreachable": "Inalcançável",
"sshReachabilityTitle": "Alcançabilidade",
"sshReclaimTaskDirDescription": "Cada tarefa executada em {{host}} mantém a sua cópia de trabalho em {{path}} depois de terminar, para que uma tarefa retomada ou desarquivada encontre o trabalho onde o deixou.",
"sshReclaimTaskDirLabel": "Eliminar o diretório de uma tarefa quando a tarefa for arquivada ou eliminada",
"sshReclaimTaskDirTitle": "Diretórios de tarefas remotos",
+10
View File
@@ -2245,6 +2245,16 @@
"launchErrorSessionPreview": "O início da tarefa requer atenção.",
"launchNeedsAttention": "O início requer atenção",
"launchErrorNoChanges": "As definições da tarefa e do branch não foram alteradas.",
"launchWarningTitle": "O anfitrião SSH pode estar inacessível",
"launchWarningHost": "Anfitrião testado: {{host}}",
"launchWarningLastSuccess": "Último sucesso: {{age}}",
"launchWarningNeverSucceeded": "Nunca foi registada uma sondagem bem-sucedida para este anfitrião.",
"launchWarningReasonConfig": "A última tentativa falhou porque a configuração da ligação é inválida.",
"launchWarningReasonTimeout": "A última tentativa expirou.",
"launchWarningReasonHostKey": "A última tentativa falhou porque a chave do anfitrião não correspondeu à impressão digital fixada.",
"launchWarningReasonAuth": "A última tentativa falhou na autenticação.",
"launchWarningReasonNetwork": "A última tentativa falhou com um erro de rede.",
"launchWarningReasonUnknown": "A última tentativa falhou por um motivo desconhecido.",
"launchPromptPreview": "Pré-visualização do prompt de início",
"retryWithDefaultBranch": "Tentar novamente com o branch predefinido",
"pickBaseBranch": "Escolher um branch base",

Some files were not shown because too many files have changed in this diff Show More