feat(executors): add continuous SSH host reachability probing (#3747)
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
@@ -256,7 +256,11 @@ func (r *SSHExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateReq
|
||||
}
|
||||
client, err := dialSSH(baseCtx, target)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh: connect to %s@%s: %w", target.User, target.Host, err)
|
||||
// Classified from this launch's own attempt, never from a stored
|
||||
// reachability record — that record can predate this dial by a full
|
||||
// probe interval and would misattribute a stale cause to a fresh
|
||||
// failure (see task-05-launch-non-gating.md).
|
||||
return nil, fmt.Errorf("ssh: connect to %s@%s: %s: %w", target.User, target.Host, ClassifyDialError(err), err)
|
||||
}
|
||||
var runtimeAPITunnel *sshRuntimeAPITunnel
|
||||
released := false
|
||||
|
||||
@@ -495,7 +495,7 @@ func dialDirect(ctx context.Context, addr string, cfg *ssh.ClientConfig) (*ssh.C
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ssh: tcp dial %s: %w", addr, err)
|
||||
}
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(tcpConn, addr, cfg)
|
||||
sshConn, chans, reqs, err := handshakeWithDeadline(ctx, tcpConn, addr, cfg)
|
||||
if err != nil {
|
||||
_ = tcpConn.Close()
|
||||
return nil, fmt.Errorf("ssh: handshake with %s: %w", addr, err)
|
||||
@@ -530,6 +530,37 @@ func (c *sshProxyJumpConn) Close() error {
|
||||
return c.closeErr
|
||||
}
|
||||
|
||||
// handshakeWithDeadline runs the SSH handshake over conn, bounding it by
|
||||
// ctx's deadline when one is set. ssh.NewClientConn itself takes no context —
|
||||
// without this, a peer that completes the TCP handshake but never speaks SSH
|
||||
// (or stalls partway through key exchange) blocks the caller indefinitely.
|
||||
// Callers that pass a context with no deadline see unchanged behavior.
|
||||
func handshakeWithDeadline(ctx context.Context, conn net.Conn, addr string, cfg *ssh.ClientConfig) (ssh.Conn, <-chan ssh.NewChannel, <-chan *ssh.Request, error) {
|
||||
cancelDone := make(chan struct{})
|
||||
stopCancellation := context.AfterFunc(ctx, func() {
|
||||
// ProxyJump tunnels wrap an x/crypto/ssh channel. That channel
|
||||
// reports unsupported deadlines, so closing the owner transport is
|
||||
// the only reliable way to unblock NewClientConn on cancellation.
|
||||
_ = conn.Close()
|
||||
close(cancelDone)
|
||||
})
|
||||
defer func() {
|
||||
if !stopCancellation() {
|
||||
<-cancelDone
|
||||
}
|
||||
}()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(deadline)
|
||||
defer func() { _ = conn.SetDeadline(time.Time{}) }()
|
||||
}
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg)
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
_ = conn.Close()
|
||||
return nil, nil, nil, ctxErr
|
||||
}
|
||||
return sshConn, chans, reqs, err
|
||||
}
|
||||
|
||||
// dialViaJump implements ProxyJump as a single bastion hop. The bastion is
|
||||
// resolved from its own ~/.ssh/config Host block, defaulting to the same
|
||||
// identity source as the target (passes the user's agent / key through).
|
||||
@@ -568,7 +599,7 @@ func dialViaJump(ctx context.Context, target *SSHTarget, finalAddr string, final
|
||||
return nil, fmt.Errorf("ssh: bastion tunnel to %s: %w", finalAddr, err)
|
||||
}
|
||||
jumpConn := &sshProxyJumpConn{Conn: tunnel, bastion: bastionClient}
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(jumpConn, finalAddr, finalCfg)
|
||||
sshConn, chans, reqs, err := handshakeWithDeadline(ctx, jumpConn, finalAddr, finalCfg)
|
||||
if err != nil {
|
||||
_ = jumpConn.Close()
|
||||
return nil, fmt.Errorf("ssh: handshake with %s via %s: %w", finalAddr, target.ProxyJump, err)
|
||||
|
||||
@@ -2,6 +2,8 @@ package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -17,6 +19,39 @@ import (
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type unsupportedDeadlineConn struct{ net.Conn }
|
||||
|
||||
func (unsupportedDeadlineConn) SetDeadline(time.Time) error {
|
||||
return errors.New("deadlines are not supported")
|
||||
}
|
||||
|
||||
func TestHandshakeWithDeadlineClosesConnectionsWhenDeadlinesAreUnsupported(t *testing.T) {
|
||||
clientConn, serverConn := net.Pipe()
|
||||
defer func() { _ = serverConn.Close() }()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := handshakeWithDeadline(ctx, unsupportedDeadlineConn{Conn: clientConn}, "test", &ssh.ClientConfig{
|
||||
User: "test",
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("handshake error = %v, want context deadline exceeded", err)
|
||||
}
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
_ = clientConn.Close()
|
||||
<-done
|
||||
t.Fatal("handshake did not stop when its context deadline expired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHControlRequestUsesBearerToken(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPost, "http://127.0.0.1/api/v1/instances", nil)
|
||||
if err != nil {
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2
|
||||
//
|
||||
// A launch's own dial failure must be attributed with the target host and
|
||||
// the same reason ClassifyDialError would produce for a standalone probe —
|
||||
// not read from any stored record, since the stored record can predate this
|
||||
// attempt by a full interval.
|
||||
func TestSSHExecutorCreateInstanceDialErrorNamesHostAndReason(t *testing.T) {
|
||||
server := newFakeSSHServer(t, nil)
|
||||
newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
|
||||
exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger())
|
||||
t.Cleanup(func() { _ = exec.Close() })
|
||||
|
||||
_, err := exec.CreateInstance(context.Background(), &ExecutorCreateRequest{
|
||||
InstanceID: "instance-1",
|
||||
Metadata: map[string]interface{}{
|
||||
MetadataKeySSHHost: "127.0.0.1",
|
||||
MetadataKeySSHPort: strconv.Itoa(server.port()),
|
||||
MetadataKeySSHUser: "kandev",
|
||||
MetadataKeySSHHostFingerprint: "SHA256:not-the-server-key",
|
||||
MetadataKeySSHIdentitySource: string(SSHIdentitySourceFile),
|
||||
MetadataKeySSHIdentityFile: identity,
|
||||
},
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected a dial failure, got nil error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "127.0.0.1") {
|
||||
t.Fatalf("error = %q, want it to name the target host", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), string(SSHReachabilityReasonHostKey)) {
|
||||
t.Fatalf("error = %q, want it to carry the classified reason %q", err.Error(), SSHReachabilityReasonHostKey)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
|
||||
)
|
||||
|
||||
const maxSSHReachabilityMessageBytes = 512
|
||||
|
||||
// SanitizeSSHReachabilityMessage keeps dial diagnostics useful while making
|
||||
// sure credentials, local paths, and oversized provider text do not cross the
|
||||
// reachability API boundary.
|
||||
func SanitizeSSHReachabilityMessage(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
message := routingerr.Sanitize(err.Error())
|
||||
if len(message) <= maxSSHReachabilityMessageBytes {
|
||||
return message
|
||||
}
|
||||
cut := maxSSHReachabilityMessageBytes - len("…")
|
||||
for cut > 0 && !utf8.RuneStart(message[cut]) {
|
||||
cut--
|
||||
}
|
||||
return message[:cut] + "…"
|
||||
}
|
||||
|
||||
// SSHReachabilityReason classifies why a probe or launch-time SSH dial
|
||||
// failed. The set is closed: ClassifyDialError always returns one of these
|
||||
// six values, walking a total order so a deadline is never reported as a
|
||||
// network failure and a rejected host key is never reported as an
|
||||
// authentication failure.
|
||||
type SSHReachabilityReason string
|
||||
|
||||
const (
|
||||
SSHReachabilityReasonConfig SSHReachabilityReason = "config"
|
||||
SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout"
|
||||
SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key"
|
||||
SSHReachabilityReasonAuth SSHReachabilityReason = "auth"
|
||||
SSHReachabilityReasonNetwork SSHReachabilityReason = "network"
|
||||
SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown"
|
||||
)
|
||||
|
||||
// SSHProbeOutcome is the result of a single reachability probe, or of a
|
||||
// launch's own dial attempt classified through the same rules. Host always
|
||||
// names the target executor's host, even when a failure occurred on a
|
||||
// bastion hop. Cancelled means the caller's context ended the attempt before
|
||||
// it produced an observation about the host; callers must discard a
|
||||
// cancelled outcome rather than recording it as a failure.
|
||||
type SSHProbeOutcome struct {
|
||||
Success bool
|
||||
Cancelled bool
|
||||
Host string
|
||||
Reason SSHReachabilityReason
|
||||
Message string
|
||||
}
|
||||
|
||||
// ProbeSSHHost opens an authenticated SSH transport to target using its
|
||||
// pinned fingerprint, closes it without running a remote command, and
|
||||
// returns a typed outcome. It bounds both the TCP dial and the SSH handshake
|
||||
// by timeout, and never dials when the target has no pinned fingerprint to
|
||||
// verify against.
|
||||
func ProbeSSHHost(ctx context.Context, target *SSHTarget, timeout time.Duration) SSHProbeOutcome {
|
||||
outcome := SSHProbeOutcome{Host: target.Host}
|
||||
if strings.TrimSpace(target.PinnedFingerprint) == "" {
|
||||
outcome.Reason = SSHReachabilityReasonConfig
|
||||
outcome.Message = "no pinned host key fingerprint is configured"
|
||||
return outcome
|
||||
}
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
client, err := dialSSH(probeCtx, target)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
outcome.Cancelled = true
|
||||
return outcome
|
||||
}
|
||||
outcome.Reason = ClassifyDialError(err)
|
||||
outcome.Message = SanitizeSSHReachabilityMessage(err)
|
||||
return outcome
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
|
||||
outcome.Success = true
|
||||
return outcome
|
||||
}
|
||||
|
||||
// ClassifyDialError maps a non-nil SSH dial error to exactly one reason from
|
||||
// the closed set, walking timeout, host_key, auth, network, unknown in that
|
||||
// order — first match wins. context.Canceled is deliberately not classified
|
||||
// here: it is not an observation about the host, and callers check for it
|
||||
// (see ProbeSSHHost) before reaching this function.
|
||||
func ClassifyDialError(err error) SSHReachabilityReason {
|
||||
switch {
|
||||
case isSSHTimeoutError(err):
|
||||
return SSHReachabilityReasonTimeout
|
||||
case isSSHHostKeyMismatch(err):
|
||||
return SSHReachabilityReasonHostKey
|
||||
case isSSHAuthFailure(err):
|
||||
return SSHReachabilityReasonAuth
|
||||
case isSSHNetworkError(err):
|
||||
return SSHReachabilityReasonNetwork
|
||||
default:
|
||||
return SSHReachabilityReasonUnknown
|
||||
}
|
||||
}
|
||||
|
||||
func isSSHTimeoutError(err error) bool {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
return errors.As(err, &netErr) && netErr.Timeout()
|
||||
}
|
||||
|
||||
// isSSHHostKeyMismatch confines the host_key discriminator to Kandev's own
|
||||
// errHostKeyMismatch (the pinned-fingerprint check) and knownhosts.KeyError
|
||||
// with a non-empty Want (a genuine mismatch, not an unknown-host TOFU
|
||||
// accept, which returns nil and never reaches this function) — the bastion
|
||||
// host-key check. golang.org/x/crypto/ssh has no stable exported error type
|
||||
// for a rejected key; keying off our own types keeps a library upgrade from
|
||||
// turning a security event into auth or unknown.
|
||||
func isSSHHostKeyMismatch(err error) bool {
|
||||
var mismatch *errHostKeyMismatch
|
||||
if errors.As(err, &mismatch) {
|
||||
return true
|
||||
}
|
||||
var keyErr *knownhosts.KeyError
|
||||
return errors.As(err, &keyErr) && len(keyErr.Want) > 0
|
||||
}
|
||||
|
||||
// isSSHAuthFailure relies on string-matching because the pinned
|
||||
// golang.org/x/crypto/ssh v0.57.0 dependency has no exported error type for
|
||||
// an authentication failure — it returns a plain fmt.Errorf. Revisit this
|
||||
// classifier when upgrading the dependency if it adds a stable type.
|
||||
func isSSHAuthFailure(err error) bool {
|
||||
return strings.Contains(err.Error(), "unable to authenticate")
|
||||
}
|
||||
|
||||
func isSSHNetworkError(err error) bool {
|
||||
var opErr *net.OpError
|
||||
return errors.As(err, &opErr)
|
||||
}
|
||||
|
||||
// SSHTargetFromExecutorConfig projects an SSH executor's persisted config map
|
||||
// into a resolved SSHTarget, applying the same ~/.ssh/config inheritance as
|
||||
// ResolveSSHTarget. A missing host or fingerprint fails resolution here,
|
||||
// which callers should classify as SSHReachabilityReasonConfig without
|
||||
// attempting a dial.
|
||||
func SSHTargetFromExecutorConfig(cfg map[string]string) (*SSHTarget, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("ssh executor has no config")
|
||||
}
|
||||
fingerprint := strings.TrimSpace(cfg["ssh_host_fingerprint"])
|
||||
if fingerprint == "" {
|
||||
return nil, fmt.Errorf("ssh host fingerprint is required for reachability probes")
|
||||
}
|
||||
port := 0
|
||||
if p := strings.TrimSpace(cfg["ssh_port"]); p != "" {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return nil, fmt.Errorf("invalid ssh_port %q", p)
|
||||
}
|
||||
port = n
|
||||
}
|
||||
return ResolveSSHTarget(SSHConnConfig{
|
||||
HostAlias: cfg["ssh_host_alias"],
|
||||
Host: cfg["ssh_host"],
|
||||
Port: port,
|
||||
User: cfg["ssh_user"],
|
||||
IdentitySource: SSHIdentitySource(cfg["ssh_identity_source"]),
|
||||
IdentityFile: cfg["ssh_identity_file"],
|
||||
ProxyJump: cfg["ssh_proxy_jump"],
|
||||
PinnedFingerprint: fingerprint,
|
||||
})
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
||||
// fakeTimeoutError is a minimal net.Error whose Timeout() reports true,
|
||||
// wrapped inside a *net.OpError below to model a timeout that is also
|
||||
// structurally shaped like a network dial error — @covers
|
||||
// AC-EXECUTORS-SSH-REACHABILITY-001.27 (timeout must win the ordering, not
|
||||
// fall through to network).
|
||||
type fakeTimeoutError struct{}
|
||||
|
||||
func (fakeTimeoutError) Error() string { return "fake timeout" }
|
||||
func (fakeTimeoutError) Timeout() bool { return true }
|
||||
func (fakeTimeoutError) Temporary() bool { return true }
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.27 — classification order
|
||||
// config, timeout, host_key, auth, network, unknown, first match wins,
|
||||
// including the two explicit ordering cases.
|
||||
func TestClassifyDialError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want SSHReachabilityReason
|
||||
}{
|
||||
{
|
||||
name: "a context deadline classifies as timeout",
|
||||
err: fmt.Errorf("ssh: tcp dial x: %w", context.DeadlineExceeded),
|
||||
want: SSHReachabilityReasonTimeout,
|
||||
},
|
||||
{
|
||||
name: "a timeout-shaped dial error wins over the network fallback",
|
||||
err: &net.OpError{Op: "dial", Net: "tcp", Err: fakeTimeoutError{}},
|
||||
want: SSHReachabilityReasonTimeout,
|
||||
},
|
||||
{
|
||||
name: "a pinned host key mismatch classifies as host_key",
|
||||
err: fmt.Errorf("ssh: handshake with x: %w", &errHostKeyMismatch{Expected: "A", Got: "B"}),
|
||||
want: SSHReachabilityReasonHostKey,
|
||||
},
|
||||
{
|
||||
name: "a bastion known_hosts mismatch classifies as host_key",
|
||||
err: fmt.Errorf("ssh: bastion dial: %w", &knownhosts.KeyError{Want: []knownhosts.KnownKey{{}}}),
|
||||
want: SSHReachabilityReasonHostKey,
|
||||
},
|
||||
{
|
||||
name: "host_key wins over an auth-shaped message in the same error",
|
||||
err: fmt.Errorf("ssh: handshake failed: ssh: unable to authenticate: %w",
|
||||
&errHostKeyMismatch{Expected: "A", Got: "B"}),
|
||||
want: SSHReachabilityReasonHostKey,
|
||||
},
|
||||
{
|
||||
name: "an authentication failure classifies as auth",
|
||||
err: errors.New("ssh: unable to authenticate, attempted methods [publickey], no supported methods remain"),
|
||||
want: SSHReachabilityReasonAuth,
|
||||
},
|
||||
{
|
||||
name: "a connection-refused dial error classifies as network",
|
||||
err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")},
|
||||
want: SSHReachabilityReasonNetwork,
|
||||
},
|
||||
{
|
||||
name: "an unrecognized error classifies as unknown",
|
||||
err: errors.New("something unexpected happened"),
|
||||
want: SSHReachabilityReasonUnknown,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := ClassifyDialError(tt.err); got != tt.want {
|
||||
t.Fatalf("ClassifyDialError(%v) = %q, want %q", tt.err, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.4, 001.5
|
||||
func TestProbeSSHHostSuccess(t *testing.T) {
|
||||
server := newFakeSSHServer(t, nil)
|
||||
newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
|
||||
target := &SSHTarget{
|
||||
Host: "127.0.0.1", Port: server.port(), User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
|
||||
PinnedFingerprint: server.hostKeyFingerprint(),
|
||||
}
|
||||
|
||||
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
|
||||
|
||||
if !outcome.Success {
|
||||
t.Fatalf("outcome = %+v, want Success", outcome)
|
||||
}
|
||||
if outcome.Host != "127.0.0.1" {
|
||||
t.Fatalf("Host = %q", outcome.Host)
|
||||
}
|
||||
if outcome.Reason != "" || outcome.Message != "" {
|
||||
t.Fatalf("a successful outcome must carry no reason/message: %+v", outcome)
|
||||
}
|
||||
if len(server.execCalls()) != 0 {
|
||||
t.Fatalf("execCalls = %v, want the probe to open no session channel", server.execCalls())
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.5
|
||||
func TestProbeSSHHostHostKeyMismatch(t *testing.T) {
|
||||
server := newFakeSSHServer(t, nil)
|
||||
newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
|
||||
target := &SSHTarget{
|
||||
Host: "127.0.0.1", Port: server.port(), User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
|
||||
PinnedFingerprint: "SHA256:not-the-server-key",
|
||||
}
|
||||
|
||||
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
|
||||
|
||||
if outcome.Success {
|
||||
t.Fatalf("outcome = %+v, want failure", outcome)
|
||||
}
|
||||
if outcome.Reason != SSHReachabilityReasonHostKey {
|
||||
t.Fatalf("Reason = %q, want host_key", outcome.Reason)
|
||||
}
|
||||
if target.PinnedFingerprint != "SHA256:not-the-server-key" {
|
||||
t.Fatalf("PinnedFingerprint mutated: %q", target.PinnedFingerprint)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.6
|
||||
func TestProbeSSHHostEmptyPinDoesNotDial(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
dialed := false
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
conn, err := ln.Accept()
|
||||
if err == nil {
|
||||
dialed = true
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
_ = ln.Close()
|
||||
wg.Wait()
|
||||
})
|
||||
addr := ln.Addr().(*net.TCPAddr)
|
||||
|
||||
target := &SSHTarget{
|
||||
Host: addr.IP.String(), Port: addr.Port, User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: "/does/not/matter",
|
||||
PinnedFingerprint: "",
|
||||
}
|
||||
|
||||
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
|
||||
|
||||
if outcome.Reason != SSHReachabilityReasonConfig {
|
||||
t.Fatalf("Reason = %q, want config", outcome.Reason)
|
||||
}
|
||||
if outcome.Success {
|
||||
t.Fatal("outcome must not be a success")
|
||||
}
|
||||
_ = ln.Close()
|
||||
wg.Wait()
|
||||
if dialed {
|
||||
t.Fatal("ProbeSSHHost dialed TCP despite an empty pinned fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
// newStallHandshakeListener accepts exactly one TCP connection and holds it
|
||||
// open without ever writing the SSH version banner, so a client's handshake
|
||||
// blocks on read until its own deadline fires. Used to prove the probe
|
||||
// bounds the handshake, not only the TCP dial — @covers
|
||||
// AC-EXECUTORS-SSH-REACHABILITY-001.7.
|
||||
func newStallHandshakeListener(t *testing.T) *net.TCPAddr {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
<-done
|
||||
_ = conn.Close()
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
close(done)
|
||||
_ = ln.Close()
|
||||
wg.Wait()
|
||||
})
|
||||
return ln.Addr().(*net.TCPAddr)
|
||||
}
|
||||
|
||||
func TestProbeSSHHostHandshakeStallIsBoundedByTheDeadline(t *testing.T) {
|
||||
newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
addr := newStallHandshakeListener(t)
|
||||
|
||||
target := &SSHTarget{
|
||||
Host: addr.IP.String(), Port: addr.Port, User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
|
||||
PinnedFingerprint: "SHA256:irrelevant",
|
||||
}
|
||||
|
||||
const probeTimeout = 300 * time.Millisecond
|
||||
start := time.Now()
|
||||
done := make(chan SSHProbeOutcome, 1)
|
||||
go func() { done <- ProbeSSHHost(context.Background(), target, probeTimeout) }()
|
||||
|
||||
select {
|
||||
case outcome := <-done:
|
||||
elapsed := time.Since(start)
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("probe took %s, want it bounded near the %s deadline", elapsed, probeTimeout)
|
||||
}
|
||||
if outcome.Reason != SSHReachabilityReasonTimeout {
|
||||
t.Fatalf("Reason = %q, want timeout; message=%q", outcome.Reason, outcome.Message)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("ProbeSSHHost did not return within 5s of a 300ms deadline — handshake is unbounded")
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.29
|
||||
func TestProbeSSHHostBastionHostKeyMismatchClassifiesAsHostKey(t *testing.T) {
|
||||
home := newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
|
||||
final := newFakeSSHServer(t, nil)
|
||||
bastion := newFakeSSHServer(t, nil)
|
||||
|
||||
_, otherPriv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("generate other key: %v", err)
|
||||
}
|
||||
otherSigner, err := ssh.NewSignerFromKey(otherPriv)
|
||||
if err != nil {
|
||||
t.Fatalf("other signer: %v", err)
|
||||
}
|
||||
writeKnownHosts(t, home, fmt.Sprintf("[127.0.0.1]:%d", bastion.port()), otherSigner.PublicKey())
|
||||
|
||||
proxyJump := "jump@127.0.0.1:" + strconv.Itoa(bastion.port())
|
||||
target := &SSHTarget{
|
||||
Host: "127.0.0.1", Port: final.port(), User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
|
||||
PinnedFingerprint: final.hostKeyFingerprint(),
|
||||
ProxyJump: proxyJump,
|
||||
}
|
||||
|
||||
outcome := ProbeSSHHost(context.Background(), target, 5*time.Second)
|
||||
|
||||
if outcome.Reason != SSHReachabilityReasonHostKey {
|
||||
t.Fatalf("Reason = %q, want host_key; message=%q", outcome.Reason, outcome.Message)
|
||||
}
|
||||
if outcome.Host != "127.0.0.1" {
|
||||
t.Fatalf("Host = %q, want the target host, not the bastion", outcome.Host)
|
||||
}
|
||||
if !strings.Contains(outcome.Message, "bastion") {
|
||||
t.Fatalf("Message = %q, want it to name the bastion", outcome.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeSSHHostCancelledContextProducesNoObservation(t *testing.T) {
|
||||
server := newFakeSSHServer(t, nil)
|
||||
newTestHomeDir(t)
|
||||
identity := writeTestIdentityFile(t)
|
||||
|
||||
target := &SSHTarget{
|
||||
Host: "127.0.0.1", Port: server.port(), User: "kandev",
|
||||
IdentitySource: SSHIdentitySourceFile, IdentityFile: identity,
|
||||
PinnedFingerprint: server.hostKeyFingerprint(),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
outcome := ProbeSSHHost(ctx, target, 5*time.Second)
|
||||
|
||||
if !outcome.Cancelled {
|
||||
t.Fatalf("outcome = %+v, want Cancelled", outcome)
|
||||
}
|
||||
if outcome.Success {
|
||||
t.Fatal("a cancelled probe must not report success")
|
||||
}
|
||||
if outcome.Reason != "" {
|
||||
t.Fatalf("Reason = %q, want no reason for a cancelled probe", outcome.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHTargetFromExecutorConfig(t *testing.T) {
|
||||
newTestHomeDir(t)
|
||||
|
||||
t.Run("a valid config resolves", func(t *testing.T) {
|
||||
target, err := SSHTargetFromExecutorConfig(map[string]string{
|
||||
"ssh_host": "target.internal",
|
||||
"ssh_port": "2222",
|
||||
"ssh_user": "deploy",
|
||||
"ssh_identity_source": "file",
|
||||
"ssh_identity_file": "/keys/id_ed25519",
|
||||
"ssh_host_fingerprint": "SHA256:abc",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SSHTargetFromExecutorConfig: %v", err)
|
||||
}
|
||||
if target.Host != "target.internal" || target.Port != 2222 || target.User != "deploy" {
|
||||
t.Fatalf("target = %+v", target)
|
||||
}
|
||||
if target.PinnedFingerprint != "SHA256:abc" {
|
||||
t.Fatalf("PinnedFingerprint = %q", target.PinnedFingerprint)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a nil config fails resolution", func(t *testing.T) {
|
||||
if _, err := SSHTargetFromExecutorConfig(nil); err == nil {
|
||||
t.Fatal("expected an error for a nil config")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an invalid port fails resolution", func(t *testing.T) {
|
||||
if _, err := SSHTargetFromExecutorConfig(map[string]string{
|
||||
"ssh_host": "target.internal",
|
||||
"ssh_port": "not-a-port",
|
||||
}); err == nil {
|
||||
t.Fatal("expected an error for an invalid port")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a missing pinned fingerprint fails resolution", func(t *testing.T) {
|
||||
if _, err := SSHTargetFromExecutorConfig(map[string]string{
|
||||
"ssh_host": "target.internal",
|
||||
"ssh_user": "deploy",
|
||||
}); err == nil {
|
||||
t.Fatal("expected an error when the pinned host fingerprint is missing")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSanitizeSSHReachabilityMessageBoundsAndRedacts(t *testing.T) {
|
||||
secret := "ssh: unable to authenticate, password=super-secret-token " + strings.Repeat("x", 700)
|
||||
got := SanitizeSSHReachabilityMessage(errors.New(secret))
|
||||
if len(got) > maxSSHReachabilityMessageBytes {
|
||||
t.Fatalf("message length = %d, want at most %d", len(got), maxSSHReachabilityMessageBytes)
|
||||
}
|
||||
if strings.Contains(got, "super-secret-token") {
|
||||
t.Fatalf("message contains the raw credential: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -250,6 +250,20 @@ type Manager struct {
|
||||
// SkillDeployer hook. Nil → skill deploy is skipped.
|
||||
agentProfileReader AgentProfileReader
|
||||
|
||||
// reachabilityReader resolves an ssh executor's stored reachability
|
||||
// record for the launch-time session.launch.warning producer. Nil →
|
||||
// no warning is ever published (feature not wired). See
|
||||
// manager_launch_reachability_warning.go and SetSSHReachabilityWarningPolicy.
|
||||
reachabilityReader ReachabilityReader
|
||||
// reachabilityProbingEnabled mirrors whether the reachability poller's
|
||||
// periodic sweep is on (interval != 0). When true, a stored unreachable
|
||||
// record is always warning-eligible regardless of how old checked_at is.
|
||||
reachabilityProbingEnabled bool
|
||||
// reachabilityWarningWindowSeconds is 3x the reachability package's own
|
||||
// default interval (not the configured/effective one), evaluated even
|
||||
// with probing disabled per AC-EXECUTORS-SSH-REACHABILITY-001.28.
|
||||
reachabilityWarningWindowSeconds int
|
||||
|
||||
// skillDeployer materialises per-profile skills + custom prompt before
|
||||
// the agent process starts. Defaults to a no-op deployer; office wires
|
||||
// its concrete implementation via SetSkillDeployer.
|
||||
|
||||
@@ -1096,6 +1096,8 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
m.scheduleSSHLaunchWarning(launchCtx, reqWithWorktree, metadata, reqWithWorktree.SessionID)
|
||||
|
||||
execInstance, err := rt.CreateInstance(launchCtx, execReq)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("failed to create execution: %w", err)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// ReachabilityReader is the narrow read accessor launchBuildExecutorRequest
|
||||
// uses to look up an ssh executor's stored reachability record immediately
|
||||
// before calling CreateInstance. It is declared here — not imported from
|
||||
// executors/reachability — because that package already imports lifecycle
|
||||
// for SSH target resolution and dial classification; importing it back here
|
||||
// would cycle. In production this is satisfied directly by the task
|
||||
// repository, which already implements this exact method for the
|
||||
// reachability HTTP routes (see internal/ssh.ReachabilityLister).
|
||||
type ReachabilityReader interface {
|
||||
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
|
||||
}
|
||||
|
||||
// SetSSHReachabilityWarningPolicy wires the launch-time session.launch.warning
|
||||
// producer (see task-05-launch-non-gating.md). reader resolves an ssh
|
||||
// executor's stored record; probingEnabled mirrors whether the reachability
|
||||
// poller's periodic sweep is on; warningWindowSeconds is 3x the reachability
|
||||
// package's own default interval, evaluated even with probing disabled.
|
||||
// Leaving this unset (reader nil) disables the producer entirely.
|
||||
func (m *Manager) SetSSHReachabilityWarningPolicy(reader ReachabilityReader, probingEnabled bool, warningWindowSeconds int) {
|
||||
m.reachabilityReader = reader
|
||||
m.reachabilityProbingEnabled = probingEnabled
|
||||
m.reachabilityWarningWindowSeconds = warningWindowSeconds
|
||||
}
|
||||
|
||||
// LaunchWarningEventPayload is the session.launch.warning payload. It is
|
||||
// published once, immediately before an SSH launch's CreateInstance call,
|
||||
// when the target executor's stored reachability record is unreachable and
|
||||
// still within the probing window.
|
||||
type LaunchWarningEventPayload struct {
|
||||
TaskID string `json:"task_id"`
|
||||
SessionID string `json:"session_id"`
|
||||
ExecutorID string `json:"executor_id"`
|
||||
Host string `json:"host"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// sshLaunchWarningReadTimeout bounds the best-effort record lookup started by
|
||||
// a launch. The lookup must not hold up CreateInstance, but it also must not
|
||||
// leave a database call running forever after a launch has finished.
|
||||
const sshLaunchWarningReadTimeout = 2 * time.Second
|
||||
|
||||
// GetSessionID satisfies the session-routing accessor
|
||||
// internal/gateway/websocket.extractSessionID uses to broadcast this event
|
||||
// to the correct session's subscribers.
|
||||
func (p LaunchWarningEventPayload) GetSessionID() string {
|
||||
return p.SessionID
|
||||
}
|
||||
|
||||
// PublishLaunchWarning publishes a session.launch.warning event to the
|
||||
// launched session's own event stream.
|
||||
func (p *EventPublisher) PublishLaunchWarning(sessionID string, payload *LaunchWarningEventPayload) {
|
||||
if p.eventBus == nil {
|
||||
return
|
||||
}
|
||||
if payload.Timestamp == "" {
|
||||
payload.Timestamp = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
subject := events.BuildSessionLaunchWarningSubject(sessionID)
|
||||
event := bus.NewEvent(events.SessionLaunchWarning, "lifecycle", payload)
|
||||
if err := p.eventBus.Publish(context.Background(), subject, event); err != nil {
|
||||
p.logger.Error("failed to publish session launch warning event",
|
||||
zap.String("session_id", sessionID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// scheduleSSHLaunchWarning starts the best-effort record lookup without
|
||||
// delaying CreateInstance. The request and metadata values used by the
|
||||
// goroutine are copied because the launch path may reuse or mutate its input
|
||||
// maps after it returns.
|
||||
func (m *Manager) scheduleSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) {
|
||||
if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) ||
|
||||
m.reachabilityReader == nil || m.eventPublisher == nil {
|
||||
return
|
||||
}
|
||||
warningReq := *req
|
||||
warningMetadata := map[string]interface{}{
|
||||
"executor_id": getMetadataString(metadata, "executor_id"),
|
||||
}
|
||||
warningCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sshLaunchWarningReadTimeout)
|
||||
go func() {
|
||||
defer cancel()
|
||||
m.maybePublishSSHLaunchWarning(warningCtx, &warningReq, warningMetadata, sessionID)
|
||||
}()
|
||||
}
|
||||
|
||||
// maybePublishSSHLaunchWarning reads the target executor's reachability
|
||||
// record and, when it is unreachable and still warning-eligible, publishes
|
||||
// exactly one session.launch.warning. It is a strict read-only accessor call,
|
||||
// made from launchBuildExecutorRequest alongside its single CreateInstance
|
||||
// call site, never from inside CreateInstance itself. A read failure or a
|
||||
// missing record produces no warning.
|
||||
func (m *Manager) maybePublishSSHLaunchWarning(ctx context.Context, req *LaunchRequest, metadata map[string]interface{}, sessionID string) {
|
||||
if req == nil || req.ExecutorType != string(models.ExecutorTypeSSH) ||
|
||||
m.reachabilityReader == nil || m.eventPublisher == nil {
|
||||
return
|
||||
}
|
||||
executorID := strings.TrimSpace(getMetadataString(metadata, "executor_id"))
|
||||
if executorID == "" {
|
||||
return
|
||||
}
|
||||
record, err := m.reachabilityReader.GetExecutorReachability(ctx, executorID)
|
||||
if err != nil || record == nil || record.State != models.ExecutorReachabilityStateUnreachable {
|
||||
return
|
||||
}
|
||||
if !m.reachabilityProbingEnabled && !withinReachabilityWarningWindow(record.CheckedAt, m.reachabilityWarningWindowSeconds) {
|
||||
return
|
||||
}
|
||||
m.eventPublisher.PublishLaunchWarning(sessionID, &LaunchWarningEventPayload{
|
||||
TaskID: req.TaskID,
|
||||
SessionID: sessionID,
|
||||
ExecutorID: executorID,
|
||||
Host: record.Host,
|
||||
State: string(record.State),
|
||||
Reason: string(record.Reason),
|
||||
LastSuccessAt: record.LastSuccessAt,
|
||||
})
|
||||
}
|
||||
|
||||
// withinReachabilityWarningWindow reports whether checkedAt is recent enough
|
||||
// to still warn even with periodic probing disabled. A nil checkedAt (never
|
||||
// probed) or a non-positive window never qualifies.
|
||||
func withinReachabilityWarningWindow(checkedAt *time.Time, windowSeconds int) bool {
|
||||
if checkedAt == nil || windowSeconds <= 0 {
|
||||
return false
|
||||
}
|
||||
return time.Since(*checkedAt) <= time.Duration(windowSeconds)*time.Second
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/executor"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
type fakeReachabilityReader struct {
|
||||
records map[string]*models.ExecutorReachability
|
||||
err error
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityReader) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
f.calls = append(f.calls, executorID)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.records[executorID], nil
|
||||
}
|
||||
|
||||
// failOnReadReachabilityReader fails the test the moment it is touched at
|
||||
// all — used to prove a code path performs no reachability read, rather than
|
||||
// merely counting reads after the fact.
|
||||
type failOnReadReachabilityReader struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (f failOnReadReachabilityReader) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) {
|
||||
f.t.Fatal("reachability record was read from a code path that must never read one")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type blockingReachabilityReader struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (r *blockingReachabilityReader) GetExecutorReachability(ctx context.Context, _ string) (*models.ExecutorReachability, error) {
|
||||
close(r.started)
|
||||
select {
|
||||
case <-r.release:
|
||||
return nil, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func newWarningTestManager(t *testing.T, reader ReachabilityReader, probingEnabled bool, windowSeconds int) (*Manager, *bus.MemoryEventBus) {
|
||||
t.Helper()
|
||||
log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"})
|
||||
if err != nil {
|
||||
t.Fatalf("new logger: %v", err)
|
||||
}
|
||||
memBus := bus.NewMemoryEventBus(log)
|
||||
mgr := &Manager{
|
||||
logger: log,
|
||||
eventPublisher: NewEventPublisher(memBus, log),
|
||||
}
|
||||
mgr.SetSSHReachabilityWarningPolicy(reader, probingEnabled, windowSeconds)
|
||||
return mgr, memBus
|
||||
}
|
||||
|
||||
func subscribeLaunchWarnings(t *testing.T, memBus *bus.MemoryEventBus, sessionID string) <-chan *bus.Event {
|
||||
t.Helper()
|
||||
received := make(chan *bus.Event, 1)
|
||||
sub, err := memBus.Subscribe(events.BuildSessionLaunchWarningSubject(sessionID), func(_ context.Context, ev *bus.Event) error {
|
||||
received <- ev
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
return received
|
||||
}
|
||||
|
||||
const warningWindowSeconds = 180 // 3x reachability.DefaultIntervalSeconds
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_UnreachableAndProbingEnabledWarns(t *testing.T) {
|
||||
old := time.Now().Add(-time.Hour)
|
||||
lastSuccess := old.Add(-time.Hour)
|
||||
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
|
||||
"executor-1": {
|
||||
ExecutorID: "executor-1",
|
||||
State: models.ExecutorReachabilityStateUnreachable,
|
||||
Reason: models.ExecutorReachabilityReasonTimeout,
|
||||
Host: "build.example",
|
||||
CheckedAt: &old,
|
||||
LastSuccessAt: &lastSuccess,
|
||||
},
|
||||
}}
|
||||
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{
|
||||
ExecutorType: string(models.ExecutorTypeSSH),
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
}
|
||||
metadata := map[string]interface{}{"executor_id": "executor-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, metadata, req.SessionID)
|
||||
|
||||
select {
|
||||
case ev := <-received:
|
||||
payload, ok := ev.Data.(*LaunchWarningEventPayload)
|
||||
if !ok {
|
||||
t.Fatalf("payload type = %T", ev.Data)
|
||||
}
|
||||
if payload.ExecutorID != "executor-1" || payload.Host != "build.example" ||
|
||||
payload.State != string(models.ExecutorReachabilityStateUnreachable) ||
|
||||
payload.Reason != string(models.ExecutorReachabilityReasonTimeout) ||
|
||||
payload.LastSuccessAt == nil || !payload.LastSuccessAt.Equal(lastSuccess) {
|
||||
t.Fatalf("payload = %+v", payload)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected a session.launch.warning event, got none")
|
||||
}
|
||||
if len(reader.calls) != 1 || reader.calls[0] != "executor-1" {
|
||||
t.Fatalf("reader.calls = %v", reader.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_ProbingDisabledButRecentlyCheckedWarns(t *testing.T) {
|
||||
recent := time.Now().Add(-time.Minute)
|
||||
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
|
||||
"executor-1": {
|
||||
ExecutorID: "executor-1",
|
||||
State: models.ExecutorReachabilityStateUnreachable,
|
||||
Reason: models.ExecutorReachabilityReasonNetwork,
|
||||
Host: "build.example",
|
||||
CheckedAt: &recent,
|
||||
},
|
||||
}}
|
||||
mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
|
||||
select {
|
||||
case <-received:
|
||||
default:
|
||||
t.Fatal("expected a session.launch.warning event when checked_at is within the warning window")
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_ProbingDisabledAndStaleCheckedAtDoesNotWarn(t *testing.T) {
|
||||
stale := time.Now().Add(-time.Hour)
|
||||
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
|
||||
"executor-1": {
|
||||
ExecutorID: "executor-1",
|
||||
State: models.ExecutorReachabilityStateUnreachable,
|
||||
Host: "build.example",
|
||||
CheckedAt: &stale,
|
||||
},
|
||||
}}
|
||||
mgr, memBus := newWarningTestManager(t, reader, false, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
|
||||
select {
|
||||
case ev := <-received:
|
||||
t.Fatalf("expected no warning for a stale checked_at with probing disabled, got %+v", ev)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_ReachableRecordDoesNotWarn(t *testing.T) {
|
||||
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{
|
||||
"executor-1": {ExecutorID: "executor-1", State: models.ExecutorReachabilityStateReachable, Host: "build.example"},
|
||||
}}
|
||||
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
|
||||
select {
|
||||
case ev := <-received:
|
||||
t.Fatalf("expected no warning for a reachable record, got %+v", ev)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_NoRecordDoesNotWarn(t *testing.T) {
|
||||
reader := &fakeReachabilityReader{records: map[string]*models.ExecutorReachability{}}
|
||||
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
|
||||
select {
|
||||
case ev := <-received:
|
||||
t.Fatalf("expected no warning when no record exists yet, got %+v", ev)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.3
|
||||
func TestMaybePublishSSHLaunchWarning_ReadFailureDoesNotWarnOrPanic(t *testing.T) {
|
||||
reader := &fakeReachabilityReader{err: errors.New("boom")}
|
||||
mgr, memBus := newWarningTestManager(t, reader, true, warningWindowSeconds)
|
||||
received := subscribeLaunchWarnings(t, memBus, "session-1")
|
||||
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
|
||||
select {
|
||||
case ev := <-received:
|
||||
t.Fatalf("expected no warning on a read failure, got %+v", ev)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// Non-ssh launches must never trigger a reachability read at all — the
|
||||
// failOnReadReachabilityReader fails the test the instant it is touched,
|
||||
// which is what proves the short-circuit rather than merely observing zero
|
||||
// warnings (a bug that read the record but decided not to warn would still
|
||||
// pass a call-count-only assertion).
|
||||
func TestMaybePublishSSHLaunchWarning_NonSSHExecutorNeverReads(t *testing.T) {
|
||||
mgr, _ := newWarningTestManager(t, failOnReadReachabilityReader{t: t}, true, warningWindowSeconds)
|
||||
|
||||
req := &LaunchRequest{ExecutorType: "local_docker", SessionID: "session-1"}
|
||||
mgr.maybePublishSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
}
|
||||
|
||||
func TestScheduleSSHLaunchWarningDoesNotWaitForRead(t *testing.T) {
|
||||
reader := &blockingReachabilityReader{started: make(chan struct{}), release: make(chan struct{})}
|
||||
mgr, _ := newWarningTestManager(t, reader, true, warningWindowSeconds)
|
||||
req := &LaunchRequest{ExecutorType: string(models.ExecutorTypeSSH), SessionID: "session-1"}
|
||||
|
||||
started := time.Now()
|
||||
mgr.scheduleSSHLaunchWarning(context.Background(), req, map[string]interface{}{"executor_id": "executor-1"}, req.SessionID)
|
||||
if elapsed := time.Since(started); elapsed > 100*time.Millisecond {
|
||||
t.Fatalf("scheduleSSHLaunchWarning waited %v for a blocked read", elapsed)
|
||||
}
|
||||
select {
|
||||
case <-reader.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("warning read did not start asynchronously")
|
||||
}
|
||||
close(reader.release)
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// launchBuildExecutorRequest is the single production call site for every
|
||||
// executor type (verified: it has exactly one caller, launchInternal, which
|
||||
// every launch entry point — WS-initiated, resume, and workflow/dependency
|
||||
// launches via orchestrator/executor's LaunchAgent — funnels through). This
|
||||
// exercises that shared call site directly with a non-ssh backend and proves
|
||||
// the reachability reader is never touched and CreateInstance still runs.
|
||||
func TestLaunchBuildExecutorRequestNonSSHExecutorNeverReadsReachability(t *testing.T) {
|
||||
log := newTestRegistryLogger()
|
||||
registry := NewExecutorRegistry(log)
|
||||
backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameDocker}}
|
||||
registry.Register(backend)
|
||||
|
||||
memBusLog, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"})
|
||||
memBus := bus.NewMemoryEventBus(memBusLog)
|
||||
mgr := &Manager{
|
||||
executorRegistry: registry,
|
||||
executorFallbackPolicy: ExecutorFallbackDeny,
|
||||
logger: log,
|
||||
eventPublisher: NewEventPublisher(memBus, memBusLog),
|
||||
}
|
||||
mgr.SetSSHReachabilityWarningPolicy(failOnReadReachabilityReader{t: t}, true, warningWindowSeconds)
|
||||
|
||||
req := &LaunchRequest{
|
||||
ExecutorType: "local_docker",
|
||||
EnvironmentFinalized: true,
|
||||
SessionID: "session-1",
|
||||
Metadata: map[string]interface{}{"executor_id": "executor-1"},
|
||||
}
|
||||
|
||||
_, _, _, err := mgr.launchBuildExecutorRequest(
|
||||
context.Background(), "instance-1", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("launchBuildExecutorRequest: %v", err)
|
||||
}
|
||||
if backend.lastRequest == nil {
|
||||
t.Fatal("expected CreateInstance to run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
||||
)
|
||||
|
||||
// SSHReachabilityReason classifies why an SSH reachability probe failed. It
|
||||
// mirrors lifecycle.SSHReachabilityReason so callers outside the runtime
|
||||
// package never import the lifecycle tier directly.
|
||||
type SSHReachabilityReason string
|
||||
|
||||
const (
|
||||
SSHReachabilityReasonConfig SSHReachabilityReason = "config"
|
||||
SSHReachabilityReasonTimeout SSHReachabilityReason = "timeout"
|
||||
SSHReachabilityReasonHostKey SSHReachabilityReason = "host_key"
|
||||
SSHReachabilityReasonAuth SSHReachabilityReason = "auth"
|
||||
SSHReachabilityReasonNetwork SSHReachabilityReason = "network"
|
||||
SSHReachabilityReasonUnknown SSHReachabilityReason = "unknown"
|
||||
)
|
||||
|
||||
// SSHProbeOutcome is the runtime-facade mirror of lifecycle.SSHProbeOutcome.
|
||||
type SSHProbeOutcome struct {
|
||||
Success bool
|
||||
Cancelled bool
|
||||
Host string
|
||||
Reason SSHReachabilityReason
|
||||
Message string
|
||||
}
|
||||
|
||||
// SanitizeSSHReachabilityMessage exposes the runtime-safe SSH diagnostic
|
||||
// projection without making higher-level adapters import lifecycle directly.
|
||||
func SanitizeSSHReachabilityMessage(err error) string {
|
||||
return lifecycle.SanitizeSSHReachabilityMessage(err)
|
||||
}
|
||||
|
||||
// SSHReachabilityProber is the runtime seam for a single SSH reachability
|
||||
// probe. Higher-level packages depend on this contract instead of the
|
||||
// lifecycle implementation details.
|
||||
type SSHReachabilityProber interface {
|
||||
// Probe resolves an SSH executor's target from its persisted config and
|
||||
// probes it, bounded by timeout. A resolution failure (missing host,
|
||||
// invalid port, unresolvable alias) never dials — it is reported as
|
||||
// SSHReachabilityReasonConfig directly, the same reason a target with no
|
||||
// pinned fingerprint uses.
|
||||
Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome
|
||||
// ValidateConfig reports whether config resolves to a dialable SSH
|
||||
// target, without dialing it.
|
||||
ValidateConfig(config map[string]string) error
|
||||
}
|
||||
|
||||
type sshReachabilityProber struct {
|
||||
resolveTarget func(map[string]string) (*lifecycle.SSHTarget, error)
|
||||
probe func(context.Context, *lifecycle.SSHTarget, time.Duration) lifecycle.SSHProbeOutcome
|
||||
}
|
||||
|
||||
// NewSSHReachabilityProber creates the production adapter over the lifecycle
|
||||
// SSH target resolver and reachability prober.
|
||||
func NewSSHReachabilityProber() SSHReachabilityProber {
|
||||
return &sshReachabilityProber{
|
||||
resolveTarget: lifecycle.SSHTargetFromExecutorConfig,
|
||||
probe: lifecycle.ProbeSSHHost,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *sshReachabilityProber) Probe(ctx context.Context, config map[string]string, timeout time.Duration) SSHProbeOutcome {
|
||||
target, err := a.resolveTarget(config)
|
||||
if err != nil {
|
||||
return SSHProbeOutcome{
|
||||
Host: config["ssh_host"],
|
||||
Reason: SSHReachabilityReasonConfig,
|
||||
Message: lifecycle.SanitizeSSHReachabilityMessage(err),
|
||||
}
|
||||
}
|
||||
outcome := a.probe(ctx, target, timeout)
|
||||
return SSHProbeOutcome{
|
||||
Success: outcome.Success,
|
||||
Cancelled: outcome.Cancelled,
|
||||
Host: outcome.Host,
|
||||
Reason: SSHReachabilityReason(outcome.Reason),
|
||||
Message: outcome.Message,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *sshReachabilityProber) ValidateConfig(config map[string]string) error {
|
||||
_, err := a.resolveTarget(config)
|
||||
return err
|
||||
}
|
||||
|
||||
var _ SSHReachabilityProber = (*sshReachabilityProber)(nil)
|
||||
@@ -220,32 +220,32 @@ func stringContains(value, want string) bool {
|
||||
|
||||
func newCanvasEditTestDependencies() (*fakeCanvasEditTaskStore, *fakeCanvasEditCanvasStore, *fakeCanvasEditReleaseStore, *fakeCanvasEditArtifactStore) {
|
||||
return &fakeCanvasEditTaskStore{
|
||||
workspace: &models.Workspace{
|
||||
ID: "workspace-a",
|
||||
DefaultAgentProfileID: stringPointer("profile-default"),
|
||||
DefaultExecutorID: stringPointer("executor-default"),
|
||||
},
|
||||
}, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{
|
||||
ID: "canvas-a",
|
||||
PluginInstanceID: "instance-a",
|
||||
WorkspaceID: "workspace-a",
|
||||
ScopeKind: canvas.ScopeWorkspace,
|
||||
Status: instances.StatusActive,
|
||||
ActiveReleaseID: "release-a",
|
||||
ActiveReleaseStatus: instances.ValidationValid,
|
||||
EffectiveGrants: []canvas.GrantProjection{{
|
||||
PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace,
|
||||
}},
|
||||
}}, &fakeCanvasEditReleaseStore{release: instances.Release{
|
||||
ID: "release-a",
|
||||
InstanceID: "instance-a",
|
||||
PackageDigest: "digest-a",
|
||||
ArtifactPath: "releases/digest-a",
|
||||
ValidationStatus: instances.ValidationValid,
|
||||
}}, &fakeCanvasEditArtifactStore{files: map[string][]byte{
|
||||
"manifest.yaml": []byte("manifest"),
|
||||
"index.html": []byte("index"),
|
||||
}}
|
||||
workspace: &models.Workspace{
|
||||
ID: "workspace-a",
|
||||
DefaultAgentProfileID: stringPointer("profile-default"),
|
||||
DefaultExecutorID: stringPointer("executor-default"),
|
||||
},
|
||||
}, &fakeCanvasEditCanvasStore{canvas: &canvas.Canvas{
|
||||
ID: "canvas-a",
|
||||
PluginInstanceID: "instance-a",
|
||||
WorkspaceID: "workspace-a",
|
||||
ScopeKind: canvas.ScopeWorkspace,
|
||||
Status: instances.StatusActive,
|
||||
ActiveReleaseID: "release-a",
|
||||
ActiveReleaseStatus: instances.ValidationValid,
|
||||
EffectiveGrants: []canvas.GrantProjection{{
|
||||
PermissionKind: "api_read", Resource: "tasks", ScopeCeiling: instances.ScopeWorkspace,
|
||||
}},
|
||||
}}, &fakeCanvasEditReleaseStore{release: instances.Release{
|
||||
ID: "release-a",
|
||||
InstanceID: "instance-a",
|
||||
PackageDigest: "digest-a",
|
||||
ArtifactPath: "releases/digest-a",
|
||||
ValidationStatus: instances.ValidationValid,
|
||||
}}, &fakeCanvasEditArtifactStore{files: map[string][]byte{
|
||||
"manifest.yaml": []byte("manifest"),
|
||||
"index.html": []byte("index"),
|
||||
}}
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string { return &value }
|
||||
|
||||
@@ -46,6 +46,7 @@ import (
|
||||
editorhandlers "github.com/kandev/kandev/internal/editors/handlers"
|
||||
"github.com/kandev/kandev/internal/entityrefs"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
"github.com/kandev/kandev/internal/failedinbox"
|
||||
gateways "github.com/kandev/kandev/internal/gateway/websocket"
|
||||
"github.com/kandev/kandev/internal/github"
|
||||
@@ -729,6 +730,7 @@ type routeParams struct {
|
||||
planCoalesceWindowConfigured bool
|
||||
homeDir string
|
||||
interimSettingsInterlockToken string
|
||||
sshReachabilityPoller *reachabilitypkg.Poller
|
||||
log *logger.Logger
|
||||
progress *startup.Reporter
|
||||
}
|
||||
@@ -1467,6 +1469,14 @@ func registerSecondaryRoutes(
|
||||
)
|
||||
p.log.Debug("Registered Kubernetes handlers (HTTP + WebSocket)")
|
||||
|
||||
// A nil *reachabilitypkg.Poller must not be passed directly as the
|
||||
// sshhandlers.ReachabilityProber interface parameter — that would
|
||||
// produce a non-nil interface holding a nil pointer, defeating the
|
||||
// handler's own nil check and panicking on first use.
|
||||
var reachabilityPoller sshhandlers.ReachabilityProber
|
||||
if p.sshReachabilityPoller != nil {
|
||||
reachabilityPoller = p.sshReachabilityPoller
|
||||
}
|
||||
sshhandlers.RegisterRoutes(
|
||||
p.router,
|
||||
p.gateway.Dispatcher,
|
||||
@@ -1475,6 +1485,8 @@ func registerSecondaryRoutes(
|
||||
p.agentRegistry,
|
||||
lifecycle.NewAgentctlResolver(p.log),
|
||||
p.log,
|
||||
p.taskRepo,
|
||||
reachabilityPoller,
|
||||
)
|
||||
p.log.Debug("Registered SSH handlers (HTTP + WebSocket)")
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestBuildHTTPServerAbortsWhenInterlockTokenGenerationFails(t *testing.T) {
|
||||
newInterimSettingsInterlockToken = func() (string, error) { return "", wantErr }
|
||||
t.Cleanup(func() { newInterimSettingsInterlockToken = original })
|
||||
|
||||
server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
||||
server, err := buildHTTPServer(&config.Config{}, testLogger(t), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("buildHTTPServer error = %v, want %v", err, wantErr)
|
||||
}
|
||||
|
||||
@@ -43,6 +43,9 @@ import (
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
|
||||
// SSH executor reachability poller
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
|
||||
// GitHub integration
|
||||
azuredevopspkg "github.com/kandev/kandev/internal/azuredevops"
|
||||
githubpkg "github.com/kandev/kandev/internal/github"
|
||||
@@ -881,6 +884,36 @@ func startAgentInfrastructure(
|
||||
log.Info("Office config sync poller started")
|
||||
}
|
||||
|
||||
// Start SSH executor reachability poller: sweeps every eligible SSH
|
||||
// executor on a configurable interval, probing reachability and
|
||||
// persisting results through a hysteresis-owning write path. Wired with
|
||||
// a publisher (state/reason changes reach WS clients) and registered as
|
||||
// the task service's executor-save observer (a changed host resets the
|
||||
// record and dispatches an off-cycle probe) so both the poller and the
|
||||
// reachability HTTP routes below share the one running instance.
|
||||
sshReachabilityPoller := startSSHReachabilityPoller(
|
||||
ctx,
|
||||
repos.Task,
|
||||
cfg.Executors.SSHReachabilityIntervalSeconds,
|
||||
log,
|
||||
reachabilitypkg.NewPublisher(eventBus, log),
|
||||
addRuntimeCleanup,
|
||||
)
|
||||
services.Task.SetExecutorSaveObserver(reachabilitypkg.NewSaveObserver(sshReachabilityPoller))
|
||||
|
||||
// Launch-time session.launch.warning producer (task 05): repos.Task
|
||||
// already implements the narrow read accessor (same method used by the
|
||||
// reachability HTTP routes). probingEnabled mirrors the poller's own
|
||||
// effective interval so a configured 0 (disabled) keeps the warning
|
||||
// gated on staleness alone. warningWindowSeconds is 3x the reachability
|
||||
// package's own default interval, not the configured one, per
|
||||
// AC-EXECUTORS-SSH-REACHABILITY-001.28.
|
||||
lifecycleMgr.SetSSHReachabilityWarningPolicy(
|
||||
repos.Task,
|
||||
sshReachabilityPoller.EffectiveIntervalSeconds() != 0,
|
||||
3*reachabilitypkg.DefaultIntervalSeconds,
|
||||
)
|
||||
|
||||
// Start the plugin system's event delivery and health monitor
|
||||
// background loops.
|
||||
if services.Plugins != nil {
|
||||
@@ -943,7 +976,7 @@ func startAgentInfrastructure(
|
||||
return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services,
|
||||
agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath,
|
||||
sessionCapacityEnvironment, storageStore, func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelWorkers,
|
||||
restoreCleanups, databaseQuiesce)
|
||||
restoreCleanups, databaseQuiesce, sshReachabilityPoller)
|
||||
}
|
||||
|
||||
// startOrchestratorAndAutomationConsumers establishes the startup chain in
|
||||
@@ -1013,6 +1046,7 @@ func startGatewayAndServe(
|
||||
cancelWorkers context.CancelFunc,
|
||||
restoreCleanups []func() error,
|
||||
databaseQuiesce func() error,
|
||||
sshReachabilityPoller *reachabilitypkg.Poller,
|
||||
) bool {
|
||||
// ============================================
|
||||
// WEBSOCKET GATEWAY
|
||||
@@ -1350,7 +1384,7 @@ func startGatewayAndServe(
|
||||
builtServer, err := buildHTTPServer(cfg, log, gateway, repos, services, agentSettingsController,
|
||||
lifecycleMgr, eventBus, orchestratorSvc, notificationCtrl, msgCreator, agentRegistry, hostUtilityMgr,
|
||||
addCleanup, repoCloner, systemSvc, storageComposition.workspaceRestorer,
|
||||
storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, startup.FromContext(ctx), persistenceHealth)
|
||||
storageComposition.tempArtifacts, dbPool, agentRuntimeAvailability, sshReachabilityPoller, startup.FromContext(ctx), persistenceHealth)
|
||||
if err != nil {
|
||||
log.Error("Failed to build HTTP server", zap.Error(err))
|
||||
closeBoundListeners(server, listeners, log)
|
||||
@@ -2599,6 +2633,7 @@ func buildHTTPServer(
|
||||
temporaryArtifacts *tempartifacts.Registry,
|
||||
dbPool *db.Pool,
|
||||
agentRuntimeAvailability *agentctlclient.Availability,
|
||||
sshReachabilityPoller *reachabilitypkg.Poller,
|
||||
progress *startup.Reporter,
|
||||
persistenceHealth ...*requiredstores.Health,
|
||||
) (*http.Server, error) {
|
||||
@@ -2730,6 +2765,7 @@ func buildHTTPServer(
|
||||
planCoalesceWindowConfigured: true,
|
||||
homeDir: cfg.ResolvedHomeDir(),
|
||||
interimSettingsInterlockToken: interimSettingsInterlockToken,
|
||||
sshReachabilityPoller: sshReachabilityPoller,
|
||||
log: log,
|
||||
progress: progress,
|
||||
})
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package backendapp
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
)
|
||||
|
||||
// startSSHReachabilityPoller launches the SSH executor reachability poller
|
||||
// and registers its Stop on addCleanup, mirroring the third-party
|
||||
// integration poller wiring alongside it in startAgentInfrastructure.
|
||||
func startSSHReachabilityPoller(
|
||||
ctx context.Context,
|
||||
repo reachabilitypkg.Repository,
|
||||
intervalSeconds int,
|
||||
log *logger.Logger,
|
||||
publisher *reachabilitypkg.Publisher,
|
||||
addCleanup func(func() error) func() error,
|
||||
) *reachabilitypkg.Poller {
|
||||
poller := reachabilitypkg.New(repo, intervalSeconds, log)
|
||||
poller.SetPublisher(publisher)
|
||||
poller.Start(ctx)
|
||||
addCleanup(func() error { poller.Stop(); return nil })
|
||||
return poller
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package backendapp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
type sshReachabilityWiringRepo struct {
|
||||
listed chan struct{}
|
||||
}
|
||||
|
||||
func (r *sshReachabilityWiringRepo) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) {
|
||||
select {
|
||||
case r.listed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *sshReachabilityWiringRepo) GetExecutorReachability(context.Context, string) (*models.ExecutorReachability, error) {
|
||||
return nil, models.ErrExecutorReachabilityNotFound
|
||||
}
|
||||
|
||||
func (r *sshReachabilityWiringRepo) UpsertExecutorReachability(context.Context, models.ExecutorReachabilityObservation) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *sshReachabilityWiringRepo) ResetExecutorReachability(context.Context, string, string, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper mirrors
|
||||
// TestInitOfficeServicesUsesTaskUsageWiringHelper: the composition root
|
||||
// (startAgentInfrastructure, too large to exercise directly in a unit test)
|
||||
// must delegate to the narrow, independently-testable wiring helper exactly
|
||||
// once rather than inlining the poller construction.
|
||||
func TestStartAgentInfrastructureUsesSSHReachabilityPollerWiringHelper(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
file, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
|
||||
var target *ast.FuncDecl
|
||||
for _, decl := range file.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if ok && fn.Name.Name == "startAgentInfrastructure" {
|
||||
target = fn
|
||||
break
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
t.Fatal("startAgentInfrastructure not found in main.go")
|
||||
}
|
||||
|
||||
var helperCalls int
|
||||
ast.Inspect(target, func(node ast.Node) bool {
|
||||
call, ok := node.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if name, ok := call.Fun.(*ast.Ident); ok && name.Name == "startSSHReachabilityPoller" {
|
||||
helperCalls++
|
||||
}
|
||||
return true
|
||||
})
|
||||
if helperCalls != 1 {
|
||||
t.Fatalf("startSSHReachabilityPoller calls = %d, want exactly one composition call", helperCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartSSHReachabilityPoller_StartsPollerAndRegistersCleanup(t *testing.T) {
|
||||
repo := &sshReachabilityWiringRepo{listed: make(chan struct{}, 1)}
|
||||
var cleanups []func() error
|
||||
addCleanup := func(fn func() error) func() error {
|
||||
cleanups = append(cleanups, fn)
|
||||
return fn
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
poller := startSSHReachabilityPoller(ctx, repo, 3600, logger.Default(), (*reachabilitypkg.Publisher)(nil), addCleanup)
|
||||
if poller == nil {
|
||||
t.Fatal("startSSHReachabilityPoller returned a nil poller")
|
||||
}
|
||||
if len(cleanups) != 1 {
|
||||
t.Fatalf("cleanups registered = %d, want exactly 1", len(cleanups))
|
||||
}
|
||||
|
||||
select {
|
||||
case <-repo.listed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("poller never ran its immediate pass")
|
||||
}
|
||||
|
||||
if err := cleanups[0](); err != nil {
|
||||
t.Fatalf("cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,7 @@ var startupCatalog = []CatalogEntry{
|
||||
{Key: "launcher.webPort", EnvVars: []string{"KANDEV_WEB_PORT"}, Owner: "launcher", Default: "automatic"},
|
||||
{Key: "launcher.healthTimeoutMs", EnvVars: []string{"KANDEV_HEALTH_TIMEOUT_MS"}, Owner: "launcher", Default: "45000"},
|
||||
{Key: "launcher.noBrowser", EnvVars: []string{"KANDEV_NO_BROWSER"}, Owner: "launcher", Default: "false"},
|
||||
{Key: "executors.sshReachabilityIntervalSeconds", EnvVars: []string{"KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS"}, Owner: "executors/reachability", Default: "60"},
|
||||
}
|
||||
|
||||
var startupExclusions = []CatalogExclusion{
|
||||
|
||||
@@ -134,6 +134,7 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment {
|
||||
{envVar: "KANDEV_WEB_PORT", class: "catalog"},
|
||||
{envVar: "KANDEV_HEALTH_TIMEOUT_MS", class: "catalog"},
|
||||
{envVar: "KANDEV_NO_BROWSER", class: "catalog"},
|
||||
{envVar: "KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", class: "catalog"},
|
||||
{envVar: InternalConfigFileEnv, class: "exclusion"},
|
||||
{envVar: InternalConfigHomeFileEnv, class: "exclusion"},
|
||||
{envVar: InternalAgentctlStartupConfigEnv, class: "exclusion"},
|
||||
|
||||
@@ -53,6 +53,7 @@ type Config struct {
|
||||
Office OfficeConfig `mapstructure:"office"`
|
||||
Features FeaturesConfig `mapstructure:"features"`
|
||||
GitHubCredentialBroker GitHubCredentialBrokerConfig `mapstructure:"githubCredentialBroker"`
|
||||
Executors ExecutorsConfig `mapstructure:"executors"`
|
||||
Source ConfigSource `mapstructure:"-" json:"-"`
|
||||
}
|
||||
|
||||
@@ -71,6 +72,15 @@ type CredentialsConfig struct {
|
||||
File string `mapstructure:"file"`
|
||||
}
|
||||
|
||||
// ExecutorsConfig contains executor-related startup settings.
|
||||
type ExecutorsConfig struct {
|
||||
// SSHReachabilityIntervalSeconds is the raw configured value, not yet
|
||||
// clamped into the reachability package's own 15-3600 bound: 0 disables
|
||||
// the poller, and reachability.ClampInterval owns the rest of the
|
||||
// normalization at construction time.
|
||||
SSHReachabilityIntervalSeconds int `mapstructure:"sshReachabilityIntervalSeconds"`
|
||||
}
|
||||
|
||||
// LimitsConfig contains process and protocol capacity limits.
|
||||
type LimitsConfig struct {
|
||||
GHMaxConcurrent int `mapstructure:"ghMaxConcurrent"`
|
||||
|
||||
@@ -65,28 +65,29 @@ type configSelection struct {
|
||||
// AutomaticEnv during Unmarshal: legacy invalid environment values must retain
|
||||
// their fallback behavior instead of turning into typed-config startup errors.
|
||||
var yamlOnlyStartupKeys = map[string]struct{}{
|
||||
"server.trustedProxies": {},
|
||||
"tasks.preparationTimeout": {},
|
||||
"tasks.stallDetectionThreshold": {},
|
||||
"credentials.file": {},
|
||||
"limits.ghMaxConcurrent": {},
|
||||
"limits.gitMaxConcurrent": {},
|
||||
"limits.lspMaxConnections": {},
|
||||
"messageQueue.maxPerSession": {},
|
||||
"agentctl.idleTimeout": {},
|
||||
"agentctl.idleReaperInterval": {},
|
||||
"agentctl.notificationQueueCapacity": {},
|
||||
"agentctl.recoveryDeadline": {},
|
||||
"agentctl.recoveryReadTimeout": {},
|
||||
"agentctl.recoveryReadRetries": {},
|
||||
"agentctl.unownedPeriod": {},
|
||||
"agentctl.detachedEventLimit": {},
|
||||
"planning.coalesceWindowMs": {},
|
||||
"office.schedulerTickMs": {},
|
||||
"observability.otlpEndpoint": {},
|
||||
"launcher.webPort": {},
|
||||
"launcher.healthTimeoutMs": {},
|
||||
"launcher.noBrowser": {},
|
||||
"server.trustedProxies": {},
|
||||
"tasks.preparationTimeout": {},
|
||||
"tasks.stallDetectionThreshold": {},
|
||||
"credentials.file": {},
|
||||
"limits.ghMaxConcurrent": {},
|
||||
"limits.gitMaxConcurrent": {},
|
||||
"limits.lspMaxConnections": {},
|
||||
"messageQueue.maxPerSession": {},
|
||||
"agentctl.idleTimeout": {},
|
||||
"agentctl.idleReaperInterval": {},
|
||||
"agentctl.notificationQueueCapacity": {},
|
||||
"agentctl.recoveryDeadline": {},
|
||||
"agentctl.recoveryReadTimeout": {},
|
||||
"agentctl.recoveryReadRetries": {},
|
||||
"agentctl.unownedPeriod": {},
|
||||
"agentctl.detachedEventLimit": {},
|
||||
"planning.coalesceWindowMs": {},
|
||||
"office.schedulerTickMs": {},
|
||||
"observability.otlpEndpoint": {},
|
||||
"launcher.webPort": {},
|
||||
"launcher.healthTimeoutMs": {},
|
||||
"launcher.noBrowser": {},
|
||||
"executors.sshReachabilityIntervalSeconds": {},
|
||||
}
|
||||
|
||||
func isYAMLOnlyStartupKey(key string) bool {
|
||||
@@ -313,6 +314,7 @@ func applyStartupDefaults(cfg *Config, yamlKeys map[string]bool, profileDefaults
|
||||
if !yamlKeys["launcher.noBrowser"] {
|
||||
cfg.Launcher.NoBrowser = false
|
||||
}
|
||||
setDefaultInt("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60)
|
||||
}
|
||||
|
||||
func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources map[string]SettingSource) {
|
||||
@@ -343,6 +345,7 @@ func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources
|
||||
applyBoundedIntEnv("launcher.webPort", &cfg.Launcher.WebPort, 0, 1, 65535, envSnapshot, sources)
|
||||
applyPositiveIntEnv("launcher.healthTimeoutMs", &cfg.Launcher.HealthTimeoutMs, launcherHealthTimeoutDefault(), envSnapshot, sources)
|
||||
applyBoolEnv("launcher.noBrowser", &cfg.Launcher.NoBrowser, false, envSnapshot, sources)
|
||||
applyNonNegativeIntEnv("executors.sshReachabilityIntervalSeconds", &cfg.Executors.SSHReachabilityIntervalSeconds, 60, envSnapshot, sources)
|
||||
}
|
||||
|
||||
func launcherHealthTimeoutDefault() int {
|
||||
|
||||
@@ -307,6 +307,74 @@ func TestHomeConfigInvalidFirstDoesNotFallThrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
|
||||
//
|
||||
// The catalog's own fallback-and-bounds logic only ever falls back to the
|
||||
// default (60): the 0-disables/15-3600-clamp behavior belongs to the
|
||||
// reachability package's own ClampInterval, not this layer, per the design's
|
||||
// split between catalog-level normalization and package-level clamping.
|
||||
func TestExecutorsSSHReachabilityIntervalSecondsEnvironment(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env string
|
||||
wantValue int
|
||||
wantSource SettingSource
|
||||
}{
|
||||
{"absent uses default", "", 60, SourceDefault},
|
||||
{"valid value passes through unclamped", "45", 45, SourceEnvironment},
|
||||
{"zero passes through as the disable sentinel", "0", 0, SourceEnvironment},
|
||||
{"negative falls back to the default", "-5", 60, SourceDefault},
|
||||
{"unparsable falls back to the default", "not-a-number", 60, SourceDefault},
|
||||
{"a value outside 15-3600 still passes through unclamped here", "9999", 9999, SourceEnvironment},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.env != "" {
|
||||
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", tt.env)
|
||||
} else {
|
||||
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "")
|
||||
}
|
||||
t.Setenv("KANDEV_SERVER_PORT", "")
|
||||
|
||||
cfg, err := LoadWithPath(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithPath: %v", err)
|
||||
}
|
||||
if cfg.Executors.SSHReachabilityIntervalSeconds != tt.wantValue {
|
||||
t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want %d", cfg.Executors.SSHReachabilityIntervalSeconds, tt.wantValue)
|
||||
}
|
||||
if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != tt.wantSource {
|
||||
t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, tt.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
|
||||
//
|
||||
// A YAML value is not passed through the environment's fallback-and-bounds
|
||||
// logic at all — it decodes straight into the typed field, negative values
|
||||
// included, leaving ClampInterval as the sole place that normalizes it.
|
||||
func TestExecutorsSSHReachabilityIntervalSecondsYAMLPassesThroughUnclamped(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte("executors:\n sshReachabilityIntervalSeconds: -5\n"), 0o600); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
t.Setenv("KANDEV_EXECUTORS_SSHREACHABILITYINTERVALSECONDS", "")
|
||||
t.Setenv("KANDEV_SERVER_PORT", "")
|
||||
|
||||
cfg, err := LoadWithPath(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithPath: %v", err)
|
||||
}
|
||||
if cfg.Executors.SSHReachabilityIntervalSeconds != -5 {
|
||||
t.Fatalf("executors.sshReachabilityIntervalSeconds = %d, want -5 (unclamped)", cfg.Executors.SSHReachabilityIntervalSeconds)
|
||||
}
|
||||
if got := cfg.SourceFor("executors.sshReachabilityIntervalSeconds"); got != SourceConfiguration {
|
||||
t.Fatalf("executors.sshReachabilityIntervalSeconds source = %q, want %q", got, SourceConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
func nestedField(t *testing.T, value any, names ...string) reflect.Value {
|
||||
t.Helper()
|
||||
current := reflect.ValueOf(value)
|
||||
|
||||
@@ -192,6 +192,10 @@ const (
|
||||
ExecutorCreated = "executor.created"
|
||||
ExecutorUpdated = "executor.updated"
|
||||
ExecutorDeleted = "executor.deleted"
|
||||
// ExecutorReachabilityChanged is published only when a probe or a
|
||||
// connection-configuration reset actually changes the stored state or
|
||||
// reason — a steady host never publishes.
|
||||
ExecutorReachabilityChanged = "executor.reachability.changed"
|
||||
)
|
||||
|
||||
// Event types for executor profiles
|
||||
@@ -310,6 +314,16 @@ const (
|
||||
AvailableCommandsUpdated = "available_commands.updated" // Available slash commands updated
|
||||
)
|
||||
|
||||
// Event types for session launch warnings
|
||||
const (
|
||||
// SessionLaunchWarning is published once, immediately before an SSH
|
||||
// launch's CreateInstance call, when the target executor's stored
|
||||
// reachability record is unreachable and the record is still within the
|
||||
// probing window. It carries Kandev's own attribution of the target
|
||||
// host, independent of whatever an agent process itself reports.
|
||||
SessionLaunchWarning = "session.launch.warning"
|
||||
)
|
||||
|
||||
// Event types for session mode
|
||||
const (
|
||||
SessionModeChanged = "session_mode.changed" // Agent session mode changed
|
||||
@@ -512,6 +526,16 @@ func BuildAgentCapabilitiesSubject(sessionID string) string {
|
||||
return AgentCapabilitiesUpdated + "." + sessionID
|
||||
}
|
||||
|
||||
// BuildSessionLaunchWarningSubject creates a session launch warning subject for a specific session
|
||||
func BuildSessionLaunchWarningSubject(sessionID string) string {
|
||||
return SessionLaunchWarning + "." + sessionID
|
||||
}
|
||||
|
||||
// BuildSessionLaunchWarningWildcardSubject creates a wildcard subscription for all session launch warning events
|
||||
func BuildSessionLaunchWarningWildcardSubject() string {
|
||||
return SessionLaunchWarning + ".*"
|
||||
}
|
||||
|
||||
// BuildAgentCapabilitiesWildcardSubject creates a wildcard subscription for all agent capabilities events
|
||||
func BuildAgentCapabilitiesWildcardSubject() string {
|
||||
return AgentCapabilitiesUpdated + ".*"
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
// TestMain asserts that no goroutines from this package outlive the test
|
||||
// process. The Poller owns the scheduled loop plus any off-cycle ProbeNow
|
||||
// call, all registered on one WaitGroup — goleak catches a Start path that
|
||||
// forgets to register, or a Stop path that returns before every probe drains.
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Package reachability sweeps every eligible SSH executor on a configurable
|
||||
// interval, probing each with bounded concurrency and persisting the result
|
||||
// through a hysteresis-owning write path. See
|
||||
// docs/specs/executors/system-design/ssh-reachability.md.
|
||||
package reachability
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
// probeTimeout bounds a single probe's TCP dial and SSH handshake.
|
||||
// sshDialTimeout is 30s; a shorter bound here keeps one hung host from
|
||||
// holding a semaphore slot for half the interval.
|
||||
probeTimeout = 10 * time.Second
|
||||
// passConcurrency bounds simultaneous outbound connections and file
|
||||
// descriptors while keeping a pass short.
|
||||
passConcurrency = 4
|
||||
// failureThreshold is the smallest value that suppresses a single-probe
|
||||
// blip; worst-case detection is two intervals.
|
||||
failureThreshold = 2
|
||||
|
||||
// DefaultIntervalSeconds is the effective interval used when the
|
||||
// configured value is absent, negative, fractional, or unparsable.
|
||||
DefaultIntervalSeconds = 60
|
||||
// MinIntervalSeconds and MaxIntervalSeconds bound the poller's cadence.
|
||||
// 0 is a separate "disabled" sentinel and is never clamped into this
|
||||
// range.
|
||||
MinIntervalSeconds = 15
|
||||
MaxIntervalSeconds = 3600
|
||||
)
|
||||
|
||||
// ClampInterval applies this package's bounds to a raw configured interval
|
||||
// (in seconds) and reports whether the value was adjusted — worth one
|
||||
// startup log line rather than a refused boot. The environment path in
|
||||
// internal/common/config already normalizes an absent, empty, fractional, or
|
||||
// unparsable value to DefaultIntervalSeconds; a YAML value reaches here
|
||||
// unnormalized; either way this function still owns the below-minimum,
|
||||
// above-maximum, and negative cases directly.
|
||||
func ClampInterval(seconds int) (effective int, clamped bool) {
|
||||
switch {
|
||||
case seconds == 0:
|
||||
return 0, false
|
||||
case seconds < 0:
|
||||
return DefaultIntervalSeconds, true
|
||||
case seconds < MinIntervalSeconds:
|
||||
return MinIntervalSeconds, true
|
||||
case seconds > MaxIntervalSeconds:
|
||||
return MaxIntervalSeconds, true
|
||||
default:
|
||||
return seconds, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package reachability
|
||||
|
||||
import "testing"
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.23, AC-EXECUTORS-SSH-REACHABILITY-001.24
|
||||
func TestClampInterval(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in int
|
||||
wantEffect int
|
||||
wantClamped bool
|
||||
}{
|
||||
{"zero disables and is not a clamp", 0, 0, false},
|
||||
{"negative falls back to the default", -5, DefaultIntervalSeconds, true},
|
||||
{"below minimum clamps up to 15", 1, MinIntervalSeconds, true},
|
||||
{"just below minimum clamps up to 15", 14, MinIntervalSeconds, true},
|
||||
{"minimum passes through unchanged", 15, 15, false},
|
||||
{"default passes through unchanged", 60, 60, false},
|
||||
{"maximum passes through unchanged", 3600, 3600, false},
|
||||
{"just above maximum clamps down to 3600", 3601, MaxIntervalSeconds, true},
|
||||
{"far above maximum clamps down to 3600", 100000, MaxIntervalSeconds, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
effective, clamped := ClampInterval(tt.in)
|
||||
if effective != tt.wantEffect || clamped != tt.wantClamped {
|
||||
t.Errorf("ClampInterval(%d) = (%d, %v), want (%d, %v)", tt.in, effective, clamped, tt.wantEffect, tt.wantClamped)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"expvar"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// expvar counters exposed via /debug/vars, following the office_stall_* and
|
||||
// routing_* precedent: "key=value;key=value..." labels so a Prometheus
|
||||
// translation layer can split on ';' and '='. Every name carries the
|
||||
// executor_ssh_reachability_ prefix.
|
||||
var (
|
||||
probeTotal = expvar.NewMap("executor_ssh_reachability_probe_total")
|
||||
stateTransitionsTotal = expvar.NewMap("executor_ssh_reachability_state_transitions_total")
|
||||
passSkippedTotal = expvar.NewInt("executor_ssh_reachability_pass_skipped_total")
|
||||
writeRefusedTotal = expvar.NewInt("executor_ssh_reachability_write_refused_total")
|
||||
readBackFailedTotal = expvar.NewInt("executor_ssh_reachability_read_back_failed_total")
|
||||
publishFailedTotal = expvar.NewInt("executor_ssh_reachability_publish_failed_total")
|
||||
resetTotal = expvar.NewInt("executor_ssh_reachability_reset_total")
|
||||
probeDiscardedTotal = expvar.NewInt("executor_ssh_reachability_probe_discarded_total")
|
||||
probeDurationMsLastRun = expvar.NewInt("executor_ssh_reachability_probe_duration_ms")
|
||||
)
|
||||
|
||||
// reachabilityLabel builds a "k1=v1;k2=v2;..." expvar map key. Returns an
|
||||
// empty label for an odd number of arguments rather than guessing.
|
||||
func reachabilityLabel(pairs ...string) string {
|
||||
if len(pairs)%2 != 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(pairs)/2)
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
parts = append(parts, pairs[i]+"="+pairs[i+1])
|
||||
}
|
||||
return strings.Join(parts, ";")
|
||||
}
|
||||
|
||||
// recordProbeOutcome increments probeTotal labelled by outcome: "reachable"
|
||||
// on success, or the failure reason string otherwise.
|
||||
func recordProbeOutcome(success bool, reason string) {
|
||||
outcome := reason
|
||||
if success {
|
||||
outcome = "reachable"
|
||||
}
|
||||
probeTotal.Add(reachabilityLabel("outcome", outcome), 1)
|
||||
}
|
||||
|
||||
// recordStateTransition increments stateTransitionsTotal labelled by the
|
||||
// destination state.
|
||||
func recordStateTransition(destination string) {
|
||||
stateTransitionsTotal.Add(reachabilityLabel("state", destination), 1)
|
||||
}
|
||||
|
||||
// RecordReset increments resetTotal. This package never calls
|
||||
// ResetExecutorReachability itself — task 04's save observer is the only
|
||||
// caller of that method — but the counter is exported here so it lives with
|
||||
// the rest of this package's reachability metrics rather than being
|
||||
// duplicated at the call site.
|
||||
func RecordReset() {
|
||||
resetTotal.Add(1)
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// probeFunc is the injectable seam over a single probe attempt. defaultProbe
|
||||
// is the production implementation; tests substitute a fake so poller
|
||||
// lifecycle/concurrency behavior can be verified under testing/synctest
|
||||
// without real SSH or network I/O.
|
||||
type probeFunc func(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome
|
||||
|
||||
// Poller sweeps every eligible SSH executor on intervalSeconds, probing each
|
||||
// with bounded concurrency and persisting results through store. An
|
||||
// intervalSeconds of 0 disables the scheduled sweep entirely — the poller
|
||||
// still starts (so ProbeNow keeps working) but never runs a pass on its own.
|
||||
//
|
||||
// Start and Stop are idempotent. Stop cancels every in-flight unit of work
|
||||
// registered on wg — the scheduled loop, the pass it is running, and any
|
||||
// off-cycle ProbeNow call — and waits for all of them to drain before
|
||||
// returning, so a probe cancelled by Stop is guaranteed to never be
|
||||
// persisted (see probeAndPersist).
|
||||
type Poller struct {
|
||||
store *store
|
||||
log *logger.Logger
|
||||
intervalSeconds int
|
||||
probe probeFunc
|
||||
// publisher is optional: nil until SetPublisher wires the event bus, so
|
||||
// task 03's own tests and callers that never call it keep working
|
||||
// unchanged. Set once at construction time in production.
|
||||
publisher *Publisher
|
||||
|
||||
// inflight coalesces concurrent ProbeAndWait callers for the same
|
||||
// executor id into a single dial. Zero value is ready to use.
|
||||
inflight singleflight.Group
|
||||
|
||||
// probeSem is owned by the poller rather than by a scheduled pass. This
|
||||
// keeps scheduled, manual, and synchronous probes under one cap.
|
||||
probeSem chan struct{}
|
||||
|
||||
// mu guards started/stopping/ctx/cancel/wg against concurrent
|
||||
// Start/Stop/ProbeNow calls. acquire() registers new work on wg under mu
|
||||
// so it can never race a concurrent Stop's wg.Wait() observing the
|
||||
// counter transition from zero.
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
stopping bool
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
stopDone chan struct{}
|
||||
|
||||
// passMu guards passRunning, the single-flight gate for the scheduled
|
||||
// pass: an overlapping tick is dropped and counted, never queued.
|
||||
passMu sync.Mutex
|
||||
passRunning bool
|
||||
}
|
||||
|
||||
// New builds a Poller. intervalSeconds is clamped via ClampInterval; a
|
||||
// clamp is logged once rather than refusing to start.
|
||||
func New(repo Repository, intervalSeconds int, log *logger.Logger) *Poller {
|
||||
effective, clamped := ClampInterval(intervalSeconds)
|
||||
if clamped {
|
||||
log.Warn("executor ssh reachability: interval clamped",
|
||||
zap.Int("configured", intervalSeconds), zap.Int("effective", effective))
|
||||
}
|
||||
return &Poller{
|
||||
store: &store{repo: repo, log: log},
|
||||
log: log,
|
||||
intervalSeconds: effective,
|
||||
probe: defaultProbe,
|
||||
probeSem: make(chan struct{}, passConcurrency),
|
||||
}
|
||||
}
|
||||
|
||||
// SetPublisher wires the event publisher used to announce a state/reason
|
||||
// change. Optional — a Poller with no publisher still probes and persists
|
||||
// exactly as before, it just never publishes.
|
||||
func (p *Poller) SetPublisher(publisher *Publisher) {
|
||||
p.publisher = publisher
|
||||
}
|
||||
|
||||
// Start launches the background loop. Calling Start more than once without
|
||||
// an intervening Stop is a no-op.
|
||||
func (p *Poller) Start(ctx context.Context) {
|
||||
p.mu.Lock()
|
||||
if p.started {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
p.started = true
|
||||
p.stopping = false
|
||||
p.ctx = runCtx
|
||||
p.cancel = cancel
|
||||
p.stopDone = make(chan struct{})
|
||||
if p.intervalSeconds > 0 {
|
||||
p.wg.Add(1)
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.intervalSeconds > 0 {
|
||||
go p.loop(runCtx)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop cancels the loop and every in-flight probe, then waits for all of
|
||||
// them to drain. Calling Stop before Start, or twice in a row, is a no-op.
|
||||
func (p *Poller) Stop() {
|
||||
p.mu.Lock()
|
||||
if !p.started {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
stopDone := p.stopDone
|
||||
if p.stopping {
|
||||
p.mu.Unlock()
|
||||
<-stopDone
|
||||
return
|
||||
}
|
||||
p.stopping = true
|
||||
cancel := p.cancel
|
||||
p.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
p.wg.Wait()
|
||||
|
||||
p.mu.Lock()
|
||||
p.started = false
|
||||
p.stopping = false
|
||||
p.ctx = nil
|
||||
p.cancel = nil
|
||||
if p.stopDone == stopDone {
|
||||
close(stopDone)
|
||||
p.stopDone = nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// ProbeNow runs one off-cycle probe for executor, outside the scheduled
|
||||
// cadence, tracked on the same WaitGroup/context as the scheduled loop so
|
||||
// Stop drains it too. Returns false without doing anything when the poller
|
||||
// is not currently accepting new work.
|
||||
func (p *Poller) ProbeNow(executor *models.Executor) bool {
|
||||
ctx, ok := p.acquire()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
p.probeAndPersist(ctx, executor)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Poller) acquireProbeSlot(ctx context.Context) bool {
|
||||
select {
|
||||
case p.probeSem <- struct{}{}:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) releaseProbeSlot() {
|
||||
<-p.probeSem
|
||||
}
|
||||
|
||||
// acquire registers one unit of in-flight work on wg, gated by mu so it can
|
||||
// never race a concurrent Stop's wg.Wait(). It refuses when the poller isn't
|
||||
// started or a Stop is already underway.
|
||||
func (p *Poller) acquire() (context.Context, bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.started || p.stopping {
|
||||
return nil, false
|
||||
}
|
||||
// loop already owns a WaitGroup reference, so this positive Add cannot
|
||||
// race a Stop waiting for the counter to reach zero.
|
||||
// Safe without passMu: dispatchPass is called only from loop, which holds
|
||||
// its own WaitGroup slot, so Stop cannot observe zero before this Add.
|
||||
p.wg.Add(1)
|
||||
return p.ctx, true
|
||||
}
|
||||
|
||||
func (p *Poller) loop(ctx context.Context) {
|
||||
defer p.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(p.intervalSeconds) * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
p.dispatchPass(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.dispatchPass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchPass starts one pass if none is currently running. An overlapping
|
||||
// dispatch is dropped and counted rather than queued, so a slow pass never
|
||||
// builds up a backlog of pending passes. The pass itself runs on its own
|
||||
// WaitGroup unit so Stop drains it even after loop itself has returned.
|
||||
func (p *Poller) dispatchPass(ctx context.Context) {
|
||||
p.passMu.Lock()
|
||||
if p.passRunning {
|
||||
p.passMu.Unlock()
|
||||
passSkippedTotal.Add(1)
|
||||
return
|
||||
}
|
||||
p.passRunning = true
|
||||
p.passMu.Unlock()
|
||||
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
defer func() {
|
||||
p.passMu.Lock()
|
||||
p.passRunning = false
|
||||
p.passMu.Unlock()
|
||||
}()
|
||||
p.runPass(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// runPass lists every eligible executor and probes each with bounded
|
||||
// concurrency. A listing failure abandons the pass without writing anything.
|
||||
// Cancellation stops dispatching new probes and waits only for the ones
|
||||
// already started to finish or self-cancel.
|
||||
func (p *Poller) runPass(ctx context.Context) {
|
||||
start := time.Now()
|
||||
executors, err := p.store.repo.ListSSHExecutorsForReachability(ctx)
|
||||
if err != nil {
|
||||
p.log.Warn("executor ssh reachability: list failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, executor := range executors {
|
||||
if !p.acquireProbeSlot(ctx) {
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(executor *models.Executor) {
|
||||
defer wg.Done()
|
||||
defer p.releaseProbeSlot()
|
||||
p.probeAndPersistWithSlot(ctx, executor)
|
||||
}(executor)
|
||||
}
|
||||
wg.Wait()
|
||||
probeDurationMsLastRun.Set(time.Since(start).Milliseconds())
|
||||
}
|
||||
|
||||
// probeAndPersist runs one probe and records it, unless the probe was
|
||||
// cancelled — a cancelled outcome is not an observation about the host and
|
||||
// must never be written (see agentruntime.SSHProbeOutcome.Cancelled).
|
||||
func (p *Poller) probeAndPersist(ctx context.Context, executor *models.Executor) {
|
||||
if !p.acquireProbeSlot(ctx) {
|
||||
return
|
||||
}
|
||||
defer p.releaseProbeSlot()
|
||||
p.probeAndPersistWithSlot(ctx, executor)
|
||||
}
|
||||
|
||||
func (p *Poller) probeAndPersistWithSlot(ctx context.Context, executor *models.Executor) {
|
||||
outcome := p.probe(ctx, executor)
|
||||
if outcome.Cancelled {
|
||||
probeDiscardedTotal.Add(1)
|
||||
return
|
||||
}
|
||||
p.observeAndPublish(ctx, executor, outcome)
|
||||
}
|
||||
|
||||
// observeAndPublish records outcome through store.Observe and, when the
|
||||
// write actually changed the stored state or reason, announces it through
|
||||
// the publisher — the one propagation path every probe primitive
|
||||
// (scheduled pass, ProbeNow, ProbeAndWait) shares.
|
||||
func (p *Poller) observeAndPublish(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome) observeResult {
|
||||
result := p.store.Observe(ctx, executor, outcome, time.Now().UTC())
|
||||
if result.Changed && p.publisher != nil {
|
||||
p.publisher.PublishChanged(ctx, result.After, p.EffectiveIntervalSeconds())
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// A scheduled-pass probe (ProbeNow's own code path, probeAndPersist) must
|
||||
// publish exactly like the immediate-probe route on a real change, and stay
|
||||
// silent on a repeat that changes nothing — the "one propagation path" rule
|
||||
// from the system design applies regardless of which primitive ran the
|
||||
// probe.
|
||||
func TestPollerProbeNow_PublishesOnlyOnChange(t *testing.T) {
|
||||
eventBus := bus.NewMemoryEventBus(logger.Default())
|
||||
published := make(chan RecordDTO, 4)
|
||||
_, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
|
||||
published <- ev.Data.(RecordDTO)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.SetPublisher(NewPublisher(eventBus))
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
|
||||
executor := sshExecutor("exec-publish")
|
||||
|
||||
// First success: unknown -> reachable is a change.
|
||||
p.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
|
||||
}
|
||||
if !p.ProbeNow(executor) {
|
||||
t.Fatal("ProbeNow refused")
|
||||
}
|
||||
select {
|
||||
case dto := <-published:
|
||||
if dto.ExecutorID != executor.ID || dto.State != "reachable" {
|
||||
t.Fatalf("dto = %+v, want reachable exec-publish", dto)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no event published for the first, change-carrying probe")
|
||||
}
|
||||
|
||||
// Second success, same outcome: reachable -> reachable is not a change.
|
||||
if !p.ProbeNow(executor) {
|
||||
t.Fatal("ProbeNow refused")
|
||||
}
|
||||
// Stop drains the second ProbeNow's probeAndPersist goroutine (it is
|
||||
// registered on the same WaitGroup) before we check for the absence of a
|
||||
// publish — a happens-before barrier instead of a raw sleep.
|
||||
p.Stop()
|
||||
select {
|
||||
case dto := <-published:
|
||||
t.Fatalf("unexpected event published for a repeat outcome: %+v", dto)
|
||||
default:
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// fakeRepository is an in-memory Repository double for poller tests: it
|
||||
// isolates poller mechanics (scheduling, concurrency, lifecycle) from the
|
||||
// SQL-backed hysteresis contract, which store_test.go already covers
|
||||
// end-to-end against a real database.
|
||||
type fakeRepository struct {
|
||||
mu sync.Mutex
|
||||
executors []*models.Executor
|
||||
records map[string]*models.ExecutorReachability
|
||||
listErr error
|
||||
upsertErr error
|
||||
|
||||
upsertCalls int32
|
||||
}
|
||||
|
||||
// errRepoRefused simulates a repository-level write failure — a test double
|
||||
// for the last-write-wins WHERE clause silently refusing a write, without
|
||||
// needing a real SQLite race to reproduce it.
|
||||
var errRepoRefused = errors.New("write refused")
|
||||
|
||||
func newFakeRepository(executors ...*models.Executor) *fakeRepository {
|
||||
return &fakeRepository{executors: executors, records: map[string]*models.ExecutorReachability{}}
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
out := append([]*models.Executor(nil), f.executors...)
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
rec, ok := f.records[executorID]
|
||||
if !ok {
|
||||
return nil, models.ErrExecutorReachabilityNotFound
|
||||
}
|
||||
cp := *rec
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) UpsertExecutorReachability(_ context.Context, obs models.ExecutorReachabilityObservation) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
atomic.AddInt32(&f.upsertCalls, 1)
|
||||
if f.upsertErr != nil {
|
||||
return f.upsertErr
|
||||
}
|
||||
state := obs.InitialState
|
||||
failures := obs.InitialFailures
|
||||
if existing, ok := f.records[obs.ExecutorID]; ok {
|
||||
if obs.Reason == "" {
|
||||
failures = 0
|
||||
state = models.ExecutorReachabilityStateReachable
|
||||
} else {
|
||||
failures = existing.ConsecutiveFailures + 1
|
||||
state = existing.State
|
||||
if isStickyReason(obs.Reason) || failures >= obs.FailureThreshold {
|
||||
state = models.ExecutorReachabilityStateUnreachable
|
||||
}
|
||||
}
|
||||
}
|
||||
f.records[obs.ExecutorID] = &models.ExecutorReachability{
|
||||
ExecutorID: obs.ExecutorID, State: state, Reason: obs.Reason, Message: obs.Message,
|
||||
ConsecutiveFailures: failures, Host: obs.Host, CheckedAt: timePtr(obs.CheckedAt), UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) ResetExecutorReachability(_ context.Context, executorID, host string, _ time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.records[executorID] = &models.ExecutorReachability{ExecutorID: executorID, State: models.ExecutorReachabilityStateUnknown, Host: host, UpdatedAt: time.Now().UTC()}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRepository) upsertCount() int32 { return atomic.LoadInt32(&f.upsertCalls) }
|
||||
|
||||
func timePtr(t time.Time) *time.Time { return &t }
|
||||
|
||||
func sshExecutor(id string) *models.Executor {
|
||||
return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive, Config: map[string]string{"ssh_host": id}}
|
||||
}
|
||||
|
||||
// concurrencyProbe blocks on a gate until released, tracking the maximum
|
||||
// number of simultaneous in-flight calls it observed.
|
||||
type concurrencyProbe struct {
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
maxSeen int
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func newConcurrencyProbe() *concurrencyProbe {
|
||||
return &concurrencyProbe{release: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (c *concurrencyProbe) probe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
c.mu.Lock()
|
||||
c.inFlight++
|
||||
if c.inFlight > c.maxSeen {
|
||||
c.maxSeen = c.inFlight
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-c.release:
|
||||
case <-ctx.Done():
|
||||
c.mu.Lock()
|
||||
c.inFlight--
|
||||
c.mu.Unlock()
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Cancelled: true}
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.inFlight--
|
||||
c.mu.Unlock()
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
func TestPollerStart_RunsImmediatePass(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"), sshExecutor("b"))
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
defer p.Stop()
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
if got := repo.upsertCount(); got != 2 {
|
||||
t.Fatalf("upsert calls = %d, want 2 (one immediate pass over two executors)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerStart_IsIdempotent(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
p.Start(ctx) // second call must be a no-op
|
||||
|
||||
synctest.Wait()
|
||||
p.Stop()
|
||||
|
||||
if got := repo.upsertCount(); got != 1 {
|
||||
t.Fatalf("upsert calls = %d, want exactly 1 (a second Start must not spawn a parallel loop)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerStop_BeforeStart_IsNoOp(t *testing.T) {
|
||||
p := New(newFakeRepository(), 60, logger.Default())
|
||||
p.Stop() // must not block or panic
|
||||
}
|
||||
|
||||
func TestPollerZeroInterval_DisablesScheduledPass(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 0, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
defer p.Stop()
|
||||
|
||||
synctest.Wait()
|
||||
// Advance well past what would have been several ticks at the
|
||||
// default interval, to prove no pass ever runs.
|
||||
time.Sleep(10 * time.Minute)
|
||||
synctest.Wait()
|
||||
|
||||
if got := repo.upsertCount(); got != 0 {
|
||||
t.Fatalf("upsert calls = %d, want 0 (interval=0 must run no scheduled pass)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerZeroInterval_ProbeNowStillWorks(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 0, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
defer p.Stop()
|
||||
synctest.Wait()
|
||||
|
||||
if ok := p.ProbeNow(sshExecutor("a")); !ok {
|
||||
t.Fatal("ProbeNow returned false while the poller is started")
|
||||
}
|
||||
synctest.Wait()
|
||||
|
||||
if got := repo.upsertCount(); got != 1 {
|
||||
t.Fatalf("upsert calls = %d, want 1 from the off-cycle probe", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerPass_NeverExceedsPassConcurrency(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var executors []*models.Executor
|
||||
for i := 0; i < passConcurrency+3; i++ {
|
||||
executors = append(executors, sshExecutor(string(rune('a'+i))))
|
||||
}
|
||||
repo := newFakeRepository(executors...)
|
||||
p := New(repo, 60, logger.Default())
|
||||
cp := newConcurrencyProbe()
|
||||
p.probe = cp.probe
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
|
||||
// Let every worker reach the gate.
|
||||
synctest.Wait()
|
||||
close(cp.release)
|
||||
p.Stop()
|
||||
|
||||
cp.mu.Lock()
|
||||
defer cp.mu.Unlock()
|
||||
if cp.maxSeen > passConcurrency {
|
||||
t.Fatalf("max simultaneous probes = %d, want at most passConcurrency (%d)", cp.maxSeen, passConcurrency)
|
||||
}
|
||||
if cp.maxSeen != passConcurrency {
|
||||
t.Fatalf("max simultaneous probes = %d, want exactly passConcurrency (%d) since more executors than slots were queued", cp.maxSeen, passConcurrency)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerProbeEntrypointsShareConcurrencyLimit(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var executors []*models.Executor
|
||||
for i := 0; i < passConcurrency; i++ {
|
||||
executors = append(executors, sshExecutor(string(rune('a'+i))))
|
||||
}
|
||||
repo := newFakeRepository(executors...)
|
||||
p := New(repo, 60, logger.Default())
|
||||
cp := newConcurrencyProbe()
|
||||
p.probe = cp.probe
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
synctest.Wait()
|
||||
|
||||
if ok := p.ProbeNow(sshExecutor("manual")); !ok {
|
||||
t.Fatal("ProbeNow returned false while the poller was running")
|
||||
}
|
||||
synctest.Wait()
|
||||
|
||||
cp.mu.Lock()
|
||||
maxSeen := cp.maxSeen
|
||||
cp.mu.Unlock()
|
||||
close(cp.release)
|
||||
p.Stop()
|
||||
if maxSeen > passConcurrency {
|
||||
t.Fatalf("max simultaneous probes = %d, want at most %d across scheduled and manual probes", maxSeen, passConcurrency)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerConcurrentStopWaitsForTheFirstStop(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 0, logger.Default())
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
var inFlight atomic.Int32
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
inFlight.Add(1)
|
||||
defer inFlight.Add(-1)
|
||||
<-release
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.ID}
|
||||
}
|
||||
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
||||
p.Start(context.Background())
|
||||
if ok := p.ProbeNow(sshExecutor("stop-race")); !ok {
|
||||
t.Fatal("ProbeNow returned false while the poller was running")
|
||||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if inFlight.Load() == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
firstDone := make(chan struct{})
|
||||
go func() { p.Stop(); close(firstDone) }()
|
||||
for time.Now().Before(deadline) {
|
||||
p.mu.Lock()
|
||||
stopping := p.stopping
|
||||
p.mu.Unlock()
|
||||
if stopping {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
secondDone := make(chan struct{})
|
||||
go func() { p.Stop(); close(secondDone) }()
|
||||
select {
|
||||
case <-secondDone:
|
||||
t.Fatal("concurrent Stop returned before the first Stop drained the probe")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
select {
|
||||
case <-firstDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first Stop did not finish")
|
||||
}
|
||||
select {
|
||||
case <-secondDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("concurrent Stop did not finish after the first Stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollerPass_OverlappingTickIsSkippedNotQueued(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 60, logger.Default())
|
||||
cp := newConcurrencyProbe()
|
||||
p.probe = cp.probe
|
||||
|
||||
before := passSkippedTotal.Value()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
defer func() {
|
||||
close(cp.release)
|
||||
p.Stop()
|
||||
}()
|
||||
|
||||
// The immediate pass is now blocked on the probe gate. Advance past
|
||||
// a full tick while it is still running.
|
||||
synctest.Wait()
|
||||
time.Sleep(61 * time.Second)
|
||||
synctest.Wait()
|
||||
|
||||
if got := passSkippedTotal.Value() - before; got != 1 {
|
||||
t.Fatalf("pass_skipped_total delta = %d, want 1 (the overlapping tick must be dropped, not queued)", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerStop_DiscardsCancelledProbeResult(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 60, logger.Default())
|
||||
cp := newConcurrencyProbe()
|
||||
p.probe = cp.probe
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
|
||||
// The immediate pass is now blocked on the probe gate (never
|
||||
// released), so Stop must cancel it in flight.
|
||||
synctest.Wait()
|
||||
p.Stop()
|
||||
|
||||
if got := repo.upsertCount(); got != 0 {
|
||||
t.Fatalf("upsert calls = %d, want 0 — a probe cancelled by Stop must never be written", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPollerProbeNow_RefusedAfterStop(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
p.Stop()
|
||||
|
||||
if ok := p.ProbeNow(sshExecutor("a")); ok {
|
||||
t.Fatal("ProbeNow returned true after Stop; a stopped poller must refuse new work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollerListFailure_AbandonsPassWithoutWriting(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
repo := newFakeRepository(sshExecutor("a"))
|
||||
repo.listErr = context.DeadlineExceeded
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Start(ctx)
|
||||
defer p.Stop()
|
||||
synctest.Wait()
|
||||
|
||||
if got := repo.upsertCount(); got != 0 {
|
||||
t.Fatalf("upsert calls = %d, want 0 when listing executors fails", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// reachabilityProber is the production seam over the runtime's SSH
|
||||
// reachability prober. Tests substitute probeFunc directly, so this stays
|
||||
// unexported and unmocked.
|
||||
var reachabilityProber = agentruntime.NewSSHReachabilityProber()
|
||||
|
||||
// defaultProbe resolves executor's SSH target from its persisted config and
|
||||
// probes it. A resolution failure (missing host, invalid port, unresolvable
|
||||
// alias) never dials — it is reported as
|
||||
// agentruntime.SSHReachabilityReasonConfig directly, the same reason a
|
||||
// target with no pinned fingerprint uses.
|
||||
func defaultProbe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return reachabilityProber.Probe(ctx, executor.Config, probeTimeout)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// ProbeResult is what ProbeAndWait returns: the durable record when available,
|
||||
// or the locally projected observation when persistence failed, plus whether
|
||||
// that specific write actually landed.
|
||||
type ProbeResult struct {
|
||||
Record *models.ExecutorReachability
|
||||
Persisted bool
|
||||
}
|
||||
|
||||
// probeAndWaitOutcome is the value singleflight.Group shares across every
|
||||
// coalesced caller for one key. It is deliberately independent of any
|
||||
// caller's ProbeResult — ProbeAndWait copies out of it before returning, so
|
||||
// no caller can mutate what another caller already received.
|
||||
type probeAndWaitOutcome struct {
|
||||
ran bool
|
||||
persisted bool
|
||||
record *models.ExecutorReachability
|
||||
}
|
||||
|
||||
// ProbeAndWait runs one probe for executor synchronously and returns its
|
||||
// persisted record, coalescing overlapping callers for the same executor id
|
||||
// into a single dial via singleflight — the primitive behind the immediate-
|
||||
// probe HTTP route. Unlike ProbeNow, this does not accept (or run on) any
|
||||
// caller-supplied context: like every other off-cycle probe it runs on the
|
||||
// poller's own context/WaitGroup, registered through acquire(), so a
|
||||
// disconnecting HTTP caller neither cancels it nor affects a caller still
|
||||
// waiting on the same coalesced result, and Stop still drains it.
|
||||
//
|
||||
// ran is false only when the poller itself refused new work (not started, or
|
||||
// stopping); it does not report whether the probe outcome was a success —
|
||||
// see ProbeResult.Persisted for whether the result was actually stored.
|
||||
func (p *Poller) ProbeAndWait(executor *models.Executor) (ProbeResult, bool) {
|
||||
v, _, _ := p.inflight.Do(probeKey(executor), func() (any, error) {
|
||||
return p.probeAndWaitOnce(executor), nil
|
||||
})
|
||||
outcome := v.(probeAndWaitOutcome)
|
||||
if !outcome.ran {
|
||||
return ProbeResult{}, false
|
||||
}
|
||||
var record *models.ExecutorReachability
|
||||
if outcome.record != nil {
|
||||
cp := *outcome.record
|
||||
record = &cp
|
||||
}
|
||||
return ProbeResult{Record: record, Persisted: outcome.persisted}, true
|
||||
}
|
||||
|
||||
// probeKey includes the connection identity, not only the executor ID. A
|
||||
// save can change the host or its pinned key while an older probe is still in
|
||||
// flight; those observations must not share a singleflight result.
|
||||
func probeKey(executor *models.Executor) string {
|
||||
if executor == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
values := make([]string, 0, len(connectionConfigKeys)+1)
|
||||
values = append(values, executor.ID)
|
||||
for _, key := range connectionConfigKeys {
|
||||
values = append(values, key+"="+executor.Config[key])
|
||||
}
|
||||
return strings.Join(values, "\x00")
|
||||
}
|
||||
|
||||
// probeAndWaitOnce is the singleflight leader's body: acquire a WaitGroup
|
||||
// slot exactly like every other off-cycle unit of work, probe, and persist.
|
||||
func (p *Poller) probeAndWaitOnce(executor *models.Executor) probeAndWaitOutcome {
|
||||
ctx, ok := p.acquire()
|
||||
if !ok {
|
||||
return probeAndWaitOutcome{}
|
||||
}
|
||||
defer p.wg.Done()
|
||||
if !p.acquireProbeSlot(ctx) {
|
||||
return probeAndWaitOutcome{ran: true}
|
||||
}
|
||||
defer p.releaseProbeSlot()
|
||||
|
||||
outcome := p.probe(ctx, executor)
|
||||
if outcome.Cancelled {
|
||||
probeDiscardedTotal.Add(1)
|
||||
return probeAndWaitOutcome{ran: true}
|
||||
}
|
||||
result := p.observeAndPublish(ctx, executor, outcome)
|
||||
record := result.After
|
||||
if record == nil {
|
||||
record = result.Observed
|
||||
}
|
||||
return probeAndWaitOutcome{ran: true, persisted: result.Applied, record: record}
|
||||
}
|
||||
|
||||
// EffectiveIntervalSeconds returns the clamped interval this poller is
|
||||
// running with (0 when scheduled probing is disabled), for a caller (the
|
||||
// reachability DTO) that needs to report probing_enabled/
|
||||
// probe_interval_seconds without duplicating ClampInterval's rules.
|
||||
func (p *Poller) EffectiveIntervalSeconds() int {
|
||||
return p.intervalSeconds
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
|
||||
//
|
||||
// Two overlapping immediate-probe callers for the same executor must
|
||||
// coalesce into exactly one dial and receive one identical record each — the
|
||||
// acceptance criterion this task adds ProbeAndWait to satisfy. The blocking
|
||||
// probeFunc plus a short settle delay before releasing mirrors
|
||||
// golang.org/x/sync/singleflight's own TestDoDupSuppress: there is no faster
|
||||
// deterministic signal for "the second caller has entered Do" without
|
||||
// instrumenting production code for the test.
|
||||
func TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
|
||||
var dialCount int32
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
if atomic.AddInt32(&dialCount, 1) == 1 {
|
||||
close(started)
|
||||
}
|
||||
<-release
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
executor := sshExecutor("exec-coalesce")
|
||||
|
||||
type call struct {
|
||||
result ProbeResult
|
||||
ran bool
|
||||
}
|
||||
results := make(chan call, 2)
|
||||
go func() {
|
||||
result, ran := p.ProbeAndWait(executor)
|
||||
results <- call{result, ran}
|
||||
}()
|
||||
<-started
|
||||
go func() {
|
||||
result, ran := p.ProbeAndWait(executor)
|
||||
results <- call{result, ran}
|
||||
}()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
close(release)
|
||||
|
||||
first := <-results
|
||||
second := <-results
|
||||
|
||||
if got := atomic.LoadInt32(&dialCount); got != 1 {
|
||||
t.Fatalf("dial count = %d, want exactly 1 (coalesced)", got)
|
||||
}
|
||||
if !first.ran || !second.ran {
|
||||
t.Fatalf("ran = %v, %v, want both true", first.ran, second.ran)
|
||||
}
|
||||
firstJSON, _ := json.Marshal(first.result)
|
||||
secondJSON, _ := json.Marshal(second.result)
|
||||
if string(firstJSON) != string(secondJSON) {
|
||||
t.Fatalf("responses differ:\n first: %s\n second: %s", firstJSON, secondJSON)
|
||||
}
|
||||
if !first.result.Persisted {
|
||||
t.Fatalf("Persisted = false, want true — the probe outcome was a plain success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollerProbeAndWaitDoesNotCoalesceDifferentConnectionConfigs(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 0, logger.Default())
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
|
||||
var dialCount int32
|
||||
started := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
atomic.AddInt32(&dialCount, 1)
|
||||
started <- struct{}{}
|
||||
<-release
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
first := sshExecutor("same-id")
|
||||
first.Config["ssh_host"] = "host-a"
|
||||
second := sshExecutor("same-id")
|
||||
second.Config["ssh_host"] = "host-b"
|
||||
results := make(chan ProbeResult, 2)
|
||||
go func() { result, _ := p.ProbeAndWait(first); results <- result }()
|
||||
go func() { result, _ := p.ProbeAndWait(second); results <- result }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first probe did not start")
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("second connection config incorrectly coalesced with the first")
|
||||
}
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
<-results
|
||||
<-results
|
||||
if got := atomic.LoadInt32(&dialCount); got != 2 {
|
||||
t.Fatalf("dial count = %d, want 2 for distinct connection configs", got)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
|
||||
//
|
||||
// The singleflight group shares one result across coalesced callers. Each
|
||||
// caller must get its own copy of the record, not a shared pointer, so one
|
||||
// caller mutating its response cannot corrupt what the other already
|
||||
// received (the risk this task's work order names explicitly).
|
||||
func TestPollerProbeAndWait_ReturnsIndependentRecordCopies(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
executor := sshExecutor("exec-copy")
|
||||
result, ran := p.ProbeAndWait(executor)
|
||||
if !ran {
|
||||
t.Fatalf("ran = false, want true")
|
||||
}
|
||||
if result.Record == nil {
|
||||
t.Fatalf("Record = nil, want the stored record")
|
||||
}
|
||||
originalHost := result.Record.Host
|
||||
result.Record.Host = "mutated-in-caller"
|
||||
|
||||
second, ran := p.ProbeAndWait(executor)
|
||||
if !ran {
|
||||
t.Fatalf("second call: ran = false, want true")
|
||||
}
|
||||
if second.Record.Host == "mutated-in-caller" {
|
||||
t.Fatalf("second caller observed the first caller's mutation")
|
||||
}
|
||||
if second.Record.Host != originalHost {
|
||||
t.Fatalf("second.Record.Host = %q, want %q", second.Record.Host, originalHost)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.28
|
||||
//
|
||||
// The immediate probe must work while the scheduled poller is disabled
|
||||
// (interval 0) — ProbeAndWait must not depend on the loop ever having run.
|
||||
func TestPollerProbeAndWait_WorksWhilePollerDisabled(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 0, logger.Default())
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
result, ran := p.ProbeAndWait(sshExecutor("exec-disabled"))
|
||||
if !ran {
|
||||
t.Fatalf("ran = false, want true even though the scheduled poller is disabled")
|
||||
}
|
||||
if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("result = %+v, want a persisted reachable record", result)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.2
|
||||
//
|
||||
// A probe that ran but was not stored (the repository refused the write)
|
||||
// still reports ran=true with Persisted=false, never an error — the route
|
||||
// this backs answers 200 either way.
|
||||
func TestPollerProbeAndWait_WriteRefusedReportsPersistedFalse(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
repo.upsertErr = errRepoRefused
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.Start(context.Background())
|
||||
defer p.Stop()
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
result, ran := p.ProbeAndWait(sshExecutor("exec-refused"))
|
||||
if !ran {
|
||||
t.Fatalf("ran = false, want true")
|
||||
}
|
||||
if result.Persisted {
|
||||
t.Fatalf("Persisted = true, want false — the repository write failed")
|
||||
}
|
||||
if result.Record == nil || result.Record.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("Record = %+v, want the actual unsaved reachable observation", result.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPollerProbeAndWait_RefusedAfterStop mirrors
|
||||
// TestPollerProbeNow_RefusedAfterStop: once Stop has run, ProbeAndWait must
|
||||
// not dial at all.
|
||||
func TestPollerProbeAndWait_RefusedAfterStop(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
p := New(repo, 60, logger.Default())
|
||||
p.Start(context.Background())
|
||||
p.Stop()
|
||||
|
||||
dialed := false
|
||||
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
dialed = true
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.Config["ssh_host"]}
|
||||
}
|
||||
|
||||
_, ran := p.ProbeAndWait(sshExecutor("exec-stopped"))
|
||||
if ran {
|
||||
t.Fatalf("ran = true, want false — the poller was stopped")
|
||||
}
|
||||
if dialed {
|
||||
t.Fatalf("probe was dialed after Stop")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// RecordDTO is the wire shape every reachability HTTP route and the change
|
||||
// event share, so a client's WS handler and its GET/POST response handler
|
||||
// stay structurally identical. Reason is always present, even empty — the
|
||||
// surface's contract is that a reachable/unknown state can carry a non-empty
|
||||
// reason (a below-threshold failure), so an omitted field would be
|
||||
// ambiguous with "definitely no reason".
|
||||
type RecordDTO struct {
|
||||
ExecutorID string `json:"executor_id"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
Host string `json:"host,omitempty"`
|
||||
CheckedAt *time.Time `json:"checked_at"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at"`
|
||||
UpdatedAt *time.Time `json:"updated_at"`
|
||||
ProbingEnabled bool `json:"probing_enabled"`
|
||||
ProbeIntervalSeconds int `json:"probe_interval_seconds"`
|
||||
Persisted bool `json:"persisted"`
|
||||
}
|
||||
|
||||
// BuildRecordDTO projects record into the wire shape. A nil record (an
|
||||
// eligible executor never probed) synthesizes the "unknown" placeholder with
|
||||
// every timestamp null, per AC-EXECUTORS-SSH-REACHABILITY-002.5 — this is
|
||||
// never a 404, so every caller building a GET response for an eligible
|
||||
// executor can call this unconditionally.
|
||||
func BuildRecordDTO(executorID string, record *models.ExecutorReachability, effectiveIntervalSeconds int, persisted bool) RecordDTO {
|
||||
dto := RecordDTO{
|
||||
ExecutorID: executorID,
|
||||
State: string(models.ExecutorReachabilityStateUnknown),
|
||||
ProbingEnabled: effectiveIntervalSeconds > 0,
|
||||
ProbeIntervalSeconds: effectiveIntervalSeconds,
|
||||
Persisted: persisted,
|
||||
}
|
||||
if record == nil {
|
||||
return dto
|
||||
}
|
||||
dto.State = string(record.State)
|
||||
dto.Reason = string(record.Reason)
|
||||
dto.Message = record.Message
|
||||
dto.ConsecutiveFailures = record.ConsecutiveFailures
|
||||
dto.Host = record.Host
|
||||
dto.CheckedAt = record.CheckedAt
|
||||
dto.LastSuccessAt = record.LastSuccessAt
|
||||
if !record.UpdatedAt.IsZero() {
|
||||
updatedAt := record.UpdatedAt
|
||||
dto.UpdatedAt = &updatedAt
|
||||
}
|
||||
return dto
|
||||
}
|
||||
|
||||
// Publisher publishes events.ExecutorReachabilityChanged. Constructed once
|
||||
// at startup wiring and shared by the poller (probe results) and the save
|
||||
// observer (configuration resets) — one propagation path for both triggers,
|
||||
// per the system design.
|
||||
type Publisher struct {
|
||||
eventBus bus.EventBus
|
||||
log *logger.Logger
|
||||
}
|
||||
|
||||
// NewPublisher builds a Publisher. A nil eventBus is accepted so a caller
|
||||
// that hasn't wired the event bus yet (tests, a degraded boot path) can
|
||||
// still construct one; PublishChanged becomes a no-op. An optional logger
|
||||
// records publication failures with the affected executor id.
|
||||
func NewPublisher(eventBus bus.EventBus, logs ...*logger.Logger) *Publisher {
|
||||
var log *logger.Logger
|
||||
if len(logs) > 0 {
|
||||
log = logs[0]
|
||||
}
|
||||
return &Publisher{eventBus: eventBus, log: log}
|
||||
}
|
||||
|
||||
// PublishChanged publishes record's current shape on
|
||||
// events.ExecutorReachabilityChanged. Safe to call on a nil *Publisher, with
|
||||
// a nil record, or with a Publisher built from a nil event bus — every
|
||||
// combination is a no-op rather than a panic, so callers on the probe/reset
|
||||
// hot paths don't need a defensive check of their own.
|
||||
func (p *Publisher) PublishChanged(ctx context.Context, record *models.ExecutorReachability, effectiveIntervalSeconds int) {
|
||||
if p == nil || p.eventBus == nil || record == nil {
|
||||
return
|
||||
}
|
||||
dto := BuildRecordDTO(record.ExecutorID, record, effectiveIntervalSeconds, true)
|
||||
event := bus.NewEvent(events.ExecutorReachabilityChanged, "executor-reachability", dto)
|
||||
if err := p.eventBus.Publish(ctx, events.ExecutorReachabilityChanged, event); err != nil {
|
||||
publishFailedTotal.Add(1)
|
||||
if p.log != nil {
|
||||
p.log.Warn("executor ssh reachability: publish failed",
|
||||
zap.String("executor_id", record.ExecutorID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
type failingPublishEventBus struct {
|
||||
*bus.MemoryEventBus
|
||||
err error
|
||||
}
|
||||
|
||||
func (b *failingPublishEventBus) Publish(context.Context, string, *bus.Event) error {
|
||||
return b.err
|
||||
}
|
||||
|
||||
func testEventBusLogger() *bus.MemoryEventBus {
|
||||
return bus.NewMemoryEventBus(logger.Default())
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// PublishChanged must publish on events.ExecutorReachabilityChanged with a
|
||||
// payload carrying the record plus the derived probing fields a client needs
|
||||
// (probing_enabled/probe_interval_seconds) without a second fetch.
|
||||
func TestPublisherPublishChanged_PublishesRecordDTO(t *testing.T) {
|
||||
eventBus := testEventBusLogger()
|
||||
var captured *bus.Event
|
||||
_, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
|
||||
captured = ev
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
|
||||
pub := NewPublisher(eventBus)
|
||||
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
record := &models.ExecutorReachability{
|
||||
ExecutorID: "exec-1",
|
||||
State: models.ExecutorReachabilityStateUnreachable,
|
||||
Reason: models.ExecutorReachabilityReasonNetwork,
|
||||
Message: "connection refused",
|
||||
Host: "10.0.0.1",
|
||||
CheckedAt: &checkedAt,
|
||||
}
|
||||
pub.PublishChanged(context.Background(), record, 60)
|
||||
|
||||
if captured == nil {
|
||||
t.Fatal("no event was published")
|
||||
}
|
||||
dto, ok := captured.Data.(RecordDTO)
|
||||
if !ok {
|
||||
t.Fatalf("event data type = %T, want RecordDTO", captured.Data)
|
||||
}
|
||||
if dto.ExecutorID != "exec-1" || dto.State != "unreachable" || dto.Reason != "network" {
|
||||
t.Fatalf("dto = %+v, want executor-1/unreachable/network", dto)
|
||||
}
|
||||
if !dto.ProbingEnabled || dto.ProbeIntervalSeconds != 60 {
|
||||
t.Fatalf("dto probing fields = %+v, want enabled/60", dto)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14
|
||||
//
|
||||
// probing_enabled must be false exactly when the effective interval is 0 —
|
||||
// the client's only way to distinguish "not probed yet" from "probing off".
|
||||
func TestPublisherPublishChanged_ProbingDisabledWhenIntervalIsZero(t *testing.T) {
|
||||
eventBus := testEventBusLogger()
|
||||
var captured *bus.Event
|
||||
_, _ = eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
|
||||
captured = ev
|
||||
return nil
|
||||
})
|
||||
|
||||
pub := NewPublisher(eventBus)
|
||||
pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 0)
|
||||
|
||||
dto := captured.Data.(RecordDTO)
|
||||
if dto.ProbingEnabled {
|
||||
t.Fatalf("ProbingEnabled = true, want false when the effective interval is 0")
|
||||
}
|
||||
}
|
||||
|
||||
// A nil publisher, nil record, or nil bus must never panic — PublishChanged
|
||||
// is called from probe/save-reset paths that don't want a defensive nil
|
||||
// check at every call site.
|
||||
func TestPublisherPublishChanged_NilSafe(t *testing.T) {
|
||||
var nilPub *Publisher
|
||||
nilPub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
|
||||
|
||||
pub := NewPublisher(testEventBusLogger())
|
||||
pub.PublishChanged(context.Background(), nil, 60)
|
||||
|
||||
pub2 := NewPublisher(nil)
|
||||
pub2.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
|
||||
}
|
||||
|
||||
func TestPublisherPublishChangedCountsPublishFailures(t *testing.T) {
|
||||
before := publishFailedTotal.Value()
|
||||
pub := NewPublisher(&failingPublishEventBus{
|
||||
MemoryEventBus: testEventBusLogger(),
|
||||
err: errors.New("event bus unavailable"),
|
||||
})
|
||||
pub.PublishChanged(context.Background(), &models.ExecutorReachability{ExecutorID: "exec-1"}, 60)
|
||||
if got := publishFailedTotal.Value(); got != before+1 {
|
||||
t.Fatalf("publishFailedTotal = %d, want %d", got, before+1)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.14, AC-EXECUTORS-SSH-REACHABILITY-002.5
|
||||
//
|
||||
// BuildRecordDTO is the single place the HTTP routes and the event publisher
|
||||
// share for projecting a stored (or absent) record into the wire shape —
|
||||
// including the "no record yet" unknown placeholder with every timestamp
|
||||
// null, never omitted.
|
||||
func TestBuildRecordDTO_SynthesizesUnknownShapeWhenRecordIsNil(t *testing.T) {
|
||||
dto := BuildRecordDTO("exec-1", nil, 60, true)
|
||||
if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Reason != "" {
|
||||
t.Fatalf("dto = %+v, want a synthesized unknown/empty-reason shape", dto)
|
||||
}
|
||||
if dto.CheckedAt != nil || dto.LastSuccessAt != nil || dto.UpdatedAt != nil {
|
||||
t.Fatalf("dto = %+v, want every timestamp nil", dto)
|
||||
}
|
||||
if dto.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("ConsecutiveFailures = %d, want 0", dto.ConsecutiveFailures)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRecordDTO_ProjectsStoredRecord(t *testing.T) {
|
||||
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
record := &models.ExecutorReachability{
|
||||
ExecutorID: "exec-1",
|
||||
State: models.ExecutorReachabilityStateReachable,
|
||||
ConsecutiveFailures: 0,
|
||||
Host: "10.0.0.1",
|
||||
CheckedAt: &checkedAt,
|
||||
LastSuccessAt: &checkedAt,
|
||||
UpdatedAt: checkedAt,
|
||||
}
|
||||
dto := BuildRecordDTO("exec-1", record, 60, true)
|
||||
if dto.State != "reachable" || dto.Host != "10.0.0.1" {
|
||||
t.Fatalf("dto = %+v, want the stored record projected", dto)
|
||||
}
|
||||
if dto.CheckedAt == nil || !dto.CheckedAt.Equal(checkedAt) {
|
||||
t.Fatalf("CheckedAt = %v, want %v", dto.CheckedAt, checkedAt)
|
||||
}
|
||||
if !dto.Persisted {
|
||||
t.Fatalf("Persisted = false, want true")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// connectionConfigKeys names the exact SSH connection-configuration fields —
|
||||
// host, port, user, identity source, optional ProxyJump, and pinned host
|
||||
// fingerprint — that a save-detection comparison must diff. Any other Config
|
||||
// key (or a non-config field) changing is not connection-relevant and must
|
||||
// not trigger a reset.
|
||||
var connectionConfigKeys = []string{
|
||||
"ssh_host",
|
||||
"ssh_host_alias",
|
||||
"ssh_port",
|
||||
"ssh_user",
|
||||
"ssh_identity_source",
|
||||
"ssh_identity_file",
|
||||
"ssh_proxy_jump",
|
||||
"ssh_host_fingerprint",
|
||||
}
|
||||
|
||||
// SaveObserver implements service.ExecutorSaveObserver: it resets the stored
|
||||
// reachability record and dispatches an off-cycle probe whenever an active
|
||||
// SSH executor's connection configuration changes. Declared in
|
||||
// internal/task/service (this package structurally satisfies it there
|
||||
// without an import back to it).
|
||||
type SaveObserver struct {
|
||||
poller *Poller
|
||||
}
|
||||
|
||||
// NewSaveObserver builds a SaveObserver over poller. poller supplies the
|
||||
// repository (for the reset and the pre-reset read), the publisher (for the
|
||||
// change event), and ProbeNow (for the post-reset off-cycle probe) — the same
|
||||
// primitives task 03/04 already exercise elsewhere in this package.
|
||||
func NewSaveObserver(poller *Poller) *SaveObserver {
|
||||
return &SaveObserver{poller: poller}
|
||||
}
|
||||
|
||||
// OnExecutorSaved resets and re-probes after when it is an active SSH
|
||||
// executor whose connection configuration changed. before is nil on create,
|
||||
// which is always treated as a change — there is no prior state to compare.
|
||||
func (o *SaveObserver) OnExecutorSaved(ctx context.Context, before, after *models.Executor) {
|
||||
if !eligibleForReachability(after) {
|
||||
return
|
||||
}
|
||||
if !connectionConfigChanged(before, after) {
|
||||
return
|
||||
}
|
||||
o.resetAndProbe(ctx, after)
|
||||
}
|
||||
|
||||
func eligibleForReachability(executor *models.Executor) bool {
|
||||
return executor != nil &&
|
||||
executor.Type == models.ExecutorTypeSSH &&
|
||||
executor.Status == models.ExecutorStatusActive
|
||||
}
|
||||
|
||||
func connectionConfigChanged(before, after *models.Executor) bool {
|
||||
if before == nil {
|
||||
return true
|
||||
}
|
||||
for _, key := range connectionConfigKeys {
|
||||
if before.Config[key] != after.Config[key] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// resetAndProbe resets the stored record to the new host, publishes a change
|
||||
// event when doing so actually changes what a client would see, and dispatches
|
||||
// an off-cycle probe. The pre-reset record is read first because after the
|
||||
// reset the "was it worth publishing" comparison is no longer possible — the
|
||||
// reset itself always leaves the unknown/empty-reason placeholder behind.
|
||||
func (o *SaveObserver) resetAndProbe(ctx context.Context, executor *models.Executor) {
|
||||
repo := o.poller.store.repo
|
||||
previous, readErr := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound) {
|
||||
o.poller.log.Warn("executor ssh reachability: read before reset failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(readErr))
|
||||
}
|
||||
|
||||
newHost := executor.Config["ssh_host"]
|
||||
if err := repo.ResetExecutorReachability(ctx, executor.ID, newHost, executor.UpdatedAt); err != nil {
|
||||
o.poller.log.Warn("executor ssh reachability: reset failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
if recordWorthPublishing(previous) || (readErr != nil && !errors.Is(readErr, models.ErrExecutorReachabilityNotFound)) {
|
||||
reset, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
o.poller.log.Warn("executor ssh reachability: read after reset failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(err))
|
||||
reset = &models.ExecutorReachability{
|
||||
ExecutorID: executor.ID,
|
||||
State: models.ExecutorReachabilityStateUnknown,
|
||||
Host: newHost,
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
if o.poller.publisher != nil {
|
||||
o.poller.publisher.PublishChanged(ctx, reset, o.poller.EffectiveIntervalSeconds())
|
||||
}
|
||||
}
|
||||
|
||||
o.poller.ProbeNow(executor)
|
||||
}
|
||||
|
||||
// recordWorthPublishing reports whether record differs from the synthesized
|
||||
// unknown/empty-reason placeholder a client already sees for "no record yet"
|
||||
// — publishing a reset that lands on that same placeholder would be a no-op
|
||||
// event.
|
||||
func recordWorthPublishing(record *models.ExecutorReachability) bool {
|
||||
if record == nil {
|
||||
return false
|
||||
}
|
||||
return record.State != models.ExecutorReachabilityStateUnknown || record.Reason != ""
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// newTestSaveObserver builds a SaveObserver over a started Poller wired to
|
||||
// repo, with probe replaced by a deterministic success so ProbeNow's
|
||||
// dispatched probe is observable through repo's upsert count instead of real
|
||||
// network I/O. Callers must defer poller.Stop().
|
||||
func newTestSaveObserver(t *testing.T, repo *fakeRepository) (*SaveObserver, *Poller) {
|
||||
t.Helper()
|
||||
poller := New(repo, 0, logger.Default())
|
||||
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
|
||||
}
|
||||
poller.Start(context.Background())
|
||||
return NewSaveObserver(poller), poller
|
||||
}
|
||||
|
||||
func sshExecutorWithConfig(id string, status models.ExecutorStatus, config map[string]string) *models.Executor {
|
||||
return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: status, Config: config}
|
||||
}
|
||||
|
||||
// waitForUpsertCount blocks until repo has observed at least want upserts, or
|
||||
// fails the test after a short deadline — a happens-before proof that the
|
||||
// observer's fire-and-forget ProbeNow dispatch actually ran, without a fixed
|
||||
// sleep.
|
||||
func waitForUpsertCount(t *testing.T, repo *fakeRepository, want int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if repo.upsertCount() >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("upsertCount = %d, want >= %d", repo.upsertCount(), want)
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2
|
||||
//
|
||||
// A host change on an active SSH executor must reset the stored record to
|
||||
// the new host and dispatch an off-cycle probe — the reset's whole purpose is
|
||||
// that a stale unreachable/reachable record about the OLD host must never be
|
||||
// shown against the new one, even for the single tick before the next probe
|
||||
// lands.
|
||||
func TestSaveObserverResetsAndProbesOnHostChange(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
observer, poller := newTestSaveObserver(t, repo)
|
||||
defer poller.Stop()
|
||||
|
||||
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"})
|
||||
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"})
|
||||
|
||||
observer.OnExecutorSaved(context.Background(), before, after)
|
||||
|
||||
rec, err := repo.GetExecutorReachability(context.Background(), "exec-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetExecutorReachability: %v", err)
|
||||
}
|
||||
if rec.Host != "10.0.0.2" {
|
||||
t.Fatalf("record.Host = %q, want the new host 10.0.0.2 (reset must run before the probe)", rec.Host)
|
||||
}
|
||||
|
||||
waitForUpsertCount(t, repo, 1)
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// Creating an executor has no prior state to compare against, so it must
|
||||
// always be treated as a change — before=nil is not "nothing changed".
|
||||
func TestSaveObserverTreatsNilBeforeAsAlwaysChanged(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
observer, poller := newTestSaveObserver(t, repo)
|
||||
defer poller.Stop()
|
||||
|
||||
after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"})
|
||||
observer.OnExecutorSaved(context.Background(), nil, after)
|
||||
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), "exec-created"); err != nil {
|
||||
t.Fatalf("GetExecutorReachability: %v, want a reset record from the create", err)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// A non-SSH executor save must never reset or probe — there is no
|
||||
// reachability record for it to begin with.
|
||||
func TestSaveObserverSkipsNonSSHExecutor(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
observer, poller := newTestSaveObserver(t, repo)
|
||||
defer poller.Stop()
|
||||
|
||||
after := &models.Executor{ID: "exec-local", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
|
||||
observer.OnExecutorSaved(context.Background(), nil, after)
|
||||
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), "exec-local"); err == nil {
|
||||
t.Fatalf("expected no record for a non-SSH executor")
|
||||
}
|
||||
if repo.upsertCount() != 0 {
|
||||
t.Fatalf("upsertCount = %d, want 0 (no probe should have been dispatched)", repo.upsertCount())
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// A disabled SSH executor must not be reset or probed — probing an executor
|
||||
// no launch path can use wastes a dial and would misleadingly imply it is
|
||||
// monitored.
|
||||
func TestSaveObserverSkipsInactiveSSHExecutor(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
observer, poller := newTestSaveObserver(t, repo)
|
||||
defer poller.Stop()
|
||||
|
||||
after := sshExecutorWithConfig("exec-disabled", models.ExecutorStatusDisabled, map[string]string{"ssh_host": "10.0.0.5"})
|
||||
observer.OnExecutorSaved(context.Background(), nil, after)
|
||||
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), "exec-disabled"); err == nil {
|
||||
t.Fatalf("expected no record for a disabled executor")
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// Only the named connection-configuration keys matter. A save that changes
|
||||
// something else entirely (here: nothing in Config at all differs) must not
|
||||
// reset an existing record — a reset would wipe legitimate history (state,
|
||||
// consecutive_failures, last_success_at) for no reason tied to the host's
|
||||
// actual reachability.
|
||||
func TestSaveObserverSkipsWhenConnectionConfigUnchanged(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
observer, poller := newTestSaveObserver(t, repo)
|
||||
defer poller.Stop()
|
||||
|
||||
config := map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"}
|
||||
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config)
|
||||
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, config)
|
||||
|
||||
observer.OnExecutorSaved(context.Background(), before, after)
|
||||
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), "exec-1"); err == nil {
|
||||
t.Fatalf("expected no record written when connection config is unchanged")
|
||||
}
|
||||
if repo.upsertCount() != 0 {
|
||||
t.Fatalf("upsertCount = %d, want 0", repo.upsertCount())
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.2, AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// A reset that actually changes what a client would see (the previously
|
||||
// stored record was not already the unknown/empty-reason placeholder) must
|
||||
// publish a change event, so a UI watching this executor doesn't have to poll
|
||||
// to notice the reset.
|
||||
func TestSaveObserverPublishesChangeEventWhenPriorRecordWasObserved(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
eventBus := bus.NewMemoryEventBus(logger.Default())
|
||||
published := make(chan RecordDTO, 4)
|
||||
if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
|
||||
published <- ev.Data.(RecordDTO)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
|
||||
poller := New(repo, 0, logger.Default())
|
||||
poller.SetPublisher(NewPublisher(eventBus))
|
||||
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
|
||||
}
|
||||
poller.Start(context.Background())
|
||||
defer poller.Stop()
|
||||
observer := NewSaveObserver(poller)
|
||||
|
||||
checkedAt := time.Now().UTC()
|
||||
repo.records["exec-1"] = &models.ExecutorReachability{
|
||||
ExecutorID: "exec-1", State: models.ExecutorReachabilityStateReachable,
|
||||
Host: "10.0.0.1", CheckedAt: &checkedAt, UpdatedAt: checkedAt,
|
||||
}
|
||||
|
||||
before := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.1"})
|
||||
after := sshExecutorWithConfig("exec-1", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.2"})
|
||||
observer.OnExecutorSaved(context.Background(), before, after)
|
||||
|
||||
select {
|
||||
case dto := <-published:
|
||||
if dto.ExecutorID != "exec-1" || dto.State != string(models.ExecutorReachabilityStateUnknown) || dto.Host != "10.0.0.2" {
|
||||
t.Fatalf("dto = %+v, want the reset unknown/10.0.0.2 shape", dto)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no change event published for a reset over a previously observed record")
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// Resetting an executor that was never probed before (no stored record at
|
||||
// all) is not itself a client-visible change — the client already sees the
|
||||
// synthesized unknown placeholder for "no record yet" — so the reset step
|
||||
// must not publish. The dispatched off-cycle probe that follows is a
|
||||
// separate, legitimate publish opportunity (a real unknown -> reachable
|
||||
// transition) and is deliberately excluded here via a blocking probe gate:
|
||||
// OnExecutorSaved's synchronous portion (the reset and its publish decision)
|
||||
// completes fully before the probe goroutine can even call probe, so
|
||||
// asserting silence right after OnExecutorSaved returns isolates the reset
|
||||
// step's own behavior without racing the probe.
|
||||
func TestSaveObserverDoesNotPublishFromResetWhenNoPriorRecord(t *testing.T) {
|
||||
repo := newFakeRepository()
|
||||
eventBus := bus.NewMemoryEventBus(logger.Default())
|
||||
published := make(chan RecordDTO, 4)
|
||||
if _, err := eventBus.Subscribe(events.ExecutorReachabilityChanged, func(_ context.Context, ev *bus.Event) error {
|
||||
published <- ev.Data.(RecordDTO)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
|
||||
probeGate := make(chan struct{})
|
||||
poller := New(repo, 0, logger.Default())
|
||||
poller.SetPublisher(NewPublisher(eventBus))
|
||||
poller.probe = func(_ context.Context, e *models.Executor) agentruntime.SSHProbeOutcome {
|
||||
<-probeGate
|
||||
return agentruntime.SSHProbeOutcome{Success: true, Host: e.Config["ssh_host"]}
|
||||
}
|
||||
poller.Start(context.Background())
|
||||
defer poller.Stop()
|
||||
observer := NewSaveObserver(poller)
|
||||
|
||||
after := sshExecutorWithConfig("exec-created", models.ExecutorStatusActive, map[string]string{"ssh_host": "10.0.0.5"})
|
||||
observer.OnExecutorSaved(context.Background(), nil, after)
|
||||
|
||||
select {
|
||||
case dto := <-published:
|
||||
t.Fatalf("reset step itself must not publish when there was no prior record: %+v", dto)
|
||||
default:
|
||||
}
|
||||
|
||||
close(probeGate)
|
||||
waitForUpsertCount(t, repo, 1)
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// Repository is the narrow slice of repository.ExecutorRepository this
|
||||
// package depends on. The poller and the off-cycle-probe primitive share it,
|
||||
// so a fake test double satisfies both without importing the full
|
||||
// task/repository interface.
|
||||
type Repository interface {
|
||||
ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error)
|
||||
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
|
||||
UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error
|
||||
ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error
|
||||
}
|
||||
|
||||
// observeResult reports what Observe actually did, for the poller's logging
|
||||
// and metrics and as the change signal a future consumer (the HTTP/event
|
||||
// surface) reads off the write path rather than re-deriving. Changed is
|
||||
// broader than StateChanged: the destination state can hold steady
|
||||
// (unreachable stays unreachable) while the stored reason moves between two
|
||||
// different failure causes, and that is still worth telling a client about.
|
||||
type observeResult struct {
|
||||
Applied bool
|
||||
StateChanged bool
|
||||
Changed bool
|
||||
Previous models.ExecutorReachabilityState
|
||||
Current models.ExecutorReachabilityState
|
||||
// After is the full record as written, for a caller (the HTTP immediate-
|
||||
// probe route, the change-event publisher) that needs more than the bare
|
||||
// state enum without a second read-back.
|
||||
After *models.ExecutorReachability
|
||||
// Observed is the outcome projected locally from the probe, even when the
|
||||
// repository rejects the write or a read-back fails. It is never published
|
||||
// as durable state, but lets a manual probe show the actual result.
|
||||
Observed *models.ExecutorReachability
|
||||
}
|
||||
|
||||
// store owns the write path's hysteresis. The consecutive-failure counter
|
||||
// and the destination state for an existing row are derived entirely inside
|
||||
// UpsertExecutorReachability's own SQL (see the Persistence section of the
|
||||
// design) — store's job is narrower: pick InitialState/InitialFailures for a
|
||||
// never-before-seen row (the insert-only path), and determine whether the
|
||||
// write actually landed and whether the resulting state changed by reading
|
||||
// the record before and after the call.
|
||||
type store struct {
|
||||
repo Repository
|
||||
log *logger.Logger
|
||||
}
|
||||
|
||||
// Observe records one probe outcome for executor. checkedAt is the probe's
|
||||
// own completion timestamp, supplied by the caller so a test can control it
|
||||
// precisely (see the design's last-write-wins contract).
|
||||
func (s *store) Observe(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) observeResult {
|
||||
before, err := s.repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
|
||||
s.log.Warn("executor ssh reachability: read before write failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(err))
|
||||
return observeResult{}
|
||||
}
|
||||
|
||||
checkedAt = checkedAt.UTC().Truncate(time.Microsecond)
|
||||
obs := buildObservation(executor, outcome, checkedAt)
|
||||
observed := projectObservation(before, obs)
|
||||
recordProbeOutcome(outcome.Success, string(obs.Reason))
|
||||
if err := s.repo.UpsertExecutorReachability(ctx, obs); err != nil {
|
||||
s.log.Warn("executor ssh reachability: write failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(err))
|
||||
return observeResult{Current: observed.State, Observed: observed}
|
||||
}
|
||||
|
||||
after, err := s.repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
s.log.Warn("executor ssh reachability: read-back after write failed",
|
||||
zap.String("executor_id", executor.ID), zap.Error(err))
|
||||
readBackFailedTotal.Add(1)
|
||||
return observeResult{Current: observed.State, Observed: observed}
|
||||
}
|
||||
|
||||
// The upsert's own WHERE (eligibility, last-write-wins on checked_at) may
|
||||
// have silently dropped this write. Comparing the stored checked_at back
|
||||
// against the one this observation carried is how we find out — the SQL
|
||||
// itself never reports it.
|
||||
if after.CheckedAt == nil || !after.CheckedAt.Equal(checkedAt) {
|
||||
writeRefusedTotal.Add(1)
|
||||
s.log.Debug("executor ssh reachability: write refused",
|
||||
zap.String("executor_id", executor.ID))
|
||||
return observeResult{Current: after.State, Observed: observed}
|
||||
}
|
||||
|
||||
previous := models.ExecutorReachabilityStateUnknown
|
||||
var previousReason models.ExecutorReachabilityReason
|
||||
if before != nil {
|
||||
previous = before.State
|
||||
previousReason = before.Reason
|
||||
}
|
||||
stateChanged := previous != after.State
|
||||
if stateChanged {
|
||||
s.logTransition(executor, previous, after.State, obs.Reason, after.ConsecutiveFailures)
|
||||
recordStateTransition(string(after.State))
|
||||
}
|
||||
changed := stateChanged || previousReason != after.Reason
|
||||
return observeResult{
|
||||
Applied: true, StateChanged: stateChanged, Changed: changed,
|
||||
Previous: previous, Current: after.State, After: after,
|
||||
Observed: after,
|
||||
}
|
||||
}
|
||||
|
||||
func projectObservation(before *models.ExecutorReachability, obs models.ExecutorReachabilityObservation) *models.ExecutorReachability {
|
||||
record := &models.ExecutorReachability{
|
||||
ExecutorID: obs.ExecutorID,
|
||||
State: obs.InitialState,
|
||||
Reason: obs.Reason,
|
||||
Message: obs.Message,
|
||||
ConsecutiveFailures: obs.InitialFailures,
|
||||
Host: obs.Host,
|
||||
CheckedAt: checkedAtPtr(obs.CheckedAt),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
if before == nil {
|
||||
if obs.Reason == "" {
|
||||
record.State = models.ExecutorReachabilityStateReachable
|
||||
record.ConsecutiveFailures = 0
|
||||
record.LastSuccessAt = checkedAtPtr(obs.CheckedAt)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
record.State = before.State
|
||||
record.ConsecutiveFailures = before.ConsecutiveFailures
|
||||
record.LastSuccessAt = before.LastSuccessAt
|
||||
if obs.Reason == "" {
|
||||
record.State = models.ExecutorReachabilityStateReachable
|
||||
record.ConsecutiveFailures = 0
|
||||
record.LastSuccessAt = checkedAtPtr(obs.CheckedAt)
|
||||
return record
|
||||
}
|
||||
record.ConsecutiveFailures++
|
||||
if isStickyReason(obs.Reason) || record.ConsecutiveFailures >= obs.FailureThreshold {
|
||||
record.State = models.ExecutorReachabilityStateUnreachable
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func checkedAtPtr(value time.Time) *time.Time {
|
||||
copy := value
|
||||
return ©
|
||||
}
|
||||
|
||||
// logTransition logs at Warn going into unreachable and Info coming out of
|
||||
// it, per the design's Observability section. Unchanged results never reach
|
||||
// here — Observe only calls this when the state actually changed.
|
||||
func (s *store) logTransition(executor *models.Executor, previous, current models.ExecutorReachabilityState, reason models.ExecutorReachabilityReason, failures int) {
|
||||
fields := []zap.Field{
|
||||
zap.String("executor_id", executor.ID),
|
||||
zap.String("host", executor.Config["ssh_host"]),
|
||||
zap.String("from", string(previous)),
|
||||
zap.String("to", string(current)),
|
||||
zap.String("reason", string(reason)),
|
||||
zap.Int("consecutive_failures", failures),
|
||||
}
|
||||
if current == models.ExecutorReachabilityStateUnreachable {
|
||||
s.log.Warn("executor ssh reachability state transition", fields...)
|
||||
return
|
||||
}
|
||||
s.log.Info("executor ssh reachability state transition", fields...)
|
||||
}
|
||||
|
||||
// buildObservation projects a probe outcome into the upsert's input. An
|
||||
// empty Reason means success. InitialState/InitialFailures matter only on
|
||||
// the insert path (no prior row); every later write derives both in SQL
|
||||
// from the stored row instead:
|
||||
// - success: reachable, zero counter.
|
||||
// - a sticky reason (config, host_key): unreachable immediately, whatever
|
||||
// the counter — both are deterministic and would repeat, so waiting out
|
||||
// the threshold buys no confidence.
|
||||
// - any other failure: the counter starts at 1, which is below
|
||||
// failureThreshold, so the state stays unknown — a record that doesn't
|
||||
// exist yet is never promoted by a single below-threshold failure, the
|
||||
// same rule the SQL enforces for an existing row.
|
||||
func buildObservation(executor *models.Executor, outcome agentruntime.SSHProbeOutcome, checkedAt time.Time) models.ExecutorReachabilityObservation {
|
||||
reason := models.ExecutorReachabilityReason(outcome.Reason)
|
||||
initialState := models.ExecutorReachabilityStateUnknown
|
||||
initialFailures := 0
|
||||
switch {
|
||||
case outcome.Success:
|
||||
initialState = models.ExecutorReachabilityStateReachable
|
||||
case isStickyReason(reason):
|
||||
initialState = models.ExecutorReachabilityStateUnreachable
|
||||
initialFailures = 1
|
||||
default:
|
||||
initialFailures = 1
|
||||
if initialFailures >= failureThreshold {
|
||||
initialState = models.ExecutorReachabilityStateUnreachable
|
||||
}
|
||||
}
|
||||
return models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID,
|
||||
SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: initialState,
|
||||
InitialFailures: initialFailures,
|
||||
Reason: reason,
|
||||
Message: outcome.Message,
|
||||
Host: outcome.Host,
|
||||
CheckedAt: checkedAt,
|
||||
FailureThreshold: failureThreshold,
|
||||
}
|
||||
}
|
||||
|
||||
func isStickyReason(reason models.ExecutorReachabilityReason) bool {
|
||||
return reason == models.ExecutorReachabilityReasonConfig || reason == models.ExecutorReachabilityReasonHostKey
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package reachability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/db"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
)
|
||||
|
||||
func newStoreTestRepo(t *testing.T) *sqlite.Repository {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "reachability-store.db")
|
||||
dbConn, err := db.OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
|
||||
repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil)
|
||||
if err != nil {
|
||||
_ = sqlxDB.Close()
|
||||
t.Fatalf("new repository: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlxDB.Close() })
|
||||
return repo
|
||||
}
|
||||
|
||||
func newStoreTestExecutor(t *testing.T, repo *sqlite.Repository) *models.Executor {
|
||||
t.Helper()
|
||||
executor := &models.Executor{
|
||||
ID: "exec-" + t.Name(),
|
||||
Name: "ssh-host",
|
||||
Type: models.ExecutorTypeSSH,
|
||||
Status: models.ExecutorStatusActive,
|
||||
Config: map[string]string{"ssh_host": "10.0.0.1"},
|
||||
}
|
||||
if err := repo.CreateExecutor(context.Background(), executor); err != nil {
|
||||
t.Fatalf("create executor: %v", err)
|
||||
}
|
||||
return executor
|
||||
}
|
||||
|
||||
type microsecondReachabilityRepository struct {
|
||||
*sqlite.Repository
|
||||
}
|
||||
|
||||
func (r *microsecondReachabilityRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
|
||||
obs.CheckedAt = obs.CheckedAt.UTC().Truncate(time.Microsecond)
|
||||
return r.Repository.UpsertExecutorReachability(ctx, obs)
|
||||
}
|
||||
|
||||
type failingGetReachabilityRepository struct {
|
||||
*sqlite.Repository
|
||||
err error
|
||||
failAfter int
|
||||
calls int
|
||||
}
|
||||
|
||||
func (r *failingGetReachabilityRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
r.calls++
|
||||
if r.calls >= r.failAfter {
|
||||
return nil, r.err
|
||||
}
|
||||
return r.Repository.GetExecutorReachability(ctx, executorID)
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9, AC-EXECUTORS-SSH-REACHABILITY-001.10
|
||||
//
|
||||
// Drives a success/fail/fail/success sequence and asserts the exact probe
|
||||
// index on which each direction flips. A naive flip-on-first-result
|
||||
// implementation would report unreachable after the very first failure
|
||||
// (index 2) instead of the second (index 3), and would report a below-
|
||||
// threshold failure as a state change even though the state stands still.
|
||||
func TestStoreObserveHysteresisSequence(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
s := &store{repo: repo, log: logger.Default()}
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
type step struct {
|
||||
label string
|
||||
outcome agentruntime.SSHProbeOutcome
|
||||
wantState models.ExecutorReachabilityState
|
||||
wantChanged bool
|
||||
wantFailures int
|
||||
}
|
||||
steps := []step{
|
||||
{
|
||||
label: "1: first success establishes reachable",
|
||||
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
|
||||
wantState: models.ExecutorReachabilityStateReachable,
|
||||
wantChanged: true,
|
||||
wantFailures: 0,
|
||||
},
|
||||
{
|
||||
label: "2: below-threshold failure leaves state reachable",
|
||||
outcome: agentruntime.SSHProbeOutcome{
|
||||
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
||||
},
|
||||
wantState: models.ExecutorReachabilityStateReachable,
|
||||
wantChanged: false,
|
||||
wantFailures: 1,
|
||||
},
|
||||
{
|
||||
label: "3: failure reaching failureThreshold flips to unreachable",
|
||||
outcome: agentruntime.SSHProbeOutcome{
|
||||
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
||||
},
|
||||
wantState: models.ExecutorReachabilityStateUnreachable,
|
||||
wantChanged: true,
|
||||
wantFailures: 2,
|
||||
},
|
||||
{
|
||||
label: "4: a single success flips back to reachable with a zero counter",
|
||||
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
|
||||
wantState: models.ExecutorReachabilityStateReachable,
|
||||
wantChanged: true,
|
||||
wantFailures: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for i, st := range steps {
|
||||
checkedAt := base.Add(time.Duration(i) * time.Second)
|
||||
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
||||
result := s.Observe(ctx, executor, st.outcome, checkedAt)
|
||||
if !result.Applied {
|
||||
t.Fatalf("%s: write was refused, want applied", st.label)
|
||||
}
|
||||
if result.Current != st.wantState {
|
||||
t.Fatalf("%s: state = %q, want %q", st.label, result.Current, st.wantState)
|
||||
}
|
||||
if result.StateChanged != st.wantChanged {
|
||||
t.Fatalf("%s: changed = %v, want %v", st.label, result.StateChanged, st.wantChanged)
|
||||
}
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: get: %v", st.label, err)
|
||||
}
|
||||
if got.ConsecutiveFailures != st.wantFailures {
|
||||
t.Fatalf("%s: consecutive_failures = %d, want %d", st.label, got.ConsecutiveFailures, st.wantFailures)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9
|
||||
//
|
||||
// A single config or host_key failure writes unreachable on that first
|
||||
// probe, with the counter still below failureThreshold — sticky reasons
|
||||
// bypass the threshold entirely.
|
||||
func TestStoreObserveStickyFailureIsUnreachableOnFirstProbe(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reason agentruntime.SSHReachabilityReason
|
||||
}{
|
||||
{"config", agentruntime.SSHReachabilityReasonConfig},
|
||||
{"host_key", agentruntime.SSHReachabilityReasonHostKey},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
s := &store{repo: repo, log: logger.Default()}
|
||||
ctx := context.Background()
|
||||
|
||||
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
||||
Host: "10.0.0.1", Reason: tt.reason, Message: "boom",
|
||||
}, time.Now().UTC())
|
||||
|
||||
if !result.Applied || result.Current != models.ExecutorReachabilityStateUnreachable {
|
||||
t.Fatalf("result = %+v, want applied unreachable on the first probe", result)
|
||||
}
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.ConsecutiveFailures >= failureThreshold {
|
||||
t.Fatalf("consecutive_failures = %d, want below failureThreshold (%d) — sticky must not need it", got.ConsecutiveFailures, failureThreshold)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// The change-publish trigger is broader than StateChanged: a steady
|
||||
// unreachable executor can still swap failure reasons (network, then
|
||||
// timeout) between probes without ever leaving the unreachable state. A
|
||||
// publisher that only watched StateChanged would miss this and never tell a
|
||||
// client the diagnostic reason on screen just went stale.
|
||||
func TestStoreObserveReportsChangedOnReasonChangeWithoutStateChange(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
s := &store{repo: repo, log: logger.Default()}
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Two failures reach failureThreshold and flip to unreachable.
|
||||
for i := 0; i < 2; i++ {
|
||||
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
||||
s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
||||
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
||||
}, base.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
|
||||
// A third failure with a different transient reason keeps the state
|
||||
// unreachable (already above threshold) but changes the stored reason.
|
||||
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
||||
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
||||
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonTimeout, Message: "handshake timed out",
|
||||
}, base.Add(2*time.Second))
|
||||
|
||||
if result.StateChanged {
|
||||
t.Fatalf("StateChanged = true, want false — state was already unreachable")
|
||||
}
|
||||
if !result.Changed {
|
||||
t.Fatalf("Changed = false, want true — reason moved from network to timeout")
|
||||
}
|
||||
if result.After == nil || result.After.Reason != models.ExecutorReachabilityReasonTimeout {
|
||||
t.Fatalf("After = %+v, want a record carrying reason %q", result.After, models.ExecutorReachabilityReasonTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// A probe outcome that reproduces the same state and reason as the stored
|
||||
// record must not be reported as a change — this is what keeps a steady host
|
||||
// silent on the event bus.
|
||||
func TestStoreObserveReportsUnchangedWhenStateAndReasonBothRepeat(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
s := &store{repo: repo, log: logger.Default()}
|
||||
ctx := context.Background()
|
||||
|
||||
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
||||
first := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC())
|
||||
if !first.Changed {
|
||||
t.Fatalf("first observation: Changed = false, want true (unknown -> reachable)")
|
||||
}
|
||||
|
||||
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
||||
second := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC().Add(time.Second))
|
||||
if second.Changed {
|
||||
t.Fatalf("second observation: Changed = true, want false — same state (reachable) and reason (success)")
|
||||
}
|
||||
if second.After == nil {
|
||||
t.Fatalf("After = nil, want the current stored record")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreObserveNormalizesDatabaseTimestampPrecision(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
s := &store{repo: µsecondReachabilityRepository{Repository: repo}, log: logger.Default()}
|
||||
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 123456789, time.UTC)
|
||||
|
||||
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
||||
Success: true, Host: "10.0.0.1",
|
||||
}, checkedAt)
|
||||
if !result.Applied {
|
||||
t.Fatalf("result = %+v, want applied despite microsecond storage precision", result)
|
||||
}
|
||||
if result.After == nil || result.After.CheckedAt == nil {
|
||||
t.Fatalf("After = %+v, want a persisted record", result.After)
|
||||
}
|
||||
if want := checkedAt.Truncate(time.Microsecond); !result.After.CheckedAt.Equal(want) {
|
||||
t.Fatalf("CheckedAt = %v, want %v", result.After.CheckedAt, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreObserveStopsOnUnexpectedInitialReadError(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
readErr := errors.New("database unavailable")
|
||||
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 1}
|
||||
s := &store{repo: wrapped, log: logger.Default()}
|
||||
|
||||
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
||||
Success: true, Host: "10.0.0.1",
|
||||
}, time.Now().UTC())
|
||||
if result.Applied || result.Observed != nil {
|
||||
t.Fatalf("result = %+v, want no observation when the initial read fails", result)
|
||||
}
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
|
||||
t.Fatalf("stored record error = %v, want no write", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreObserveReturnsUnsavedObservationOnReadBackError(t *testing.T) {
|
||||
repo := newStoreTestRepo(t)
|
||||
executor := newStoreTestExecutor(t, repo)
|
||||
readErr := errors.New("read-back unavailable")
|
||||
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 2}
|
||||
s := &store{repo: wrapped, log: logger.Default()}
|
||||
|
||||
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
||||
Success: true, Host: "10.0.0.1",
|
||||
}, time.Now().UTC())
|
||||
if result.Applied || result.Observed == nil || result.Observed.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("result = %+v, want an unsaved reachable observation", result)
|
||||
}
|
||||
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); err != nil {
|
||||
t.Fatalf("stored record lookup: %v, want the write to have landed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustGetUpdatedAt(t *testing.T, repo *sqlite.Repository, id string) time.Time {
|
||||
t.Helper()
|
||||
executor, err := repo.GetExecutor(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("get executor: %v", err)
|
||||
}
|
||||
return executor.UpdatedAt
|
||||
}
|
||||
@@ -59,6 +59,9 @@ type Hub struct {
|
||||
pluginConversationService *plugins.Service
|
||||
conversationSourceReader ConversationSourceReader
|
||||
conversationEpoch string
|
||||
// sessionLaunchWarnings keeps the latest launch warning long enough for a
|
||||
// client that subscribes after the event was published to receive it.
|
||||
sessionLaunchWarnings map[string][]byte
|
||||
|
||||
// clientDisconnectListener releases connection-bound resources after a
|
||||
// client is removed from the hub. It runs asynchronously so durable cleanup
|
||||
@@ -100,6 +103,7 @@ func NewHub(dispatcher *ws.Dispatcher, log *logger.Logger) *Hub {
|
||||
broadcast: make(chan *ws.Message, 256),
|
||||
dispatcher: dispatcher,
|
||||
sessionMode: newSessionModeTracker(),
|
||||
sessionLaunchWarnings: make(map[string][]byte),
|
||||
logger: log.WithFields(zap.String("component", "ws_hub")),
|
||||
conversationEpoch: uuid.NewString(),
|
||||
}
|
||||
@@ -185,6 +189,7 @@ func (h *Hub) closeAllClients() {
|
||||
listener := h.clientDisconnectListener
|
||||
h.taskSubscribers = make(map[string]map[*Client]bool)
|
||||
h.sessionSubscribers = make(map[string]map[*Client]bool)
|
||||
h.sessionLaunchWarnings = make(map[string][]byte)
|
||||
h.runSubscribers = make(map[string]map[*Client]bool)
|
||||
h.systemMetricsSubscribers = make(map[*Client]bool)
|
||||
h.sessionMode.focusByClient = make(map[string]map[*Client]bool)
|
||||
@@ -684,6 +689,9 @@ func (h *Hub) SubscribeToSession(client *Client, sessionID string) bool {
|
||||
zap.String("session_id", sessionID))
|
||||
|
||||
h.recomputeSessionMode(sessionID)
|
||||
if !wasSubscribed {
|
||||
h.replaySessionLaunchWarning(client, sessionID)
|
||||
}
|
||||
return !wasSubscribed
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package websocket
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/auth/authn"
|
||||
@@ -47,6 +48,45 @@ func TestBroadcastToSessionReachesFocusedAndSubscribedClients(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionLaunchWarningReplaysToLateSubscriber(t *testing.T) {
|
||||
h := newTestHub(t)
|
||||
warning, err := ws.NewNotification(ws.ActionSessionLaunchWarning, map[string]any{
|
||||
"session_id": "sess-1",
|
||||
"host": "10.0.0.5",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("notification: %v", err)
|
||||
}
|
||||
h.rememberAndBroadcastSessionLaunchWarning("sess-1", warning)
|
||||
|
||||
late := newTestClient("late")
|
||||
registerTestClient(h, late)
|
||||
if !h.SubscribeToSession(late, "sess-1") {
|
||||
t.Fatal("late subscriber should create a new membership")
|
||||
}
|
||||
|
||||
select {
|
||||
case data := <-late.send:
|
||||
var got ws.Message
|
||||
if err := json.Unmarshal(data, &got); err != nil {
|
||||
t.Fatalf("decode replay: %v", err)
|
||||
}
|
||||
if got.Action != ws.ActionSessionLaunchWarning {
|
||||
t.Fatalf("replayed action = %q, want %q", got.Action, ws.ActionSessionLaunchWarning)
|
||||
}
|
||||
default:
|
||||
t.Fatal("late subscriber did not receive the cached launch warning")
|
||||
}
|
||||
|
||||
h.clearSessionLaunchWarning("sess-1")
|
||||
newClient := newTestClient("new")
|
||||
registerTestClient(h, newClient)
|
||||
h.SubscribeToSession(newClient, "sess-1")
|
||||
if clientReceived(newClient) {
|
||||
t.Fatal("cleared launch warning was replayed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBroadcastToSessionDeduplicatesFocusedSubscription(t *testing.T) {
|
||||
h := newTestHub(t)
|
||||
c := newTestClient("c1")
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package websocket
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
ws "github.com/kandev/kandev/pkg/websocket"
|
||||
)
|
||||
|
||||
// rememberAndBroadcastSessionLaunchWarning stores the latest warning before
|
||||
// the live fan-out. This ordering closes the subscribe-versus-publish race:
|
||||
// a client either receives the live notification or replays the cached frame
|
||||
// when it joins later.
|
||||
func (h *Hub) rememberAndBroadcastSessionLaunchWarning(sessionID string, message *ws.Message) {
|
||||
if sessionID == "" || message == nil {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.sessionLaunchWarnings == nil {
|
||||
h.sessionLaunchWarnings = make(map[string][]byte)
|
||||
}
|
||||
h.sessionLaunchWarnings[sessionID] = append([]byte(nil), data...)
|
||||
h.mu.Unlock()
|
||||
h.BroadcastToSession(sessionID, message)
|
||||
}
|
||||
|
||||
func (h *Hub) clearSessionLaunchWarning(sessionID string) {
|
||||
if sessionID == "" {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
delete(h.sessionLaunchWarnings, sessionID)
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *Hub) replaySessionLaunchWarning(client *Client, sessionID string) {
|
||||
if client == nil || sessionID == "" {
|
||||
return
|
||||
}
|
||||
h.mu.RLock()
|
||||
data := append([]byte(nil), h.sessionLaunchWarnings[sessionID]...)
|
||||
h.mu.RUnlock()
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
client.sendNotificationFrame(newOutboundNotification(data, ws.ActionSessionLaunchWarning))
|
||||
}
|
||||
@@ -53,6 +53,7 @@ func RegisterSessionStreamNotifications(ctx context.Context, eventBus bus.EventB
|
||||
b.subscribe(eventBus, events.BuildSessionModelsWildcardSubject(), ws.ActionSessionModelsUpdated)
|
||||
b.subscribe(eventBus, events.BuildSessionModelFallbackWildcardSubject(), ws.ActionSessionModelFallback)
|
||||
b.subscribe(eventBus, events.BuildSessionModelSelectionWarningWildcardSubject(), ws.ActionSessionModelSelectionWarning)
|
||||
b.subscribe(eventBus, events.BuildSessionLaunchWarningWildcardSubject(), ws.ActionSessionLaunchWarning)
|
||||
b.subscribe(eventBus, events.BuildSessionMCPStatusWildcardSubject(), ws.ActionSessionMCPStatusUpdated)
|
||||
b.subscribe(eventBus, events.BuildSessionInfoWildcardSubject(), ws.ActionSessionInfoUpdated)
|
||||
b.subscribe(eventBus, events.BuildSessionTodosWildcardSubject(), ws.ActionSessionTodosUpdated)
|
||||
@@ -95,6 +96,10 @@ func (b *SessionStreamBroadcaster) subscribe(eventBus bus.EventBus, subject, act
|
||||
b.logger.Error("failed to build websocket notification", zap.String("action", action), zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
if action == ws.ActionSessionLaunchWarning {
|
||||
b.hub.rememberAndBroadcastSessionLaunchWarning(sessionID, msg)
|
||||
return nil
|
||||
}
|
||||
b.hub.BroadcastToSession(sessionID, msg)
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -236,3 +236,78 @@ func TestSessionStreamBroadcaster_BroadcastsFallbackNotification(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject verifies the
|
||||
// session stream broadcaster subscribes the session.launch.warning wildcard
|
||||
// subject task 05 publishes on. Without this the event never leaves the bus.
|
||||
func TestSessionStreamBroadcaster_SubscribesLaunchWarningSubject(t *testing.T) {
|
||||
log := testLogger()
|
||||
eventBus := &subscriptionRecordingEventBus{MemoryEventBus: bus.NewMemoryEventBus(log)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
hub := NewHub(nil, log)
|
||||
b := RegisterSessionStreamNotifications(ctx, eventBus, hub, log)
|
||||
_ = b
|
||||
|
||||
for _, subject := range eventBus.subjects {
|
||||
if subject == events.BuildSessionLaunchWarningWildcardSubject() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("session stream broadcaster must subscribe the launch warning wildcard subject %q",
|
||||
events.BuildSessionLaunchWarningWildcardSubject())
|
||||
}
|
||||
|
||||
// TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification verifies a
|
||||
// published session.launch.warning event is routed to the session's
|
||||
// subscribers as the session.launch.warning action.
|
||||
func TestSessionStreamBroadcaster_BroadcastsLaunchWarningNotification(t *testing.T) {
|
||||
hub := newAccessTestHub(t)
|
||||
eventBus := bus.NewMemoryEventBus(testLogger())
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
_ = RegisterSessionStreamNotifications(ctx, eventBus, hub, testLogger())
|
||||
|
||||
client := registerAccessClient(t, hub, "client-launch-warning", authn.Identity{UserID: "user-1", Role: authn.RoleMember})
|
||||
raw, err := json.Marshal(map[string]interface{}{"session_id": "session-warn-1"})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal subscribe payload: %v", err)
|
||||
}
|
||||
client.handleSessionSubscribe(&ws.Message{
|
||||
ID: "1", Type: ws.MessageTypeRequest, Action: ws.ActionSessionSubscribe, Payload: raw,
|
||||
})
|
||||
|
||||
sessionID := "session-warn-1"
|
||||
_ = eventBus.Publish(ctx, events.BuildSessionLaunchWarningSubject(sessionID), bus.NewEvent(
|
||||
events.SessionLaunchWarning,
|
||||
"test",
|
||||
&lifecycle.LaunchWarningEventPayload{
|
||||
TaskID: "task-1",
|
||||
SessionID: sessionID,
|
||||
ExecutorID: "executor-1",
|
||||
Host: "10.0.0.5",
|
||||
State: "unreachable",
|
||||
Reason: "timeout",
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
},
|
||||
))
|
||||
|
||||
timeout := time.After(2 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case raw := <-client.send:
|
||||
var msg ws.Message
|
||||
if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning {
|
||||
return
|
||||
}
|
||||
case raw := <-client.controlSend:
|
||||
var msg ws.Message
|
||||
if err := json.Unmarshal(raw, &msg); err == nil && msg.Action == ws.ActionSessionLaunchWarning {
|
||||
return
|
||||
}
|
||||
case <-timeout:
|
||||
t.Fatalf("client did not receive %s", ws.ActionSessionLaunchWarning)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ func RegisterTaskNotifications(ctx context.Context, eventBus bus.EventBus, hub *
|
||||
b.subscribe(eventBus, events.ExecutorCreated, ws.ActionExecutorCreated)
|
||||
b.subscribe(eventBus, events.ExecutorUpdated, ws.ActionExecutorUpdated)
|
||||
b.subscribe(eventBus, events.ExecutorDeleted, ws.ActionExecutorDeleted)
|
||||
b.subscribe(eventBus, events.ExecutorReachabilityChanged, ws.ActionExecutorReachabilityChanged)
|
||||
b.subscribe(eventBus, events.ExecutorProfileCreated, ws.ActionExecutorProfileCreated)
|
||||
b.subscribe(eventBus, events.ExecutorProfileUpdated, ws.ActionExecutorProfileUpdated)
|
||||
b.subscribe(eventBus, events.ExecutorProfileDeleted, ws.ActionExecutorProfileDeleted)
|
||||
@@ -290,6 +291,9 @@ func (b *TaskEventBroadcaster) routeBroadcast(
|
||||
return nil
|
||||
}
|
||||
case ws.ActionSessionStateChanged:
|
||||
if sessionID != "" && extractStringField(data, "new_state") == string(models.TaskSessionStateRunning) {
|
||||
b.hub.clearSessionLaunchWarning(sessionID)
|
||||
}
|
||||
// Broadcast beyond the session subscribers so the sidebar task
|
||||
// switcher can track state changes for all tasks — but scoped to
|
||||
// the owning workspace's user when auth is enabled.
|
||||
@@ -304,6 +308,7 @@ func (b *TaskEventBroadcaster) routeBroadcast(
|
||||
}
|
||||
case ws.ActionSessionRemoved:
|
||||
if sessionID != "" {
|
||||
b.hub.clearSessionLaunchWarning(sessionID)
|
||||
if _, hasReceipt := conversationReceiptFromData(data); hasReceipt {
|
||||
b.hub.BroadcastConversationMutation(data)
|
||||
}
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package websocket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/events/bus"
|
||||
ws "github.com/kandev/kandev/pkg/websocket"
|
||||
)
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
||||
//
|
||||
// events.ExecutorReachabilityChanged must reach WS clients as
|
||||
// ws.ActionExecutorReachabilityChanged. Executor events carry no
|
||||
// workspace_id, so — like ExecutorUpdated before it — this rides the
|
||||
// broadcaster's trailing default case (global broadcast) rather than a
|
||||
// dedicated routeBroadcast switch entry.
|
||||
func TestTaskEventBroadcaster_BroadcastsExecutorReachabilityChanged(t *testing.T) {
|
||||
log := testLogger()
|
||||
eventBus := bus.NewMemoryEventBus(log)
|
||||
hub := NewHub(nil, log)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
_ = RegisterTaskNotifications(ctx, eventBus, hub, log)
|
||||
|
||||
payload := map[string]interface{}{"executor_id": "exec-1", "state": "unreachable", "reason": "network"}
|
||||
_ = eventBus.Publish(ctx, events.ExecutorReachabilityChanged, bus.NewEvent(
|
||||
events.ExecutorReachabilityChanged, "test", payload,
|
||||
))
|
||||
|
||||
msg := <-hub.broadcast
|
||||
if msg.Action != ws.ActionExecutorReachabilityChanged {
|
||||
t.Fatalf("broadcast action = %q, want %q", msg.Action, ws.ActionExecutorReachabilityChanged)
|
||||
}
|
||||
}
|
||||
@@ -226,7 +226,7 @@ func TestTaskEventBroadcaster_NoDuplicateSubscriptions(t *testing.T) {
|
||||
//
|
||||
// Update this number when adding or removing event subscriptions in
|
||||
// RegisterTaskNotifications — it is intentionally exact.
|
||||
const wantSubscriptions = 75
|
||||
const wantSubscriptions = 76
|
||||
if got := len(b.subscriptions); got != wantSubscriptions {
|
||||
t.Errorf("RegisterTaskNotifications created %d subscriptions, want %d — "+
|
||||
"did an event get subscribed twice?", got, wantSubscriptions)
|
||||
|
||||
@@ -299,13 +299,13 @@ func (s *Sweeper) sweepRuns(ctx context.Context, q queryer, cfg TableSettings, n
|
||||
}}, satelliteResults{}
|
||||
}
|
||||
return SweptTableResult{TableSweepResult: TableSweepResult{
|
||||
Deleted: result.RunsDeleted,
|
||||
Backlog: eligible > int64(batchLimit),
|
||||
}}, satelliteResults{
|
||||
RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted},
|
||||
RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted},
|
||||
RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted},
|
||||
}
|
||||
Deleted: result.RunsDeleted,
|
||||
Backlog: eligible > int64(batchLimit),
|
||||
}}, satelliteResults{
|
||||
RunEvents: TableSweepResult{Deleted: result.RunEventsDeleted},
|
||||
RouteAttempts: TableSweepResult{Deleted: result.RouteAttemptsDeleted},
|
||||
RunSkills: TableSweepResult{Deleted: result.RunSkillsDeleted},
|
||||
}
|
||||
}
|
||||
|
||||
// retentionCutoff turns a table's configured window into the instant a
|
||||
|
||||
@@ -25,7 +25,10 @@ import (
|
||||
ws "github.com/kandev/kandev/pkg/websocket"
|
||||
)
|
||||
|
||||
const errorJSONKey = "error"
|
||||
const (
|
||||
errorJSONKey = "error"
|
||||
errExecutorIDRequired = "executor id required"
|
||||
)
|
||||
|
||||
// ExecutorRunningLister is the narrow repository slice we need to surface
|
||||
// active SSH sessions. We list all running executors and resolve each row's
|
||||
@@ -54,11 +57,13 @@ type AgentLister interface {
|
||||
|
||||
// Handler exposes /api/v1/ssh routes used by the settings UI.
|
||||
type Handler struct {
|
||||
repo ExecutorRunningLister
|
||||
executorFetcher ExecutorFetcher
|
||||
agents AgentLister
|
||||
resolver *lifecycle.AgentctlResolver
|
||||
logger *logger.Logger
|
||||
repo ExecutorRunningLister
|
||||
executorFetcher ExecutorFetcher
|
||||
agents AgentLister
|
||||
resolver *lifecycle.AgentctlResolver
|
||||
logger *logger.Logger
|
||||
reachabilityRepo ReachabilityLister
|
||||
reachabilityPoller ReachabilityProber
|
||||
}
|
||||
|
||||
// NewHandler builds an SSH HTTP/WS handler.
|
||||
@@ -68,13 +73,17 @@ func NewHandler(
|
||||
agents AgentLister,
|
||||
resolver *lifecycle.AgentctlResolver,
|
||||
log *logger.Logger,
|
||||
reachabilityRepo ReachabilityLister,
|
||||
reachabilityPoller ReachabilityProber,
|
||||
) *Handler {
|
||||
return &Handler{
|
||||
repo: repo,
|
||||
executorFetcher: executorFetcher,
|
||||
agents: agents,
|
||||
resolver: resolver,
|
||||
logger: log.WithFields(zap.String("component", "ssh-handler")),
|
||||
repo: repo,
|
||||
executorFetcher: executorFetcher,
|
||||
agents: agents,
|
||||
resolver: resolver,
|
||||
logger: log.WithFields(zap.String("component", "ssh-handler")),
|
||||
reachabilityRepo: reachabilityRepo,
|
||||
reachabilityPoller: reachabilityPoller,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,12 +96,14 @@ func RegisterRoutes(
|
||||
registry *registry.Registry,
|
||||
resolver *lifecycle.AgentctlResolver,
|
||||
log *logger.Logger,
|
||||
reachabilityRepo ReachabilityLister,
|
||||
reachabilityPoller ReachabilityProber,
|
||||
) {
|
||||
var agentLister AgentLister
|
||||
if registry != nil {
|
||||
agentLister = registry
|
||||
}
|
||||
h := NewHandler(repo, executorFetcher, agentLister, resolver, log)
|
||||
h := NewHandler(repo, executorFetcher, agentLister, resolver, log, reachabilityRepo, reachabilityPoller)
|
||||
h.registerHTTP(router)
|
||||
h.registerWS(dispatcher)
|
||||
}
|
||||
@@ -103,6 +114,9 @@ func (h *Handler) registerHTTP(router *gin.Engine) {
|
||||
api.GET("/executors/:id/sessions", h.httpListSessions)
|
||||
api.POST("/executors/:id/probe-agents", h.httpProbeAgents)
|
||||
api.POST("/executors/:id/probe-shells", h.httpProbeShells)
|
||||
api.GET("/reachability", h.httpListReachability)
|
||||
api.GET("/executors/:id/reachability", h.httpGetReachability)
|
||||
api.POST("/executors/:id/reachability/probe", h.httpProbeReachability)
|
||||
}
|
||||
|
||||
func (h *Handler) registerWS(dispatcher *ws.Dispatcher) {
|
||||
@@ -179,7 +193,7 @@ func (h *Handler) httpTest(c *gin.Context) {
|
||||
func (h *Handler) httpListSessions(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
|
||||
return
|
||||
}
|
||||
rows, err := h.listSessions(c.Request.Context(), id)
|
||||
@@ -232,7 +246,7 @@ type ProbeAgentsRequest struct {
|
||||
func (h *Handler) httpProbeAgents(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
|
||||
return
|
||||
}
|
||||
if h.executorFetcher == nil || h.agents == nil {
|
||||
@@ -332,7 +346,7 @@ type ProbeShellsResponse struct {
|
||||
func (h *Handler) httpProbeShells(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "executor id required"})
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
|
||||
return
|
||||
}
|
||||
if h.executorFetcher == nil {
|
||||
@@ -421,40 +435,13 @@ func (h *Handler) resolveSSHTarget(ctx context.Context, executorID string) (*lif
|
||||
if executor == nil || executor.Type != models.ExecutorTypeSSH {
|
||||
return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID)
|
||||
}
|
||||
target, err := sshTargetFromExecutorConfig(executor.Config)
|
||||
target, err := lifecycle.SSHTargetFromExecutorConfig(executor.Config)
|
||||
if err != nil {
|
||||
return nil, http.StatusBadRequest, err
|
||||
}
|
||||
return target, http.StatusOK, nil
|
||||
}
|
||||
|
||||
// sshTargetFromExecutorConfig projects an executor.Config into the
|
||||
// SSHConnConfig the dialer expects. Keeps the handler decoupled from
|
||||
// lifecycle's internal metadata-vs-config representation.
|
||||
func sshTargetFromExecutorConfig(cfg map[string]string) (*lifecycle.SSHTarget, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("ssh executor has no config")
|
||||
}
|
||||
port := 0
|
||||
if p := strings.TrimSpace(cfg["ssh_port"]); p != "" {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return nil, fmt.Errorf("invalid ssh_port %q", p)
|
||||
}
|
||||
port = n
|
||||
}
|
||||
return lifecycle.ResolveSSHTarget(lifecycle.SSHConnConfig{
|
||||
HostAlias: cfg["ssh_host_alias"],
|
||||
Host: cfg["ssh_host"],
|
||||
Port: port,
|
||||
User: cfg["ssh_user"],
|
||||
IdentitySource: lifecycle.SSHIdentitySource(cfg["ssh_identity_source"]),
|
||||
IdentityFile: cfg["ssh_identity_file"],
|
||||
ProxyJump: cfg["ssh_proxy_jump"],
|
||||
PinnedFingerprint: cfg["ssh_host_fingerprint"],
|
||||
})
|
||||
}
|
||||
|
||||
// --- WS handlers ---
|
||||
|
||||
func (h *Handler) wsTest(ctx context.Context, msg *ws.Message) (*ws.Message, error) {
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// reachabilityConfigValidator is the production seam over the runtime's SSH
|
||||
// reachability prober, used only to check whether an executor's config
|
||||
// resolves to a dialable target — no probe is actually run here.
|
||||
var reachabilityConfigValidator = agentruntime.NewSSHReachabilityProber()
|
||||
|
||||
// ReachabilityLister is the narrow repository slice the reachability GET
|
||||
// routes need: every stored record in one query (the list route) or a
|
||||
// single record (the single-executor route). Executor lookup itself reuses
|
||||
// ExecutorFetcher, already wired on Handler for the existing SSH routes.
|
||||
type ReachabilityLister interface {
|
||||
ListExecutors(ctx context.Context) ([]*models.Executor, error)
|
||||
ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error)
|
||||
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
|
||||
}
|
||||
|
||||
// ReachabilityProber is the narrow slice of *reachability.Poller the
|
||||
// immediate-probe route needs. Declared here (not imported as the concrete
|
||||
// type) so a handler test can substitute a fake without spinning up a real
|
||||
// poller when it only needs to exercise coalescing or 404/400/409
|
||||
// short-circuits.
|
||||
type ReachabilityProber interface {
|
||||
ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool)
|
||||
EffectiveIntervalSeconds() int
|
||||
}
|
||||
|
||||
// --- HTTP handlers ---
|
||||
|
||||
func (h *Handler) httpListReachability(c *gin.Context) {
|
||||
dtos, status, err := h.listReachability(c.Request.Context())
|
||||
if err != nil {
|
||||
h.writeReachabilityError(c, status, err)
|
||||
return
|
||||
}
|
||||
c.JSON(status, dtos)
|
||||
}
|
||||
|
||||
func (h *Handler) httpGetReachability(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
|
||||
return
|
||||
}
|
||||
dto, status, err := h.getReachability(c.Request.Context(), id)
|
||||
if err != nil {
|
||||
h.writeReachabilityError(c, status, err)
|
||||
return
|
||||
}
|
||||
c.JSON(status, dto)
|
||||
}
|
||||
|
||||
func (h *Handler) httpProbeReachability(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: errExecutorIDRequired})
|
||||
return
|
||||
}
|
||||
dto, status, err := h.probeReachability(c.Request.Context(), id)
|
||||
if err != nil {
|
||||
h.writeReachabilityError(c, status, err)
|
||||
return
|
||||
}
|
||||
c.JSON(status, dto)
|
||||
}
|
||||
|
||||
func (h *Handler) writeReachabilityError(c *gin.Context, status int, err error) {
|
||||
message := "reachability request failed"
|
||||
if status >= http.StatusBadRequest && status < http.StatusInternalServerError {
|
||||
message = err.Error()
|
||||
} else if h.logger != nil {
|
||||
h.logger.Error("reachability request failed", zap.Int("status", status), zap.Error(err))
|
||||
}
|
||||
c.JSON(status, gin.H{errorJSONKey: message})
|
||||
}
|
||||
|
||||
// --- Testable cores ---
|
||||
|
||||
// listReachability returns one DTO per SSH executor that is not
|
||||
// soft-deleted, ordered by executor_id ascending — the same total order the
|
||||
// poller's own pass uses. An executor whose status is not active is still
|
||||
// included, carrying its retained record.
|
||||
func (h *Handler) listReachability(ctx context.Context) ([]reachabilitypkg.RecordDTO, int, error) {
|
||||
if h.reachabilityRepo == nil {
|
||||
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired")
|
||||
}
|
||||
executors, err := h.reachabilityRepo.ListExecutors(ctx)
|
||||
if err != nil {
|
||||
return nil, http.StatusInternalServerError, fmt.Errorf("list executors: %w", err)
|
||||
}
|
||||
records, err := h.reachabilityRepo.ListExecutorReachability(ctx)
|
||||
if err != nil {
|
||||
return nil, http.StatusInternalServerError, fmt.Errorf("list executor reachability: %w", err)
|
||||
}
|
||||
byExecutor := make(map[string]*models.ExecutorReachability, len(records))
|
||||
for _, record := range records {
|
||||
byExecutor[record.ExecutorID] = record
|
||||
}
|
||||
|
||||
sshExecutors := filterSSHExecutors(executors)
|
||||
sort.Slice(sshExecutors, func(i, j int) bool { return sshExecutors[i].ID < sshExecutors[j].ID })
|
||||
|
||||
interval := h.reachabilityIntervalSeconds()
|
||||
dtos := make([]reachabilitypkg.RecordDTO, 0, len(sshExecutors))
|
||||
for _, executor := range sshExecutors {
|
||||
dtos = append(dtos, buildReachabilityDTOFromRecord(executor, byExecutor[executor.ID], interval))
|
||||
}
|
||||
return dtos, http.StatusOK, nil
|
||||
}
|
||||
|
||||
// getReachability returns the single record for executorID. Never 404s for
|
||||
// a missing record — only for a missing or soft-deleted executor — per
|
||||
// AC-EXECUTORS-SSH-REACHABILITY-002.5.
|
||||
func (h *Handler) getReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) {
|
||||
executor, status, err := h.loadSSHExecutor(ctx, executorID)
|
||||
if err != nil {
|
||||
return nil, status, err
|
||||
}
|
||||
if h.reachabilityRepo == nil {
|
||||
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability listing not wired")
|
||||
}
|
||||
record, err := h.reachabilityRepo.GetExecutorReachability(ctx, executorID)
|
||||
if err != nil && !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
|
||||
return nil, http.StatusInternalServerError, fmt.Errorf("get executor reachability: %w", err)
|
||||
}
|
||||
if errors.Is(err, models.ErrExecutorReachabilityNotFound) {
|
||||
record = nil
|
||||
}
|
||||
dto := buildReachabilityDTOFromRecord(executor, record, h.reachabilityIntervalSeconds())
|
||||
return &dto, http.StatusOK, nil
|
||||
}
|
||||
|
||||
// probeReachability runs (or joins an in-flight) synchronous probe for
|
||||
// executorID and returns its persisted record. Coalesces concurrent callers
|
||||
// through reachabilityPoller.ProbeAndWait's own singleflight group.
|
||||
func (h *Handler) probeReachability(ctx context.Context, executorID string) (*reachabilitypkg.RecordDTO, int, error) {
|
||||
executor, status, err := h.loadSSHExecutor(ctx, executorID)
|
||||
if err != nil {
|
||||
return nil, status, err
|
||||
}
|
||||
if executor.Status != models.ExecutorStatusActive {
|
||||
return nil, http.StatusConflict, fmt.Errorf("executor %q is not active", executorID)
|
||||
}
|
||||
if h.reachabilityPoller == nil {
|
||||
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability probing not wired")
|
||||
}
|
||||
result, ok := h.reachabilityPoller.ProbeAndWait(executor)
|
||||
if !ok {
|
||||
return nil, http.StatusServiceUnavailable, fmt.Errorf("reachability poller is not accepting probes")
|
||||
}
|
||||
dto := reachabilitypkg.BuildRecordDTO(executorID, result.Record, h.reachabilityPoller.EffectiveIntervalSeconds(), result.Persisted)
|
||||
return &dto, http.StatusOK, nil
|
||||
}
|
||||
|
||||
// loadSSHExecutor enforces the 404/400 split the reachability routes need:
|
||||
// 404 for a nonexistent or soft-deleted id, 400 only when the executor
|
||||
// exists but isn't type ssh. Deliberately does not attempt config
|
||||
// resolution (unlike resolveSSHTarget) — an unresolvable ssh config is a 200
|
||||
// with reason config, decided by the caller, never a 400 here.
|
||||
func (h *Handler) loadSSHExecutor(ctx context.Context, executorID string) (*models.Executor, int, error) {
|
||||
executor, err := h.executorFetcher.GetExecutor(ctx, executorID)
|
||||
if err != nil {
|
||||
if errors.Is(err, models.ErrExecutorNotFound) {
|
||||
return nil, http.StatusNotFound, fmt.Errorf("executor %q not found: %w", executorID, err)
|
||||
}
|
||||
return nil, http.StatusInternalServerError, fmt.Errorf("look up executor %q: %w", executorID, err)
|
||||
}
|
||||
if executor.Type != models.ExecutorTypeSSH {
|
||||
return nil, http.StatusBadRequest, fmt.Errorf("executor %q is not an SSH executor", executorID)
|
||||
}
|
||||
return executor, http.StatusOK, nil
|
||||
}
|
||||
|
||||
func (h *Handler) reachabilityIntervalSeconds() int {
|
||||
if h.reachabilityPoller == nil {
|
||||
return 0
|
||||
}
|
||||
return h.reachabilityPoller.EffectiveIntervalSeconds()
|
||||
}
|
||||
|
||||
func filterSSHExecutors(executors []*models.Executor) []*models.Executor {
|
||||
out := make([]*models.Executor, 0, len(executors))
|
||||
for _, executor := range executors {
|
||||
if executor != nil && executor.Type == models.ExecutorTypeSSH {
|
||||
out = append(out, executor)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildReachabilityDTOFromRecord projects executor/record into the wire
|
||||
// shape without any I/O. record nil means "no record yet" — that synthesizes
|
||||
// the unknown placeholder unless the executor's own config can't resolve a
|
||||
// target, in which case AC-EXECUTORS-SSH-REACHABILITY-002.5's sibling rule
|
||||
// applies: 200 with reason config, the same outcome a real probe would reach
|
||||
// on its very first attempt, rather than a misleading "not probed yet".
|
||||
func buildReachabilityDTOFromRecord(executor *models.Executor, record *models.ExecutorReachability, intervalSeconds int) reachabilitypkg.RecordDTO {
|
||||
if record != nil {
|
||||
return reachabilitypkg.BuildRecordDTO(executor.ID, record, intervalSeconds, true)
|
||||
}
|
||||
if err := reachabilityConfigValidator.ValidateConfig(executor.Config); err != nil {
|
||||
synthetic := &models.ExecutorReachability{
|
||||
ExecutorID: executor.ID,
|
||||
State: models.ExecutorReachabilityStateUnreachable,
|
||||
Reason: models.ExecutorReachabilityReasonConfig,
|
||||
Message: agentruntime.SanitizeSSHReachabilityMessage(err),
|
||||
Host: executor.Config["ssh_host"],
|
||||
}
|
||||
return reachabilitypkg.BuildRecordDTO(executor.ID, synthetic, intervalSeconds, false)
|
||||
}
|
||||
return reachabilitypkg.BuildRecordDTO(executor.ID, nil, intervalSeconds, false)
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
package ssh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
reachabilitypkg "github.com/kandev/kandev/internal/executors/reachability"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
func testHandlerLogger() *logger.Logger {
|
||||
return logger.Default()
|
||||
}
|
||||
|
||||
type fakeReachabilityExecutorFetcher struct {
|
||||
executors map[string]*models.Executor
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityExecutorFetcher) GetExecutor(_ context.Context, id string) (*models.Executor, error) {
|
||||
executor, ok := f.executors[id]
|
||||
if !ok {
|
||||
return nil, models.ErrExecutorNotFound
|
||||
}
|
||||
return executor, nil
|
||||
}
|
||||
|
||||
type fakeReachabilityLister struct {
|
||||
executors []*models.Executor
|
||||
records []*models.ExecutorReachability
|
||||
byID map[string]*models.ExecutorReachability
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityLister) ListExecutors(_ context.Context) ([]*models.Executor, error) {
|
||||
return f.executors, nil
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityLister) ListExecutorReachability(_ context.Context) ([]*models.ExecutorReachability, error) {
|
||||
return f.records, nil
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityLister) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
if record, ok := f.byID[executorID]; ok {
|
||||
return record, nil
|
||||
}
|
||||
return nil, models.ErrExecutorReachabilityNotFound
|
||||
}
|
||||
|
||||
func sshExecutor(id string, status models.ExecutorStatus, config map[string]string) *models.Executor {
|
||||
return &models.Executor{ID: id, Type: models.ExecutorTypeSSH, Status: status, Config: config}
|
||||
}
|
||||
|
||||
func resolvableSSHConfig() map[string]string {
|
||||
return map[string]string{
|
||||
"ssh_host": "10.0.0.1",
|
||||
"ssh_user": "deploy",
|
||||
"ssh_host_fingerprint": "SHA256:test",
|
||||
}
|
||||
}
|
||||
|
||||
// fakeReachabilityProber models the exact guarantee *reachability.Poller
|
||||
// makes for concurrent callers of the same executor id: it coalesces
|
||||
// through the same golang.org/x/sync/singleflight primitive the production
|
||||
// Poller uses, so this test proves the handler's own plumbing (always
|
||||
// routing through the one shared prober instance) rather than re-deriving
|
||||
// singleflight's own coalescing guarantee, which is already covered by
|
||||
// TestPollerProbeAndWait_CoalescesConcurrentCallsForSameExecutor.
|
||||
type fakeReachabilityProber struct {
|
||||
group singleflight.Group
|
||||
dialCount int32
|
||||
callsEntered int32
|
||||
gate chan struct{}
|
||||
result reachabilitypkg.ProbeResult
|
||||
interval int
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityProber) ProbeAndWait(executor *models.Executor) (reachabilitypkg.ProbeResult, bool) {
|
||||
atomic.AddInt32(&f.callsEntered, 1)
|
||||
v, _, _ := f.group.Do(executor.ID, func() (any, error) {
|
||||
atomic.AddInt32(&f.dialCount, 1)
|
||||
<-f.gate
|
||||
return f.result, nil
|
||||
})
|
||||
return v.(reachabilitypkg.ProbeResult), true
|
||||
}
|
||||
|
||||
func (f *fakeReachabilityProber) EffectiveIntervalSeconds() int { return f.interval }
|
||||
|
||||
func waitForCallsEntered(t *testing.T, prober *fakeReachabilityProber, want int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if atomic.LoadInt32(&prober.callsEntered) >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatalf("callsEntered did not reach %d within 1s (got %d)", want, atomic.LoadInt32(&prober.callsEntered))
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002 (coalescing)
|
||||
//
|
||||
// Two overlapping probe requests for the same executor must share one dial
|
||||
// and receive one identical record. Written first as a RED test against a
|
||||
// handler that has not yet been implemented, per the task-04 work order.
|
||||
func TestProbeReachability_CoalescesConcurrentProbes(t *testing.T) {
|
||||
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
updatedAt := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
prober := &fakeReachabilityProber{
|
||||
gate: make(chan struct{}),
|
||||
result: reachabilitypkg.ProbeResult{
|
||||
Record: &models.ExecutorReachability{
|
||||
ExecutorID: "exec-1",
|
||||
State: models.ExecutorReachabilityStateReachable,
|
||||
UpdatedAt: updatedAt,
|
||||
},
|
||||
Persisted: true,
|
||||
},
|
||||
interval: 30,
|
||||
}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober)
|
||||
|
||||
const callers = 2
|
||||
dtos := make([]*reachabilitypkg.RecordDTO, callers)
|
||||
statuses := make([]int, callers)
|
||||
errs := make([]error, callers)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < callers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
dto, status, err := h.probeReachability(context.Background(), "exec-1")
|
||||
dtos[i] = dto
|
||||
statuses[i] = status
|
||||
errs[i] = err
|
||||
}(i)
|
||||
}
|
||||
|
||||
waitForCallsEntered(t, prober, callers)
|
||||
close(prober.gate)
|
||||
wg.Wait()
|
||||
|
||||
if got := atomic.LoadInt32(&prober.dialCount); got != 1 {
|
||||
t.Fatalf("dialCount = %d, want 1 (concurrent probes must coalesce into one dial)", got)
|
||||
}
|
||||
for i := 0; i < callers; i++ {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("caller %d: unexpected error: %v", i, errs[i])
|
||||
}
|
||||
if statuses[i] != 200 {
|
||||
t.Fatalf("caller %d: status = %d, want 200", i, statuses[i])
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(dtos[0], dtos[1]) {
|
||||
t.Fatalf("responses differ: %+v vs %+v", dtos[0], dtos[1])
|
||||
}
|
||||
if dtos[0].State != string(models.ExecutorReachabilityStateReachable) {
|
||||
t.Fatalf("state = %q, want %q", dtos[0].State, models.ExecutorReachabilityStateReachable)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
|
||||
func TestGetReachability_NoRecordAndResolvableConfigReturnsUnknown(t *testing.T) {
|
||||
executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig())
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
lister := &fakeReachabilityLister{}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
|
||||
|
||||
dto, status, err := h.getReachability(context.Background(), "exec-1")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200", status)
|
||||
}
|
||||
if dto.State != string(models.ExecutorReachabilityStateUnknown) {
|
||||
t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateUnknown)
|
||||
}
|
||||
if dto.Reason != "" {
|
||||
t.Fatalf("reason = %q, want empty", dto.Reason)
|
||||
}
|
||||
if dto.Persisted {
|
||||
t.Fatal("persisted = true for a synthesized never-probed record")
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
|
||||
//
|
||||
// An ssh executor whose config cannot resolve a target (no ssh_host here)
|
||||
// must answer 200 with reason "config", never 400 — resolveSSHTarget's own
|
||||
// 400-on-unresolvable-config mapping does not apply to the reachability
|
||||
// routes.
|
||||
func TestGetReachability_NoRecordAndUnresolvableConfigReturnsReasonConfig(t *testing.T) {
|
||||
executor := sshExecutor("exec-1", models.ExecutorStatusActive, map[string]string{})
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
lister := &fakeReachabilityLister{}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
|
||||
|
||||
dto, status, err := h.getReachability(context.Background(), "exec-1")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200", status)
|
||||
}
|
||||
if dto.Reason != string(models.ExecutorReachabilityReasonConfig) {
|
||||
t.Fatalf("reason = %q, want %q", dto.Reason, models.ExecutorReachabilityReasonConfig)
|
||||
}
|
||||
if dto.Persisted {
|
||||
t.Fatal("persisted = true for a synthesized record that was never written")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetReachability_ReturnsStoredRecordVerbatim(t *testing.T) {
|
||||
executor := sshExecutor("exec-1", models.ExecutorStatusActive, resolvableSSHConfig())
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
stored := &models.ExecutorReachability{
|
||||
ExecutorID: "exec-1",
|
||||
State: models.ExecutorReachabilityStateReachable,
|
||||
Host: "10.0.0.1",
|
||||
}
|
||||
lister := &fakeReachabilityLister{byID: map[string]*models.ExecutorReachability{"exec-1": stored}}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
|
||||
|
||||
dto, status, err := h.getReachability(context.Background(), "exec-1")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200", status)
|
||||
}
|
||||
if !dto.Persisted {
|
||||
t.Fatal("persisted = false for an actually-stored record")
|
||||
}
|
||||
if dto.State != string(models.ExecutorReachabilityStateReachable) {
|
||||
t.Fatalf("state = %q, want %q", dto.State, models.ExecutorReachabilityStateReachable)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
|
||||
func TestGetReachability_NonexistentExecutorReturns404(t *testing.T) {
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil)
|
||||
|
||||
_, status, err := h.getReachability(context.Background(), "missing")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a nonexistent executor")
|
||||
}
|
||||
if status != 404 {
|
||||
t.Fatalf("status = %d, want 404", status)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.5
|
||||
//
|
||||
// A wrong-type executor id is a 400, distinct from the config-unresolvable
|
||||
// case (which is a 200 with reason config).
|
||||
func TestGetReachability_WrongTypeExecutorReturns400(t *testing.T) {
|
||||
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), &fakeReachabilityLister{}, nil)
|
||||
|
||||
_, status, err := h.getReachability(context.Background(), "exec-1")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a non-ssh executor")
|
||||
}
|
||||
if status != 400 {
|
||||
t.Fatalf("status = %d, want 400", status)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.4
|
||||
//
|
||||
// The list route orders by executor_id ascending, includes a deactivated
|
||||
// ssh executor's retained record, and omits non-ssh executors entirely.
|
||||
func TestListReachability_OrdersFiltersAndIncludesDeactivated(t *testing.T) {
|
||||
sshA := sshExecutor("ssh-b", models.ExecutorStatusActive, resolvableSSHConfig())
|
||||
sshB := sshExecutor("ssh-a", models.ExecutorStatusDisabled, resolvableSSHConfig())
|
||||
nonSSH := &models.Executor{ID: "local-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
|
||||
fetcher := &fakeReachabilityExecutorFetcher{}
|
||||
lister := &fakeReachabilityLister{
|
||||
executors: []*models.Executor{sshA, sshB, nonSSH},
|
||||
records: []*models.ExecutorReachability{
|
||||
{ExecutorID: "ssh-a", State: models.ExecutorReachabilityStateUnreachable, Reason: models.ExecutorReachabilityReasonTimeout},
|
||||
},
|
||||
}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), lister, nil)
|
||||
|
||||
dtos, status, err := h.listReachability(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if status != 200 {
|
||||
t.Fatalf("status = %d, want 200", status)
|
||||
}
|
||||
if len(dtos) != 2 {
|
||||
t.Fatalf("len(dtos) = %d, want 2 (non-ssh executor must be omitted)", len(dtos))
|
||||
}
|
||||
if dtos[0].ExecutorID != "ssh-a" || dtos[1].ExecutorID != "ssh-b" {
|
||||
t.Fatalf("order = [%s, %s], want [ssh-a, ssh-b] (executor_id ascending)", dtos[0].ExecutorID, dtos[1].ExecutorID)
|
||||
}
|
||||
if dtos[0].State != string(models.ExecutorReachabilityStateUnreachable) {
|
||||
t.Fatalf("deactivated executor's record state = %q, want %q (retained record)", dtos[0].State, models.ExecutorReachabilityStateUnreachable)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.6
|
||||
//
|
||||
// A POST against a deactivated ssh executor answers 409 without probing.
|
||||
func TestProbeReachability_InactiveExecutorReturns409WithoutProbing(t *testing.T) {
|
||||
executor := sshExecutor("exec-1", models.ExecutorStatusDisabled, resolvableSSHConfig())
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
prober := &fakeReachabilityProber{gate: make(chan struct{})}
|
||||
close(prober.gate) // would return immediately if ever called
|
||||
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, prober)
|
||||
_, status, err := h.probeReachability(context.Background(), "exec-1")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a probe against an inactive executor")
|
||||
}
|
||||
if status != 409 {
|
||||
t.Fatalf("status = %d, want 409", status)
|
||||
}
|
||||
if got := atomic.LoadInt32(&prober.callsEntered); got != 0 {
|
||||
t.Fatalf("callsEntered = %d, want 0 (must not probe an inactive executor)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeReachability_NonexistentExecutorReturns404(t *testing.T) {
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{}}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{})
|
||||
|
||||
_, status, err := h.probeReachability(context.Background(), "missing")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a nonexistent executor")
|
||||
}
|
||||
if status != 404 {
|
||||
t.Fatalf("status = %d, want 404", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeReachability_WrongTypeExecutorReturns400(t *testing.T) {
|
||||
executor := &models.Executor{ID: "exec-1", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive}
|
||||
fetcher := &fakeReachabilityExecutorFetcher{executors: map[string]*models.Executor{"exec-1": executor}}
|
||||
h := NewHandler(nil, fetcher, nil, nil, testHandlerLogger(), nil, &fakeReachabilityProber{})
|
||||
|
||||
_, status, err := h.probeReachability(context.Background(), "exec-1")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a non-ssh executor")
|
||||
}
|
||||
if status != 400 {
|
||||
t.Fatalf("status = %d, want 400", status)
|
||||
}
|
||||
}
|
||||
@@ -607,6 +607,24 @@ func (m *mockRepository) UpdateExecutorRunningStatus(ctx context.Context, sessio
|
||||
func (m *mockRepository) RepairExecutorRunningDead(ctx context.Context, sessionID string) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockRepository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
return nil, models.ErrExecutorReachabilityNotFound
|
||||
}
|
||||
func (m *mockRepository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockRepository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockRepository) DeleteExecutorReachability(ctx context.Context, executorID string) error {
|
||||
return nil
|
||||
}
|
||||
func (m *mockRepository) CreateEnvironment(ctx context.Context, environment *models.Environment) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrExecutorReachabilityNotFound is returned when no reachability record
|
||||
// exists for the requested executor.
|
||||
var ErrExecutorReachabilityNotFound = errors.New("executor reachability record not found")
|
||||
|
||||
// ExecutorReachabilityState is the coarse status shown to operators.
|
||||
type ExecutorReachabilityState string
|
||||
|
||||
const (
|
||||
ExecutorReachabilityStateUnknown ExecutorReachabilityState = "unknown"
|
||||
ExecutorReachabilityStateReachable ExecutorReachabilityState = "reachable"
|
||||
ExecutorReachabilityStateUnreachable ExecutorReachabilityState = "unreachable"
|
||||
)
|
||||
|
||||
// ExecutorReachabilityReason mirrors lifecycle.SSHReachabilityReason's closed
|
||||
// set as plain strings — this package sits below the agent runtime tier and
|
||||
// must not import it. An empty reason means the observation was a success.
|
||||
type ExecutorReachabilityReason string
|
||||
|
||||
const (
|
||||
ExecutorReachabilityReasonConfig ExecutorReachabilityReason = "config"
|
||||
ExecutorReachabilityReasonTimeout ExecutorReachabilityReason = "timeout"
|
||||
ExecutorReachabilityReasonHostKey ExecutorReachabilityReason = "host_key"
|
||||
ExecutorReachabilityReasonAuth ExecutorReachabilityReason = "auth"
|
||||
ExecutorReachabilityReasonNetwork ExecutorReachabilityReason = "network"
|
||||
ExecutorReachabilityReasonUnknown ExecutorReachabilityReason = "unknown"
|
||||
)
|
||||
|
||||
// ExecutorReachability is the single stored row for one SSH executor's
|
||||
// observed reachability. It lives in its own table, never on Executor
|
||||
// itself — Executor.Config is user-authored input and Executor.Status is a
|
||||
// user-controlled switch, and neither should share a row with observed
|
||||
// state.
|
||||
type ExecutorReachability struct {
|
||||
ExecutorID string `json:"executor_id"`
|
||||
State ExecutorReachabilityState `json:"state"`
|
||||
Reason ExecutorReachabilityReason `json:"reason"`
|
||||
Message string `json:"message"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
Host string `json:"host"`
|
||||
CheckedAt *time.Time `json:"checked_at,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ExecutorReachabilityObservation is what a single probe (or a launch's own
|
||||
// dial attempt) reports to UpsertExecutorReachability. The derived counter
|
||||
// and state for an existing row live entirely in the upsert's own SQL — this
|
||||
// struct carries only the observation and the parameters the SQL needs to
|
||||
// derive from it. InitialState and InitialFailures are used only when no row
|
||||
// exists yet for the executor; every later write derives both from the
|
||||
// stored row instead.
|
||||
type ExecutorReachabilityObservation struct {
|
||||
ExecutorID string
|
||||
SeenUpdatedAt time.Time
|
||||
InitialState ExecutorReachabilityState
|
||||
InitialFailures int
|
||||
Reason ExecutorReachabilityReason
|
||||
Message string
|
||||
Host string
|
||||
CheckedAt time.Time
|
||||
FailureThreshold int
|
||||
}
|
||||
@@ -706,6 +706,33 @@ type ExecutorRepository interface {
|
||||
// the resume-safety invariant instead of deleting a resumable row.
|
||||
// Returns models.ErrExecutorRunningNotFound if no row exists for the session.
|
||||
RepairExecutorRunningDead(ctx context.Context, sessionID string) error
|
||||
|
||||
// ListSSHExecutorsForReachability returns every eligible SSH executor
|
||||
// (type=ssh, not soft-deleted, status=active) ordered ascending by id —
|
||||
// the poller's per-pass work list.
|
||||
ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error)
|
||||
// GetExecutorReachability returns the stored record for one executor.
|
||||
// Returns models.ErrExecutorReachabilityNotFound if none exists.
|
||||
GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error)
|
||||
// ListExecutorReachability returns every stored reachability record.
|
||||
ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error)
|
||||
// UpsertExecutorReachability records a single probe (or launch dial)
|
||||
// observation. The consecutive-failure counter and derived state are
|
||||
// computed by the statement itself from the row's own prior values, and
|
||||
// a write is discarded when obs.CheckedAt is not strictly later than the
|
||||
// stored checked_at — see the system design's Persistence section.
|
||||
UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error
|
||||
// ResetExecutorReachability invalidates the stored record after a
|
||||
// connection-configuration save when seenUpdatedAt still matches the
|
||||
// executor row. The version guard prevents a delayed save callback from
|
||||
// resetting a newer configuration. State becomes unknown, the counter and
|
||||
// reason/message clear, host is set to the newly saved value, and both
|
||||
// timestamps become NULL. A no-op (zero rows affected) when the executor
|
||||
// is not an active SSH executor or the version is stale.
|
||||
ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error
|
||||
// DeleteExecutorReachability removes the stored record. DeleteExecutor
|
||||
// calls this in the same transaction as the soft delete.
|
||||
DeleteExecutorReachability(ctx context.Context, executorID string) error
|
||||
}
|
||||
|
||||
// EnvironmentRepository handles environment CRUD.
|
||||
|
||||
@@ -423,6 +423,24 @@ func (r *Repository) runMigrations(ctx context.Context) error {
|
||||
// allocated.
|
||||
_ = r.migrate.Apply("workflow_step_entries.marker_positions", `ALTER TABLE workflow_step_entries ADD COLUMN marker_positions TEXT NOT NULL DEFAULT ''`)
|
||||
|
||||
// One row per SSH executor holding observed reachability — deliberately
|
||||
// not columns on executors, which holds user-authored config and a
|
||||
// user-controlled status switch. No foreign key: deletion is explicit
|
||||
// (DeleteExecutor deletes the row in the same transaction as the soft
|
||||
// delete), not a cascade.
|
||||
_ = r.migrate.Apply("executor_reachability.table", `
|
||||
CREATE TABLE IF NOT EXISTS executor_reachability (
|
||||
executor_id TEXT PRIMARY KEY,
|
||||
state TEXT NOT NULL DEFAULT 'unknown',
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
host TEXT NOT NULL DEFAULT '',
|
||||
checked_at TIMESTAMP,
|
||||
last_success_at TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL
|
||||
)`)
|
||||
|
||||
// Checked last so a failure on any required migration above --
|
||||
// including this file's own marker_positions column -- fails startup
|
||||
// instead of leaving a schema that allocateStepEntryIfPending can't write to.
|
||||
|
||||
@@ -94,7 +94,13 @@ func (r *Repository) UpdateExecutor(ctx context.Context, executor *models.Execut
|
||||
|
||||
func (r *Repository) DeleteExecutor(ctx context.Context, id string) error {
|
||||
now := time.Now().UTC()
|
||||
result, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
tx, err := r.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin executor delete: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
result, err := tx.ExecContext(ctx, tx.Rebind(`
|
||||
UPDATE executors SET deleted_at = ?, updated_at = ? WHERE id = ? AND deleted_at IS NULL
|
||||
`), now, now, id)
|
||||
if err != nil {
|
||||
@@ -104,9 +110,42 @@ func (r *Repository) DeleteExecutor(ctx context.Context, id string) error {
|
||||
if rows == 0 {
|
||||
return fmt.Errorf("executor not found: %s", id)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, tx.Rebind(`
|
||||
DELETE FROM executor_reachability WHERE executor_id = ?
|
||||
`), id); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit executor delete: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanExecutorRow scans one row of the standard executors column list
|
||||
// (id, name, type, status, is_system, resumable, config, created_at,
|
||||
// updated_at, deleted_at) into a models.Executor. Shared by every query that
|
||||
// selects that exact column set.
|
||||
func scanExecutorRow(scanner interface{ Scan(...any) error }) (*models.Executor, error) {
|
||||
executor := &models.Executor{}
|
||||
var configJSON string
|
||||
var isSystem int
|
||||
var resumable int
|
||||
if err := scanner.Scan(
|
||||
&executor.ID, &executor.Name, &executor.Type, &executor.Status,
|
||||
&isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
executor.IsSystem = isSystem == 1
|
||||
executor.Resumable = resumable == 1
|
||||
if configJSON != "" && configJSON != "{}" {
|
||||
if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil {
|
||||
return nil, fmt.Errorf("failed to deserialize executor config: %w", err)
|
||||
}
|
||||
}
|
||||
return executor, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, error) {
|
||||
rows, err := r.ro.QueryContext(ctx, `
|
||||
SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at
|
||||
@@ -119,23 +158,10 @@ func (r *Repository) ListExecutors(ctx context.Context) ([]*models.Executor, err
|
||||
|
||||
var result []*models.Executor
|
||||
for rows.Next() {
|
||||
executor := &models.Executor{}
|
||||
var configJSON string
|
||||
var isSystem int
|
||||
var resumable int
|
||||
if err := rows.Scan(
|
||||
&executor.ID, &executor.Name, &executor.Type, &executor.Status,
|
||||
&isSystem, &resumable, &configJSON, &executor.CreatedAt, &executor.UpdatedAt, &executor.DeletedAt,
|
||||
); err != nil {
|
||||
executor, err := scanExecutorRow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
executor.IsSystem = isSystem == 1
|
||||
executor.Resumable = resumable == 1
|
||||
if configJSON != "" && configJSON != "{}" {
|
||||
if err := json.Unmarshal([]byte(configJSON), &executor.Config); err != nil {
|
||||
return nil, fmt.Errorf("failed to deserialize executor config: %w", err)
|
||||
}
|
||||
}
|
||||
result = append(result, executor)
|
||||
}
|
||||
return result, rows.Err()
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
const executorReachabilityColumns = `executor_id, state, reason, message, consecutive_failures, host, checked_at, last_success_at, updated_at`
|
||||
|
||||
func scanExecutorReachability(scanner interface{ Scan(...any) error }) (*models.ExecutorReachability, error) {
|
||||
r := &models.ExecutorReachability{}
|
||||
var state, reason string
|
||||
var checkedAt, lastSuccessAt sql.NullTime
|
||||
if err := scanner.Scan(
|
||||
&r.ExecutorID, &state, &reason, &r.Message, &r.ConsecutiveFailures, &r.Host,
|
||||
&checkedAt, &lastSuccessAt, &r.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.State = models.ExecutorReachabilityState(state)
|
||||
r.Reason = models.ExecutorReachabilityReason(reason)
|
||||
if checkedAt.Valid {
|
||||
v := checkedAt.Time
|
||||
r.CheckedAt = &v
|
||||
}
|
||||
if lastSuccessAt.Valid {
|
||||
v := lastSuccessAt.Time
|
||||
r.LastSuccessAt = &v
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// ListSSHExecutorsForReachability returns every eligible SSH executor
|
||||
// (type=ssh, not soft-deleted, status=active) ordered ascending by id, the
|
||||
// primary key, never reused, so the order is total and needs no tiebreak.
|
||||
func (r *Repository) ListSSHExecutorsForReachability(ctx context.Context) ([]*models.Executor, error) {
|
||||
rows, err := r.ro.QueryContext(ctx, `
|
||||
SELECT id, name, type, status, is_system, resumable, config, created_at, updated_at, deleted_at
|
||||
FROM executors
|
||||
WHERE type = 'ssh' AND deleted_at IS NULL AND status = 'active'
|
||||
ORDER BY id ASC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []*models.Executor
|
||||
for rows.Next() {
|
||||
executor, err := scanExecutorRow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, executor)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// GetExecutorReachability returns the stored record for one executor.
|
||||
// Returns models.ErrExecutorReachabilityNotFound if none exists.
|
||||
func (r *Repository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
|
||||
row := r.ro.QueryRowContext(ctx, r.ro.Rebind(`
|
||||
SELECT `+executorReachabilityColumns+` FROM executor_reachability WHERE executor_id = ?
|
||||
`), executorID)
|
||||
rec, err := scanExecutorReachability(row)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, models.ErrExecutorReachabilityNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// ListExecutorReachability returns every stored reachability record.
|
||||
func (r *Repository) ListExecutorReachability(ctx context.Context) ([]*models.ExecutorReachability, error) {
|
||||
rows, err := r.ro.QueryContext(ctx, `
|
||||
SELECT `+executorReachabilityColumns+` FROM executor_reachability ORDER BY executor_id ASC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []*models.ExecutorReachability
|
||||
for rows.Next() {
|
||||
rec, err := scanExecutorReachability(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, rec)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// UpsertExecutorReachability records a single probe (or launch dial)
|
||||
// observation. The write is a single statement: the consecutive-failure
|
||||
// counter and the derived state for an existing row are computed by the
|
||||
// statement itself from the row's own prior values, so no read-modify-write
|
||||
// in Go can lose a streak to an interleaving. A write whose CheckedAt is not
|
||||
// strictly later than the stored checked_at is discarded (last-write-wins),
|
||||
// and the row is only touched at all when the executor is still the active
|
||||
// SSH executor obs.SeenUpdatedAt was captured against — see the system
|
||||
// design's Persistence section for the full contract.
|
||||
func (r *Repository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
|
||||
now := time.Now().UTC()
|
||||
checkedAt := obs.CheckedAt.UTC()
|
||||
var lastSuccessAt any
|
||||
if obs.Reason == "" {
|
||||
lastSuccessAt = checkedAt
|
||||
}
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
INSERT INTO executor_reachability (
|
||||
executor_id, state, reason, message, consecutive_failures, host,
|
||||
checked_at, last_success_at, updated_at
|
||||
)
|
||||
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
WHERE EXISTS (SELECT 1 FROM executors e
|
||||
WHERE e.id = ? AND e.deleted_at IS NULL
|
||||
AND e.status = 'active' AND e.updated_at = ?)
|
||||
ON CONFLICT (executor_id) DO UPDATE SET
|
||||
consecutive_failures = CASE WHEN excluded.reason = '' THEN 0
|
||||
ELSE executor_reachability.consecutive_failures + 1 END,
|
||||
state = CASE WHEN excluded.reason = '' THEN 'reachable'
|
||||
WHEN excluded.reason IN ('config','host_key') THEN 'unreachable'
|
||||
WHEN executor_reachability.consecutive_failures + 1 >= ?
|
||||
THEN 'unreachable'
|
||||
ELSE executor_reachability.state END,
|
||||
reason = excluded.reason, message = excluded.message, host = excluded.host,
|
||||
checked_at = excluded.checked_at,
|
||||
last_success_at = CASE WHEN excluded.reason = '' THEN excluded.checked_at
|
||||
ELSE executor_reachability.last_success_at END,
|
||||
updated_at = excluded.updated_at
|
||||
WHERE executor_reachability.checked_at IS NULL
|
||||
OR excluded.checked_at > executor_reachability.checked_at
|
||||
`),
|
||||
obs.ExecutorID, string(obs.InitialState), string(obs.Reason), obs.Message, obs.InitialFailures, obs.Host,
|
||||
checkedAt, lastSuccessAt, now,
|
||||
obs.ExecutorID, obs.SeenUpdatedAt.UTC(),
|
||||
obs.FailureThreshold,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// ResetExecutorReachability invalidates the stored record after a
|
||||
// connection-configuration save. It is its own statement, not a branch of
|
||||
// the upsert above: the upsert's state CASE has no unknown branch and its
|
||||
// WHERE admits only a strictly later checked_at, which a reset does not
|
||||
// carry. The executor updated_at predicate couples the reset to the save
|
||||
// version, so a delayed callback cannot reset a newer configuration. A
|
||||
// no-op when the executor isn't an active SSH executor or the version is
|
||||
// stale: AC-EXECUTORS-SSH-REACHABILITY-001.17 keeps a disabled executor's
|
||||
// record untouched, so eligibility wins over the reset trigger.
|
||||
func (r *Repository) ResetExecutorReachability(ctx context.Context, executorID, host string, seenUpdatedAt time.Time) error {
|
||||
now := time.Now().UTC()
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
INSERT INTO executor_reachability (
|
||||
executor_id, state, reason, message, consecutive_failures,
|
||||
checked_at, host, last_success_at, updated_at
|
||||
)
|
||||
SELECT ?, 'unknown', '', '', 0, NULL, ?, NULL, ?
|
||||
WHERE EXISTS (SELECT 1 FROM executors e
|
||||
WHERE e.id = ? AND e.type = 'ssh'
|
||||
AND e.deleted_at IS NULL AND e.status = 'active'
|
||||
AND e.updated_at = ?)
|
||||
ON CONFLICT (executor_id) DO UPDATE SET
|
||||
state = 'unknown', reason = '', message = '', consecutive_failures = 0,
|
||||
host = excluded.host, checked_at = NULL, last_success_at = NULL,
|
||||
updated_at = excluded.updated_at
|
||||
`), executorID, host, now, executorID, seenUpdatedAt.UTC())
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteExecutorReachability removes the stored record for one executor.
|
||||
func (r *Repository) DeleteExecutorReachability(ctx context.Context, executorID string) error {
|
||||
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
|
||||
DELETE FROM executor_reachability WHERE executor_id = ?
|
||||
`), executorID)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/testutil"
|
||||
)
|
||||
|
||||
// TestPostgresExecutorReachabilityCheckedAtOrdering guards the SQLite-only
|
||||
// risk the design calls out: TIMESTAMP takes NUMERIC affinity on SQLite and
|
||||
// compares the driver's text encoding lexically, so a whole-second value can
|
||||
// sort after a later sub-second one unless writes are normalized to UTC and
|
||||
// bind time.Time rather than a pre-formatted string. Postgres compares
|
||||
// temporally and is unaffected either way, so this test alone would pass
|
||||
// even with the bug — it exists to prove the fix doesn't regress on
|
||||
// Postgres, not to catch the bug (that's the SQLite test's job).
|
||||
func TestPostgresExecutorReachabilityCheckedAtOrdering(t *testing.T) {
|
||||
db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t))
|
||||
repo, err := NewWithDB(db, db, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("init postgres schema: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
executor := &models.Executor{ID: "exec-reachability-pg", Name: "ssh-host", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
|
||||
if err := repo.CreateExecutor(ctx, executor); err != nil {
|
||||
t.Fatalf("create executor: %v", err)
|
||||
}
|
||||
|
||||
older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC)
|
||||
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert newer: %v", err)
|
||||
}
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1,
|
||||
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
|
||||
Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert older: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("record = %+v, want the newer write to survive", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/kandev/kandev/internal/db"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
func newReachabilityTestRepo(t *testing.T) (*Repository, string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "reachability.db")
|
||||
dbConn, err := db.OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
|
||||
repo, err := NewWithDB(sqlxDB, sqlxDB, nil)
|
||||
if err != nil {
|
||||
_ = sqlxDB.Close()
|
||||
t.Fatalf("new repository: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sqlxDB.Close() })
|
||||
return repo, path
|
||||
}
|
||||
|
||||
// createSSHExecutor inserts an active SSH executor and returns it, so
|
||||
// UpsertExecutorReachability's eligibility EXISTS clause is satisfied.
|
||||
func createSSHExecutor(t *testing.T, repo *Repository) *models.Executor {
|
||||
t.Helper()
|
||||
executor := &models.Executor{
|
||||
ID: "exec-" + t.Name(),
|
||||
Name: "ssh-host",
|
||||
Type: models.ExecutorTypeSSH,
|
||||
Status: models.ExecutorStatusActive,
|
||||
}
|
||||
if err := repo.CreateExecutor(context.Background(), executor); err != nil {
|
||||
t.Fatalf("create executor: %v", err)
|
||||
}
|
||||
return executor
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — last-write-wins on
|
||||
// checked_at regardless of commit order, including a whole-second value
|
||||
// racing a sub-second one.
|
||||
func TestUpsertExecutorReachabilityNewerCheckedAtWinsRegardlessOfOrder(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
|
||||
older := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
newer := time.Date(2026, 1, 1, 12, 0, 0, 500_000_000, time.UTC) // same whole second, later fraction
|
||||
|
||||
// Issue the newer write first, the older write second — the newer must survive.
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.2", CheckedAt: newer, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert newer: %v", err)
|
||||
}
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateUnreachable,
|
||||
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
|
||||
Host: "10.0.0.1", CheckedAt: older, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert older: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Host != "10.0.0.2" || got.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("record = %+v, want the newer (reachable, 10.0.0.2) write to survive", got)
|
||||
}
|
||||
if got.CheckedAt == nil || !got.CheckedAt.Equal(newer) {
|
||||
t.Fatalf("CheckedAt = %v, want %v", got.CheckedAt, newer)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — an equal checked_at leaves
|
||||
// the row unchanged.
|
||||
func TestUpsertExecutorReachabilityEqualCheckedAtIsDiscarded(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
|
||||
at := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
obs := models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
|
||||
}
|
||||
if err := repo.UpsertExecutorReachability(ctx, obs); err != nil {
|
||||
t.Fatalf("first upsert: %v", err)
|
||||
}
|
||||
|
||||
obs.Host = "10.0.0.9" // a second write at the SAME checked_at, different host
|
||||
if err := repo.UpsertExecutorReachability(ctx, obs); err != nil {
|
||||
t.Fatalf("second upsert: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Host != "10.0.0.1" {
|
||||
t.Fatalf("Host = %q, want the first write's host unchanged (equal checked_at must be discarded)", got.Host)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.16 — a record survives a
|
||||
// repository re-open against the same database file, byte-identical
|
||||
// including both timestamps.
|
||||
func TestExecutorReachabilityRoundTripAcrossReopen(t *testing.T) {
|
||||
repo, path := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
|
||||
at := time.Date(2026, 3, 4, 5, 6, 7, 123_000_000, time.UTC)
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
before, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get before reopen: %v", err)
|
||||
}
|
||||
|
||||
dbConn, err := db.OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen sqlite: %v", err)
|
||||
}
|
||||
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
|
||||
t.Cleanup(func() { _ = sqlxDB.Close() })
|
||||
reopened, err := NewWithDB(sqlxDB, sqlxDB, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen repository: %v", err)
|
||||
}
|
||||
|
||||
after, err := reopened.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get after reopen: %v", err)
|
||||
}
|
||||
if after.Host != before.Host || after.State != before.State {
|
||||
t.Fatalf("record changed across reopen: before=%+v after=%+v", before, after)
|
||||
}
|
||||
if after.CheckedAt == nil || before.CheckedAt == nil || !after.CheckedAt.Equal(*before.CheckedAt) {
|
||||
t.Fatalf("CheckedAt changed across reopen: before=%v after=%v", before.CheckedAt, after.CheckedAt)
|
||||
}
|
||||
if after.LastSuccessAt == nil || before.LastSuccessAt == nil || !after.LastSuccessAt.Equal(*before.LastSuccessAt) {
|
||||
t.Fatalf("LastSuccessAt changed across reopen: before=%v after=%v", before.LastSuccessAt, after.LastSuccessAt)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.21
|
||||
func TestDeleteExecutorRemovesReachabilityRecord(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.DeleteExecutor(ctx, executor.ID); err != nil {
|
||||
t.Fatalf("delete executor: %v", err)
|
||||
}
|
||||
|
||||
if _, err := repo.GetExecutorReachability(ctx, executor.ID); err != models.ErrExecutorReachabilityNotFound {
|
||||
t.Fatalf("err = %v, want ErrExecutorReachabilityNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.22
|
||||
func TestResetExecutorReachabilityClearsTheRecordButAdvancesUpdatedAt(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
|
||||
at := time.Now().UTC()
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateUnreachable, InitialFailures: 1,
|
||||
Reason: models.ExecutorReachabilityReasonNetwork, Message: "connection refused",
|
||||
Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
before, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get before reset: %v", err)
|
||||
}
|
||||
if before.ConsecutiveFailures == 0 || before.CheckedAt == nil {
|
||||
t.Fatalf("precondition: expected a populated failure record, got %+v", before)
|
||||
}
|
||||
|
||||
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil {
|
||||
t.Fatalf("reset: %v", err)
|
||||
}
|
||||
|
||||
after, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get after reset: %v", err)
|
||||
}
|
||||
if after.State != models.ExecutorReachabilityStateUnknown {
|
||||
t.Fatalf("State = %q, want unknown", after.State)
|
||||
}
|
||||
if after.ConsecutiveFailures != 0 || after.Reason != "" || after.Message != "" {
|
||||
t.Fatalf("record not cleared: %+v", after)
|
||||
}
|
||||
if after.Host != "10.0.0.9" {
|
||||
t.Fatalf("Host = %q, want the newly saved host", after.Host)
|
||||
}
|
||||
if after.CheckedAt != nil || after.LastSuccessAt != nil {
|
||||
t.Fatalf("timestamps not cleared: checked_at=%v last_success_at=%v", after.CheckedAt, after.LastSuccessAt)
|
||||
}
|
||||
if !after.UpdatedAt.After(before.UpdatedAt) {
|
||||
t.Fatalf("UpdatedAt did not advance: before=%v after=%v", before.UpdatedAt, after.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetExecutorReachabilityIgnoresStaleExecutorVersion(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Host: "10.0.0.1", CheckedAt: time.Now().UTC(), FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
staleVersion := executor.UpdatedAt
|
||||
executor.Config = map[string]string{"ssh_host": "10.0.0.2"}
|
||||
if err := repo.UpdateExecutor(ctx, executor); err != nil {
|
||||
t.Fatalf("update executor: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.2", staleVersion); err != nil {
|
||||
t.Fatalf("reset: %v", err)
|
||||
}
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.State != models.ExecutorReachabilityStateReachable || got.Host != "10.0.0.1" {
|
||||
t.Fatalf("record = %+v, want the newer executor version to remain untouched", got)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.17, 001.22
|
||||
func TestResetExecutorReachabilityIsANoOpForANonActiveExecutor(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
executor := createSSHExecutor(t, repo)
|
||||
at := time.Now().UTC()
|
||||
if err := repo.UpsertExecutorReachability(ctx, models.ExecutorReachabilityObservation{
|
||||
ExecutorID: executor.ID, SeenUpdatedAt: executor.UpdatedAt,
|
||||
InitialState: models.ExecutorReachabilityStateReachable,
|
||||
Reason: "", Host: "10.0.0.1", CheckedAt: at, FailureThreshold: 3,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
|
||||
executor.Status = models.ExecutorStatusDisabled
|
||||
if err := repo.UpdateExecutor(ctx, executor); err != nil {
|
||||
t.Fatalf("disable executor: %v", err)
|
||||
}
|
||||
|
||||
if err := repo.ResetExecutorReachability(ctx, executor.ID, "10.0.0.9", executor.UpdatedAt); err != nil {
|
||||
t.Fatalf("reset: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Host != "10.0.0.1" || got.State != models.ExecutorReachabilityStateReachable {
|
||||
t.Fatalf("record = %+v, want it untouched by a reset on a disabled executor", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSSHExecutorsForReachabilityOrdersByID(t *testing.T) {
|
||||
repo, _ := newReachabilityTestRepo(t)
|
||||
ctx := context.Background()
|
||||
|
||||
mustCreate := func(id string, execType models.ExecutorType, status models.ExecutorStatus) {
|
||||
t.Helper()
|
||||
if err := repo.CreateExecutor(ctx, &models.Executor{ID: id, Name: id, Type: execType, Status: status}); err != nil {
|
||||
t.Fatalf("create %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
mustCreate("b-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive)
|
||||
mustCreate("a-ssh", models.ExecutorTypeSSH, models.ExecutorStatusActive)
|
||||
mustCreate("c-ssh-disabled", models.ExecutorTypeSSH, models.ExecutorStatusDisabled)
|
||||
mustCreate("d-docker", models.ExecutorTypeLocalDocker, models.ExecutorStatusActive)
|
||||
|
||||
deleted := &models.Executor{ID: "e-ssh-deleted", Name: "e-ssh-deleted", Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive}
|
||||
if err := repo.CreateExecutor(ctx, deleted); err != nil {
|
||||
t.Fatalf("create deleted: %v", err)
|
||||
}
|
||||
if err := repo.DeleteExecutor(ctx, deleted.ID); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.ListSSHExecutorsForReachability(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(got) != 2 || got[0].ID != "a-ssh" || got[1].ID != "b-ssh" {
|
||||
ids := make([]string, len(got))
|
||||
for i, e := range got {
|
||||
ids[i] = e.ID
|
||||
}
|
||||
t.Fatalf("ids = %v, want [a-ssh b-ssh]", ids)
|
||||
}
|
||||
}
|
||||
@@ -280,6 +280,17 @@ type WorkflowStepCreator interface {
|
||||
CreateStepsFromTemplate(ctx context.Context, workflowID, templateID string) error
|
||||
}
|
||||
|
||||
// ExecutorSaveObserver is notified after an executor create or update
|
||||
// commits, with before the pre-update snapshot (nil on create) and after the
|
||||
// saved executor. CreateExecutor and UpdateExecutor are the only call sites
|
||||
// that can form this before/after comparison — before must be captured prior
|
||||
// to any in-place mutation of the loaded executor. Implementations decide for
|
||||
// themselves whether the save is worth acting on (e.g. only SSH executors
|
||||
// whose connection configuration changed).
|
||||
type ExecutorSaveObserver interface {
|
||||
OnExecutorSaved(ctx context.Context, before, after *models.Executor)
|
||||
}
|
||||
|
||||
// WorkspaceBootstrapper owns the atomic persistence of a standard Kanban
|
||||
// workspace and its initial workflow state.
|
||||
type WorkspaceBootstrapper interface {
|
||||
@@ -501,6 +512,7 @@ type Service struct {
|
||||
providerProber ProviderDefaultBranchProber
|
||||
gitArchiveCapture GitArchiveCapture
|
||||
workflowStepCreator WorkflowStepCreator
|
||||
executorSaveObserver ExecutorSaveObserver
|
||||
workspaceBootstrapper WorkspaceBootstrapper
|
||||
workflowStepGetter WorkflowStepGetter
|
||||
workflowMovePreflight WorkflowMovePreflight
|
||||
@@ -930,6 +942,13 @@ func (s *Service) SetWorkflowStepCreator(creator WorkflowStepCreator) {
|
||||
s.workflowStepCreator = creator
|
||||
}
|
||||
|
||||
// SetExecutorSaveObserver wires the observer notified after every executor
|
||||
// create/update commits. Optional — a Service with no observer wired saves
|
||||
// executors exactly as before.
|
||||
func (s *Service) SetExecutorSaveObserver(observer ExecutorSaveObserver) {
|
||||
s.executorSaveObserver = observer
|
||||
}
|
||||
|
||||
func (s *Service) SetWorkspaceBootstrapper(bootstrapper WorkspaceBootstrapper) {
|
||||
s.workspaceBootstrapper = bootstrapper
|
||||
}
|
||||
|
||||
@@ -1806,9 +1806,19 @@ func (s *Service) CreateExecutor(ctx context.Context, req *CreateExecutorRequest
|
||||
return nil, err
|
||||
}
|
||||
s.publishExecutorEvent(ctx, events.ExecutorCreated, executor)
|
||||
s.notifyExecutorSaved(ctx, nil, executor)
|
||||
return executor, nil
|
||||
}
|
||||
|
||||
// notifyExecutorSaved forwards a committed executor create/update to the
|
||||
// wired ExecutorSaveObserver, if any. before is nil on create.
|
||||
func (s *Service) notifyExecutorSaved(ctx context.Context, before, after *models.Executor) {
|
||||
if s.executorSaveObserver == nil {
|
||||
return
|
||||
}
|
||||
s.executorSaveObserver.OnExecutorSaved(ctx, before, after)
|
||||
}
|
||||
|
||||
func (s *Service) GetExecutor(ctx context.Context, id string) (*models.Executor, error) {
|
||||
return s.executors.GetExecutor(ctx, id)
|
||||
}
|
||||
@@ -1842,12 +1852,14 @@ func (s *Service) UpdateExecutor(ctx context.Context, id string, req *UpdateExec
|
||||
return nil, ErrActiveTaskSessions
|
||||
}
|
||||
}
|
||||
before := *executor
|
||||
applyExecutorUpdates(executor, req)
|
||||
executor.UpdatedAt = time.Now().UTC()
|
||||
if err := s.executors.UpdateExecutor(ctx, executor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.publishExecutorEvent(ctx, events.ExecutorUpdated, executor)
|
||||
s.notifyExecutorSaved(ctx, &before, executor)
|
||||
return executor, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
// fakeExecutorSaveObserver records every OnExecutorSaved call it receives, so
|
||||
// a test can assert the exact before/after pair the service handed it without
|
||||
// depending on the reachability package's own behavior.
|
||||
type fakeExecutorSaveObserver struct {
|
||||
calls []executorSaveCall
|
||||
}
|
||||
|
||||
type executorSaveCall struct {
|
||||
before *models.Executor
|
||||
after *models.Executor
|
||||
}
|
||||
|
||||
func (f *fakeExecutorSaveObserver) OnExecutorSaved(_ context.Context, before, after *models.Executor) {
|
||||
f.calls = append(f.calls, executorSaveCall{before: before, after: after})
|
||||
}
|
||||
|
||||
func sshExecutorCreateRequest(name string, config map[string]string) *CreateExecutorRequest {
|
||||
return &CreateExecutorRequest{
|
||||
Name: name, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive,
|
||||
Resumable: true, Config: config,
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// Creating an SSH executor always counts as a connection-configuration
|
||||
// change — there is no prior state to compare against — so the observer must
|
||||
// see before=nil paired with the newly created executor.
|
||||
func TestCreateExecutorNotifiesSaveObserverWithNilBefore(t *testing.T) {
|
||||
svc, _, _ := createTestService(t)
|
||||
observer := &fakeExecutorSaveObserver{}
|
||||
svc.SetExecutorSaveObserver(observer)
|
||||
ctx := context.Background()
|
||||
|
||||
executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{
|
||||
"ssh_host": "10.0.0.5", "ssh_user": "deploy",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("CreateExecutor: %v", err)
|
||||
}
|
||||
|
||||
if len(observer.calls) != 1 {
|
||||
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
|
||||
}
|
||||
call := observer.calls[0]
|
||||
if call.before != nil {
|
||||
t.Fatalf("before = %+v, want nil on create", call.before)
|
||||
}
|
||||
if call.after == nil || call.after.ID != executor.ID {
|
||||
t.Fatalf("after = %+v, want the created executor", call.after)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// The service notifies the observer on every executor create, regardless of
|
||||
// type — deciding whether a save is SSH-reachability-relevant is the
|
||||
// observer implementation's job (internal/executors/reachability.SaveObserver),
|
||||
// not the service's. This keeps the service layer decoupled from SSH-specific
|
||||
// eligibility rules, the same way publishExecutorEvent fires unconditionally
|
||||
// and lets subscribers filter.
|
||||
func TestCreateExecutorNotifiesSaveObserverForNonSSHType(t *testing.T) {
|
||||
svc, _, _ := createTestService(t)
|
||||
observer := &fakeExecutorSaveObserver{}
|
||||
svc.SetExecutorSaveObserver(observer)
|
||||
ctx := context.Background()
|
||||
|
||||
executor, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{
|
||||
Name: "Local runner", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive, Resumable: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateExecutor: %v", err)
|
||||
}
|
||||
|
||||
if len(observer.calls) != 1 {
|
||||
t.Fatalf("observer calls = %d, want 1 (gating is the observer's job, not the service's)", len(observer.calls))
|
||||
}
|
||||
if observer.calls[0].after == nil || observer.calls[0].after.ID != executor.ID {
|
||||
t.Fatalf("after = %+v, want the created executor", observer.calls[0].after)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1, AC-EXECUTORS-SSH-REACHABILITY-003.2
|
||||
//
|
||||
// Updating an SSH executor's host must notify the observer with the literal
|
||||
// old-host/new-host pair — captured strictly before applyExecutorUpdates
|
||||
// mutates the executor in place, per the work order's own risk callout. A
|
||||
// self-comparison (before and after pointing at the same, already-mutated
|
||||
// struct) would silently make this test pass with either host string, so it
|
||||
// asserts both values explicitly.
|
||||
func TestUpdateExecutorNotifiesSaveObserverOnHostChange(t *testing.T) {
|
||||
svc, _, _ := createTestService(t)
|
||||
executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH)
|
||||
ctx := context.Background()
|
||||
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
|
||||
Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed UpdateExecutor: %v", err)
|
||||
}
|
||||
|
||||
observer := &fakeExecutorSaveObserver{}
|
||||
svc.SetExecutorSaveObserver(observer)
|
||||
|
||||
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
|
||||
Config: map[string]string{"ssh_host": "10.0.0.2", "ssh_user": "deploy"},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateExecutor: %v", err)
|
||||
}
|
||||
|
||||
if len(observer.calls) != 1 {
|
||||
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
|
||||
}
|
||||
call := observer.calls[0]
|
||||
if call.before == nil || call.before.Config["ssh_host"] != "10.0.0.1" {
|
||||
t.Fatalf("before host = %+v, want 10.0.0.1", call.before)
|
||||
}
|
||||
if call.after == nil || call.after.Config["ssh_host"] != "10.0.0.2" {
|
||||
t.Fatalf("after host = %+v, want 10.0.0.2", call.after)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// A name-only update still notifies the observer (the service does not
|
||||
// pre-filter by which fields changed) — the observer's own
|
||||
// connectionConfigChanged comparison is what decides a name-only save isn't
|
||||
// reachability-relevant, and that decision is covered in the reachability
|
||||
// package's own save_observer_test.go, not here. This test only pins that the
|
||||
// service keeps calling with the correct before/after regardless of which
|
||||
// fields the caller supplied.
|
||||
func TestUpdateExecutorNotifiesSaveObserverForNameOnlyChange(t *testing.T) {
|
||||
svc, _, _ := createTestService(t)
|
||||
executor := createTestExecutor(t, svc, "SSH box", models.ExecutorTypeSSH)
|
||||
ctx := context.Background()
|
||||
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
|
||||
Config: map[string]string{"ssh_host": "10.0.0.1", "ssh_user": "deploy"},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed UpdateExecutor: %v", err)
|
||||
}
|
||||
|
||||
observer := &fakeExecutorSaveObserver{}
|
||||
svc.SetExecutorSaveObserver(observer)
|
||||
|
||||
newName := "Renamed SSH box"
|
||||
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{Name: &newName}); err != nil {
|
||||
t.Fatalf("UpdateExecutor: %v", err)
|
||||
}
|
||||
|
||||
if len(observer.calls) != 1 {
|
||||
t.Fatalf("observer calls = %d, want 1", len(observer.calls))
|
||||
}
|
||||
if observer.calls[0].before.Config["ssh_host"] != "10.0.0.1" || observer.calls[0].after.Config["ssh_host"] != "10.0.0.1" {
|
||||
t.Fatalf("host should be unchanged across a name-only update: %+v", observer.calls[0])
|
||||
}
|
||||
if observer.calls[0].after.Name != newName {
|
||||
t.Fatalf("after.Name = %q, want %q", observer.calls[0].after.Name, newName)
|
||||
}
|
||||
}
|
||||
|
||||
// @covers AC-EXECUTORS-SSH-REACHABILITY-003.1
|
||||
//
|
||||
// A nil observer (the default in every test not exercising reachability) must
|
||||
// never be dereferenced by CreateExecutor/UpdateExecutor.
|
||||
func TestExecutorSaveObserverNilSafe(t *testing.T) {
|
||||
svc, _, _ := createTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
executor, err := svc.CreateExecutor(ctx, sshExecutorCreateRequest("SSH box", map[string]string{
|
||||
"ssh_host": "10.0.0.5",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("CreateExecutor: %v", err)
|
||||
}
|
||||
|
||||
newHost := "10.0.0.9"
|
||||
if _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{
|
||||
Config: map[string]string{"ssh_host": newHost},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateExecutor: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -271,6 +271,7 @@ const (
|
||||
ActionSessionModelsUpdated = "session.models_updated"
|
||||
ActionSessionModelFallback = "session.model_fallback"
|
||||
ActionSessionModelSelectionWarning = "session.model_selection_warning"
|
||||
ActionSessionLaunchWarning = "session.launch.warning"
|
||||
ActionSessionMCPStatusUpdated = "session.mcp_status_updated"
|
||||
ActionSessionInfoUpdated = "session.info_updated"
|
||||
ActionSessionSetMode = "session.set_mode"
|
||||
@@ -298,6 +299,7 @@ const (
|
||||
ActionExecutorCreated = "executor.created"
|
||||
ActionExecutorUpdated = "executor.updated"
|
||||
ActionExecutorDeleted = "executor.deleted"
|
||||
ActionExecutorReachabilityChanged = "executor.reachability.changed"
|
||||
ActionEnvironmentCreated = "environment.created"
|
||||
ActionEnvironmentUpdated = "environment.updated"
|
||||
ActionEnvironmentDeleted = "environment.deleted"
|
||||
|
||||
@@ -13,6 +13,7 @@ import { fetchExecutor, listExecutors, updateExecutor } from "@/lib/api/domains/
|
||||
import { SSHConnectionCard } from "@/components/settings/ssh-connection-card";
|
||||
import type { SSHExecutorConfig } from "@/components/settings/ssh-connection-card";
|
||||
import { SSHSessionsCard } from "@/components/settings/ssh-sessions-card";
|
||||
import { SSHReachabilityCard } from "@/components/settings/ssh-reachability-card";
|
||||
import { listSSHSessions } from "@/lib/api/domains/ssh-api";
|
||||
import { getExecutorLabel } from "@/lib/executor-icons";
|
||||
import {
|
||||
@@ -135,6 +136,7 @@ function SSHExecutorView({
|
||||
coordinatedSaveId={`ssh-executor:${executor.id}`}
|
||||
runningSessionCount={sessionCount}
|
||||
/>
|
||||
<SSHReachabilityCard executorId={executor.id} />
|
||||
<SSHSessionsCard executorId={executor.id} />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { StateProvider, useAppStoreApi } from "@/components/state-provider";
|
||||
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
|
||||
|
||||
const getSSHExecutorReachability = vi.fn();
|
||||
const probeSSHExecutorReachability = vi.fn();
|
||||
|
||||
vi.mock("@/lib/api/domains/ssh-api", () => ({
|
||||
getSSHExecutorReachability: (...args: unknown[]) => getSSHExecutorReachability(...args),
|
||||
probeSSHExecutorReachability: (...args: unknown[]) => probeSSHExecutorReachability(...args),
|
||||
}));
|
||||
|
||||
import { SSHReachabilityCard, isReachabilityStale } from "./ssh-reachability-card";
|
||||
|
||||
const EXECUTOR_ID = "executor-1";
|
||||
const BASE_TIMESTAMP = "2026-09-17T00:00:00.000Z";
|
||||
const STATE_TESTID = "ssh-reachability-state";
|
||||
|
||||
function record(overrides: Partial<SSHReachabilityRecord> = {}): SSHReachabilityRecord {
|
||||
return {
|
||||
executor_id: EXECUTOR_ID,
|
||||
state: "reachable",
|
||||
reason: "",
|
||||
consecutive_failures: 0,
|
||||
host: "10.0.0.5",
|
||||
checked_at: BASE_TIMESTAMP,
|
||||
last_success_at: BASE_TIMESTAMP,
|
||||
updated_at: BASE_TIMESTAMP,
|
||||
probing_enabled: true,
|
||||
probe_interval_seconds: 60,
|
||||
persisted: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function renderCard(executorId = EXECUTOR_ID) {
|
||||
return render(
|
||||
<StateProvider>
|
||||
<SSHReachabilityCard executorId={executorId} />
|
||||
</StateProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
function PushReachability({ record: pushed }: { record: SSHReachabilityRecord }) {
|
||||
const storeApi = useAppStoreApi();
|
||||
return (
|
||||
<button type="button" onClick={() => storeApi.getState().setSSHReachability(pushed)}>
|
||||
push
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function renderCardWithPush(pushed: SSHReachabilityRecord) {
|
||||
return render(
|
||||
<StateProvider>
|
||||
<SSHReachabilityCard executorId={EXECUTOR_ID} />
|
||||
<PushReachability record={pushed} />
|
||||
</StateProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForState(text: string) {
|
||||
await waitFor(() => expect(screen.getByTestId(STATE_TESTID).textContent).toBe(text));
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
getSSHExecutorReachability.mockReset();
|
||||
probeSSHExecutorReachability.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("isReachabilityStale", () => {
|
||||
it("is false when there is no probing cadence or no completed probe", () => {
|
||||
expect(isReachabilityStale(null, 60, true, Date.now())).toBe(false);
|
||||
expect(isReachabilityStale(BASE_TIMESTAMP, 60, false, Date.now())).toBe(false);
|
||||
});
|
||||
|
||||
it("is false within three intervals and true just past it", () => {
|
||||
const checkedMs = Date.parse(BASE_TIMESTAMP);
|
||||
expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000)).toBe(false);
|
||||
expect(isReachabilityStale(BASE_TIMESTAMP, 60, true, checkedMs + 3 * 60_000 + 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SSHReachabilityCard rendering", () => {
|
||||
it("renders state, host, failure count and age for a reachable record", async () => {
|
||||
getSSHExecutorReachability.mockResolvedValue(record());
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
|
||||
expect(screen.getByTestId("ssh-reachability-host").textContent).toContain("10.0.0.5");
|
||||
expect(screen.queryByTestId("ssh-reachability-reason")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders reason and message for an unreachable record", async () => {
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
message: "dial tcp: i/o timeout",
|
||||
consecutive_failures: 3,
|
||||
}),
|
||||
);
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId("ssh-reachability-reason")).toBeTruthy());
|
||||
expect(screen.getByTestId("ssh-reachability-message").textContent).toBe(
|
||||
"dial tcp: i/o timeout",
|
||||
);
|
||||
expect(screen.getByTestId("ssh-reachability-failures").textContent).toContain("3");
|
||||
});
|
||||
|
||||
it("marks a record older than three intervals as stale", async () => {
|
||||
const old = new Date(Date.now() - 4 * 60_000).toISOString();
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({ probe_interval_seconds: 60, checked_at: old }),
|
||||
);
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId("ssh-reachability-stale")).toBeTruthy());
|
||||
});
|
||||
|
||||
it("does not mark a fresh record as stale", async () => {
|
||||
const fresh = new Date().toISOString();
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({
|
||||
probe_interval_seconds: 60,
|
||||
checked_at: fresh,
|
||||
last_success_at: fresh,
|
||||
updated_at: fresh,
|
||||
}),
|
||||
);
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
|
||||
expect(screen.queryByTestId("ssh-reachability-stale")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("SSHReachabilityCard live updates", () => {
|
||||
it("applies a pushed executor.reachability.changed record live, without a reload", async () => {
|
||||
getSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" }));
|
||||
renderCardWithPush(record({ state: "unreachable", reason: "auth", message: "boom" }));
|
||||
await waitForState("Reachable");
|
||||
fireEvent.click(screen.getByRole("button", { name: "push" }));
|
||||
await waitForState("Unreachable");
|
||||
expect(getSSHExecutorReachability).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("replaces a stored record with a reset (null checked_at, newer updated_at)", async () => {
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({ state: "unreachable", reason: "timeout", updated_at: BASE_TIMESTAMP }),
|
||||
);
|
||||
renderCardWithPush(
|
||||
record({
|
||||
state: "unknown",
|
||||
reason: "",
|
||||
checked_at: null,
|
||||
last_success_at: null,
|
||||
updated_at: "2026-09-17T00:01:00.000Z",
|
||||
}),
|
||||
);
|
||||
await waitForState("Unreachable");
|
||||
fireEvent.click(screen.getByRole("button", { name: "push" }));
|
||||
await waitForState("Unknown");
|
||||
});
|
||||
});
|
||||
|
||||
describe("SSHReachabilityCard refresh cadence", () => {
|
||||
it("says periodic probing is off and starts no refresh timer when probing_enabled is false", async () => {
|
||||
const setIntervalSpy = vi.spyOn(window, "setInterval");
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({ probing_enabled: false, probe_interval_seconds: 0 }),
|
||||
);
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId("ssh-reachability-probing-off")).toBeTruthy());
|
||||
// Exclude testing-library's own internal waitFor polling interval (50ms);
|
||||
// this asserts the card itself never armed a refresh timer.
|
||||
const ownIntervalCalls = setIntervalSpy.mock.calls.filter((call) => call[1] !== 50);
|
||||
expect(ownIntervalCalls).toHaveLength(0);
|
||||
expect(screen.queryByTestId(STATE_TESTID)?.textContent).not.toBe("Unreachable");
|
||||
});
|
||||
|
||||
it("starts a refresh timer at the configured probe interval when probing is enabled", async () => {
|
||||
const setIntervalSpy = vi.spyOn(window, "setInterval");
|
||||
getSSHExecutorReachability.mockResolvedValue(record({ probe_interval_seconds: 45 }));
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId(STATE_TESTID)).toBeTruthy());
|
||||
expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 45_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SSHReachabilityCard errors and probe-now", () => {
|
||||
it("reports reachability as not known, never unreachable, when the initial load fails", async () => {
|
||||
getSSHExecutorReachability.mockRejectedValue(new Error("network error"));
|
||||
renderCard();
|
||||
await waitFor(() => expect(screen.getByTestId("ssh-reachability-not-known")).toBeTruthy());
|
||||
expect(screen.queryByTestId(STATE_TESTID)).toBeNull();
|
||||
});
|
||||
|
||||
it("runs an immediate probe and applies the result", async () => {
|
||||
getSSHExecutorReachability.mockResolvedValue(
|
||||
record({ state: "unreachable", reason: "network" }),
|
||||
);
|
||||
probeSSHExecutorReachability.mockResolvedValue(record({ state: "reachable" }));
|
||||
renderCard();
|
||||
await waitForState("Unreachable");
|
||||
fireEvent.click(screen.getByTestId("ssh-reachability-probe-now"));
|
||||
await waitForState("Reachable");
|
||||
expect(probeSSHExecutorReachability).toHaveBeenCalledWith(EXECUTOR_ID);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
"use client";
|
||||
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Badge } from "@kandev/ui/badge";
|
||||
import { Button } from "@kandev/ui/button";
|
||||
import { CardContent } from "@kandev/ui/card";
|
||||
import { IconLoader2 } from "@tabler/icons-react";
|
||||
import type { SSHReachabilityRecord, SSHReachabilityReason } from "@/lib/types/http-ssh";
|
||||
import { formatRelative } from "@/lib/i18n/formats";
|
||||
import { useSSHReachability } from "@/hooks/domains/settings/use-ssh-reachability";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/settings-card-header";
|
||||
import { settingsActionClassName } from "@/components/settings/settings-control";
|
||||
|
||||
export interface SSHReachabilityCardProps {
|
||||
executorId: string;
|
||||
}
|
||||
|
||||
const STALE_INTERVAL_MULTIPLIER = 3;
|
||||
|
||||
/**
|
||||
* A record older than three probe intervals is stale rather than current.
|
||||
* Disabled probing has no cadence to measure against, and a never-probed
|
||||
* record has nothing to be stale relative to, so both read as not-stale.
|
||||
*/
|
||||
export function isReachabilityStale(
|
||||
checkedAt: string | null,
|
||||
probeIntervalSeconds: number,
|
||||
probingEnabled: boolean,
|
||||
now: number = Date.now(),
|
||||
): boolean {
|
||||
if (!probingEnabled || !checkedAt || probeIntervalSeconds <= 0) return false;
|
||||
const checkedMs = Date.parse(checkedAt);
|
||||
if (Number.isNaN(checkedMs)) return false;
|
||||
return now - checkedMs > probeIntervalSeconds * 1000 * STALE_INTERVAL_MULTIPLIER;
|
||||
}
|
||||
|
||||
export function reachabilityReasonKey(reason: SSHReachabilityReason): string | null {
|
||||
switch (reason) {
|
||||
case "config":
|
||||
return "executors:sshReachabilityReasonConfig";
|
||||
case "timeout":
|
||||
return "executors:sshReachabilityReasonTimeout";
|
||||
case "host_key":
|
||||
return "executors:sshReachabilityReasonHostKey";
|
||||
case "auth":
|
||||
return "executors:sshReachabilityReasonAuth";
|
||||
case "network":
|
||||
return "executors:sshReachabilityReasonNetwork";
|
||||
case "unknown":
|
||||
return "executors:sshReachabilityReasonUnknown";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function reachabilityStateKey(state: SSHReachabilityRecord["state"]): string {
|
||||
switch (state) {
|
||||
case "reachable":
|
||||
return "executors:sshReachabilityStateReachable";
|
||||
case "unreachable":
|
||||
return "executors:sshReachabilityStateUnreachable";
|
||||
default:
|
||||
return "executors:sshReachabilityStateUnknown";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the most recent SSH reachability record for one executor: state,
|
||||
* probed host, failure reason/message, consecutive-failure count, and the
|
||||
* age of the last completed and last successful probe. Never fetches
|
||||
* directly into local-only state — every fetch result is written through the
|
||||
* store so a live executor.reachability.changed push and this card's own
|
||||
* refresh share one reconciliation path.
|
||||
*/
|
||||
export function SSHReachabilityCard({ executorId }: SSHReachabilityCardProps) {
|
||||
const { t } = useTranslation();
|
||||
const { record, loadError, probing, probeNow, now } = useSSHReachability(executorId);
|
||||
|
||||
return (
|
||||
<SettingsCard data-testid="ssh-reachability-card">
|
||||
<SettingsCardHeader
|
||||
title={t("executors:sshReachabilityTitle")}
|
||||
description={t("executors:sshReachabilityDescription")}
|
||||
actions={
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void probeNow()}
|
||||
disabled={probing}
|
||||
data-testid="ssh-reachability-probe-now"
|
||||
className={settingsActionClassName("cursor-pointer")}
|
||||
>
|
||||
{probing ? <IconLoader2 className="mr-1.5 h-4 w-4 animate-spin" /> : null}
|
||||
{t("executors:sshReachabilityProbeNow")}
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
<CardContent>
|
||||
<ReachabilityBody record={record} loadError={loadError} now={now} />
|
||||
</CardContent>
|
||||
</SettingsCard>
|
||||
);
|
||||
}
|
||||
|
||||
function ReachabilityBody({
|
||||
record,
|
||||
loadError,
|
||||
now,
|
||||
}: {
|
||||
record: SSHReachabilityRecord | undefined;
|
||||
loadError: boolean;
|
||||
now: number;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
if (!record) {
|
||||
if (loadError) {
|
||||
return (
|
||||
<p data-testid="ssh-reachability-not-known" className="text-sm text-muted-foreground">
|
||||
{t("executors:sshReachabilityLoadFailed")}
|
||||
</p>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const stale = isReachabilityStale(
|
||||
record.checked_at,
|
||||
record.probe_interval_seconds,
|
||||
record.probing_enabled,
|
||||
now,
|
||||
);
|
||||
const reasonKey = record.reason ? reachabilityReasonKey(record.reason) : null;
|
||||
return (
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
<StateBadge state={record.state} />
|
||||
{stale ? (
|
||||
<Badge
|
||||
data-testid="ssh-reachability-stale"
|
||||
variant="outline"
|
||||
className="border-amber-500/30 bg-amber-500/10 text-amber-700"
|
||||
>
|
||||
{t("executors:sshReachabilityStale")}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
{record.host ? (
|
||||
<p data-testid="ssh-reachability-host">
|
||||
{t("executors:sshReachabilityHost", { host: record.host })}
|
||||
</p>
|
||||
) : null}
|
||||
{record.state === "unreachable" && reasonKey ? (
|
||||
<p data-testid="ssh-reachability-reason">{t(reasonKey)}</p>
|
||||
) : null}
|
||||
{record.state === "unreachable" && record.message ? (
|
||||
<p data-testid="ssh-reachability-message" className="text-muted-foreground">
|
||||
{record.message}
|
||||
</p>
|
||||
) : null}
|
||||
<p data-testid="ssh-reachability-failures">
|
||||
{t("executors:sshReachabilityConsecutiveFailures", { count: record.consecutive_failures })}
|
||||
</p>
|
||||
<p data-testid="ssh-reachability-age">
|
||||
{record.checked_at
|
||||
? t("executors:sshReachabilityAgeLastProbe", { age: formatRelative(record.checked_at) })
|
||||
: t("executors:sshReachabilityNeverProbed")}
|
||||
</p>
|
||||
<p data-testid="ssh-reachability-last-success">
|
||||
{record.last_success_at
|
||||
? t("executors:sshReachabilityAgeLastSuccess", {
|
||||
age: formatRelative(record.last_success_at),
|
||||
})
|
||||
: t("executors:sshReachabilityNoSuccess")}
|
||||
</p>
|
||||
{!record.probing_enabled ? (
|
||||
<p data-testid="ssh-reachability-probing-off" className="text-muted-foreground">
|
||||
{t("executors:sshReachabilityProbingOff")}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const STATE_BADGE_CLASSNAMES: Record<SSHReachabilityRecord["state"], string> = {
|
||||
reachable: "border-green-500/30 bg-green-500/10 text-green-700",
|
||||
unreachable: "border-red-500/30 bg-red-500/10 text-red-700",
|
||||
unknown: "border-muted-foreground/30 bg-muted text-muted-foreground",
|
||||
};
|
||||
|
||||
function StateBadge({ state }: { state: SSHReachabilityRecord["state"] }) {
|
||||
const { t } = useTranslation();
|
||||
const variantClass = STATE_BADGE_CLASSNAMES[state];
|
||||
return (
|
||||
<Badge data-testid="ssh-reachability-state" variant="outline" className={variantClass}>
|
||||
{t(reachabilityStateKey(state))}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
@@ -7,3 +7,5 @@ export { SSHConnectionCard } from "./ssh-connection-card";
|
||||
export type { SSHConnectionCardProps, SSHExecutorConfig } from "./ssh-connection-card";
|
||||
export { SSHSessionsCard } from "./ssh-sessions-card";
|
||||
export type { SSHSessionsCardProps } from "./ssh-sessions-card";
|
||||
export { SSHReachabilityCard } from "./ssh-reachability-card";
|
||||
export type { SSHReachabilityCardProps } from "./ssh-reachability-card";
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { cleanup, render, screen } from "@testing-library/react";
|
||||
import { StateProvider } from "@/components/state-provider";
|
||||
import type { LaunchWarningEntry } from "@/lib/state/slices/session-runtime/types";
|
||||
import { LaunchWarning } from "./launch-warning";
|
||||
|
||||
const SESSION_ID = "session-1";
|
||||
|
||||
function makeEntry(overrides: Partial<LaunchWarningEntry> = {}): LaunchWarningEntry {
|
||||
return {
|
||||
executorId: "executor-1",
|
||||
host: "10.0.0.5",
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
function renderWithEntry(entry?: LaunchWarningEntry) {
|
||||
return render(
|
||||
<StateProvider
|
||||
initialState={{
|
||||
launchWarning: { bySessionId: entry ? { [SESSION_ID]: entry } : {} },
|
||||
}}
|
||||
>
|
||||
<LaunchWarning sessionId={SESSION_ID} />
|
||||
</StateProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("LaunchWarning", () => {
|
||||
it("renders nothing when no launch warning was published for this session", () => {
|
||||
const { container } = renderWithEntry(undefined);
|
||||
expect(container.textContent).toBe("");
|
||||
});
|
||||
|
||||
it("names the probed host from the event payload", () => {
|
||||
renderWithEntry(makeEntry());
|
||||
expect(screen.getByTestId("launch-warning").textContent).toContain("10.0.0.5");
|
||||
});
|
||||
|
||||
it("shows the age of the last successful probe", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-11T00:05:00.000Z"));
|
||||
try {
|
||||
renderWithEntry(makeEntry({ lastSuccessAt: "2026-06-11T00:00:00.000Z" }));
|
||||
expect(screen.getByTestId("launch-warning-last-success").textContent).toMatch(/5m/);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("states plainly that no successful probe was ever recorded, not a zero age", () => {
|
||||
renderWithEntry(makeEntry({ lastSuccessAt: undefined }));
|
||||
const text = screen.getByTestId("launch-warning-last-success").textContent ?? "";
|
||||
expect(text).not.toMatch(/0[ms]/);
|
||||
expect(text.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it.each(["config", "timeout", "host_key", "auth", "network", "unknown"])(
|
||||
"renders a reason sentence for %s without throwing",
|
||||
(reason) => {
|
||||
renderWithEntry(makeEntry({ reason }));
|
||||
expect(screen.getByTestId("launch-warning-reason").textContent?.length).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("never renders any confirmation control — the launch is never blocked", () => {
|
||||
renderWithEntry(makeEntry());
|
||||
expect(screen.queryByRole("button")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
"use client";
|
||||
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { IconAlertTriangle } from "@tabler/icons-react";
|
||||
import { useAppStore } from "@/components/state-provider";
|
||||
import { formatRelative } from "@/lib/i18n/formats";
|
||||
|
||||
type LaunchWarningProps = {
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
function reasonKey(reason: string): string {
|
||||
switch (reason) {
|
||||
case "config":
|
||||
return "task:launchWarningReasonConfig";
|
||||
case "timeout":
|
||||
return "task:launchWarningReasonTimeout";
|
||||
case "host_key":
|
||||
return "task:launchWarningReasonHostKey";
|
||||
case "auth":
|
||||
return "task:launchWarningReasonAuth";
|
||||
case "network":
|
||||
return "task:launchWarningReasonNetwork";
|
||||
default:
|
||||
return "task:launchWarningReasonUnknown";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the session.launch.warning event verbatim at the point the user
|
||||
* initiated the launch — never re-derives reachability from the settings
|
||||
* store slice, and never blocks or confirms the launch it describes.
|
||||
*/
|
||||
export function LaunchWarning({ sessionId }: LaunchWarningProps) {
|
||||
const { t } = useTranslation();
|
||||
const entry = useAppStore((state) => state.launchWarning.bySessionId[sessionId]);
|
||||
|
||||
if (!entry) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex min-w-0 gap-3 border-b border-border/50 py-3"
|
||||
data-testid="launch-warning"
|
||||
role="status"
|
||||
>
|
||||
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-md bg-amber-500/10 text-amber-600 dark:text-amber-400">
|
||||
<IconAlertTriangle className="h-4 w-4" aria-hidden="true" />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1 space-y-1 text-sm">
|
||||
<p className="font-medium">{t("task:launchWarningTitle")}</p>
|
||||
{entry.host ? (
|
||||
<p className="break-words text-muted-foreground">
|
||||
{t("task:launchWarningHost", { host: entry.host })}
|
||||
</p>
|
||||
) : null}
|
||||
<p data-testid="launch-warning-last-success" className="text-muted-foreground">
|
||||
{entry.lastSuccessAt
|
||||
? t("task:launchWarningLastSuccess", { age: formatRelative(entry.lastSuccessAt) })
|
||||
: t("task:launchWarningNeverSucceeded")}
|
||||
</p>
|
||||
<p data-testid="launch-warning-reason" className="text-muted-foreground">
|
||||
{t(reasonKey(entry.reason))}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -44,6 +44,7 @@ const appStoreState = {
|
||||
},
|
||||
agentProfiles: { items: [] },
|
||||
messages: { bySession: { [SESSION_ID]: [priorTranscriptMessage] } },
|
||||
launchWarning: { bySessionId: {} },
|
||||
};
|
||||
|
||||
const panelState = {
|
||||
|
||||
@@ -54,6 +54,7 @@ import { routePanelMouseDown } from "./chat/route-panel-mouse-down";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import { loadMessageWindowAround } from "@/hooks/domains/session/load-message-window";
|
||||
import { LaunchWarning } from "./launch-warning";
|
||||
import { useTaskLaunchErrorContext } from "./task-launch-error-context";
|
||||
import { useTaskStatusSummary } from "@/hooks/domains/task/use-task-status-summary";
|
||||
import { TaskMarkdownFileLinkProvider } from "@/components/shared/task-markdown-file-link-provider";
|
||||
@@ -1253,54 +1254,67 @@ export const TaskChatPanel = memo(function TaskChatPanel({
|
||||
{!hideLaunchQueueStatus && <LaunchQueueStatus queue={launchStatusSummary?.launch_queue} />}
|
||||
{!hideWipQueueStatus && <WipQueueStatus taskId={summaryTaskId} />}
|
||||
<ParkedSessionNote visible={hasWorkflowParkingMarker(session?.metadata)} />
|
||||
<PanelBody padding={false} scroll={false} className="relative overflow-hidden">
|
||||
<TaskMarkdownFileLinkProvider
|
||||
taskId={taskId}
|
||||
sessionId={resolvedSessionId}
|
||||
worktreePath={getSessionWorkspacePath(session)}
|
||||
onOpenFile={onOpenFile}
|
||||
>
|
||||
<MessageList
|
||||
ref={messageListRef}
|
||||
items={groupedItems}
|
||||
messages={allMessages}
|
||||
footerActionMessages={footerActionMessages}
|
||||
permissionsByToolCallId={permissionsByToolCallId}
|
||||
childrenByParentToolCallId={childrenByParentToolCallId}
|
||||
taskId={taskId ?? undefined}
|
||||
sessionId={resolvedSessionId}
|
||||
messagesLoading={messagesLoading}
|
||||
historyRefreshPending={historyRefreshPending}
|
||||
historyStatus={historyStatus}
|
||||
historyError={historyError}
|
||||
onRetryHistory={retryHistory}
|
||||
isWorking={isWorking}
|
||||
sessionState={session?.state}
|
||||
worktreePath={getSessionWorkspacePath(session)}
|
||||
onOpenFile={onOpenFile}
|
||||
dividerBeforeItemKey={dividerBeforeItemKey}
|
||||
lastPromptMessageId={lastPromptMessageId}
|
||||
onLastPromptEdgeChange={setLastPromptEdge}
|
||||
firstMessageId={firstMessageId}
|
||||
onFirstMessageHiddenChange={setIsFirstMessageHidden}
|
||||
anchoredBarHeight={showAnchoredBar && lastPromptMessage ? anchoredBarHeight : 0}
|
||||
isVisible={transcriptIsVisible}
|
||||
launchErrorOwned={launchErrorOwned}
|
||||
launchErrorStamp={launchErrorOwned ? taskLaunchError?.stamp : undefined}
|
||||
launchErrorOccurredAt={launchErrorOwned ? taskLaunchError?.occurred_at : undefined}
|
||||
stickyPromptBar={
|
||||
showAnchoredBar && lastPromptMessage ? (
|
||||
<AnchoredLastPromptBar
|
||||
promptText={lastPromptMessage.content}
|
||||
isVisible={anchoredBarVisible}
|
||||
onScrollUp={scrollToLastPrompt}
|
||||
showScrollToLastPrompt={showScrollToLastPrompt}
|
||||
onHeightChange={setAnchoredBarHeight}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
</TaskMarkdownFileLinkProvider>
|
||||
<PanelBody
|
||||
padding={false}
|
||||
scroll={false}
|
||||
className="relative flex min-h-0 flex-col overflow-hidden"
|
||||
>
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
{resolvedSessionId ? (
|
||||
<div className="shrink-0 px-2">
|
||||
<LaunchWarning sessionId={resolvedSessionId} />
|
||||
</div>
|
||||
) : null}
|
||||
<div className="min-h-0 flex-1">
|
||||
<TaskMarkdownFileLinkProvider
|
||||
taskId={taskId}
|
||||
sessionId={resolvedSessionId}
|
||||
worktreePath={getSessionWorkspacePath(session)}
|
||||
onOpenFile={onOpenFile}
|
||||
>
|
||||
<MessageList
|
||||
ref={messageListRef}
|
||||
items={groupedItems}
|
||||
messages={allMessages}
|
||||
footerActionMessages={footerActionMessages}
|
||||
permissionsByToolCallId={permissionsByToolCallId}
|
||||
childrenByParentToolCallId={childrenByParentToolCallId}
|
||||
taskId={taskId ?? undefined}
|
||||
sessionId={resolvedSessionId}
|
||||
messagesLoading={messagesLoading}
|
||||
historyRefreshPending={historyRefreshPending}
|
||||
historyStatus={historyStatus}
|
||||
historyError={historyError}
|
||||
onRetryHistory={retryHistory}
|
||||
isWorking={isWorking}
|
||||
sessionState={session?.state}
|
||||
worktreePath={getSessionWorkspacePath(session)}
|
||||
onOpenFile={onOpenFile}
|
||||
dividerBeforeItemKey={dividerBeforeItemKey}
|
||||
lastPromptMessageId={lastPromptMessageId}
|
||||
onLastPromptEdgeChange={setLastPromptEdge}
|
||||
firstMessageId={firstMessageId}
|
||||
onFirstMessageHiddenChange={setIsFirstMessageHidden}
|
||||
anchoredBarHeight={showAnchoredBar && lastPromptMessage ? anchoredBarHeight : 0}
|
||||
isVisible={transcriptIsVisible}
|
||||
launchErrorOwned={launchErrorOwned}
|
||||
launchErrorStamp={launchErrorOwned ? taskLaunchError?.stamp : undefined}
|
||||
launchErrorOccurredAt={launchErrorOwned ? taskLaunchError?.occurred_at : undefined}
|
||||
stickyPromptBar={
|
||||
showAnchoredBar && lastPromptMessage ? (
|
||||
<AnchoredLastPromptBar
|
||||
promptText={lastPromptMessage.content}
|
||||
isVisible={anchoredBarVisible}
|
||||
onScrollUp={scrollToLastPrompt}
|
||||
showScrollToLastPrompt={showScrollToLastPrompt}
|
||||
onHeightChange={setAnchoredBarHeight}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
</TaskMarkdownFileLinkProvider>
|
||||
</div>
|
||||
</div>
|
||||
{isJumpLoading && (
|
||||
<div
|
||||
data-testid="transcript-jump-loading"
|
||||
|
||||
@@ -195,4 +195,36 @@ export class SSHSettingsPage {
|
||||
sessionRow(sessionId: string): Locator {
|
||||
return this.page.getByTestId(`ssh-session-row-${sessionId}`);
|
||||
}
|
||||
|
||||
// --- reachability card ---
|
||||
|
||||
get reachabilityCard(): Locator {
|
||||
return this.page.getByTestId("ssh-reachability-card");
|
||||
}
|
||||
|
||||
get reachabilityState(): Locator {
|
||||
return this.page.getByTestId("ssh-reachability-state");
|
||||
}
|
||||
|
||||
get reachabilityHost(): Locator {
|
||||
return this.page.getByTestId("ssh-reachability-host");
|
||||
}
|
||||
|
||||
get reachabilityReason(): Locator {
|
||||
return this.page.getByTestId("ssh-reachability-reason");
|
||||
}
|
||||
|
||||
get reachabilityProbeNowButton(): Locator {
|
||||
return this.page.getByTestId("ssh-reachability-probe-now");
|
||||
}
|
||||
|
||||
/**
|
||||
* Click "Probe now" and wait for the synchronous probe route to return.
|
||||
* The button re-enabling is the completion signal — the click handler
|
||||
* awaits the request before clearing its own disabled state.
|
||||
*/
|
||||
async probeNow(): Promise<void> {
|
||||
await this.reachabilityProbeNowButton.click();
|
||||
await expect(this.reachabilityProbeNowButton).toBeEnabled({ timeout: 30_000 });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { test, expect } from "../../fixtures/ssh-test-base";
|
||||
import { dropTrafficToPort22, restoreTraffic } from "../../helpers/ssh";
|
||||
import { SSHSettingsPage } from "../../pages/SSHSettingsPage";
|
||||
|
||||
/**
|
||||
* SSH reachability: settings-panel state transitions and non-gating launch
|
||||
* behavior against a real sshd target.
|
||||
*
|
||||
* Drives the failure threshold through repeated "Probe now" clicks rather
|
||||
* than the scheduled poller's wall-clock interval — `httpProbeReachability`
|
||||
* runs the exact same hysteresis path (`store.Observe`) an automatic pass
|
||||
* would, so this proves the same threshold logic without making the spec's
|
||||
* runtime depend on a configured interval.
|
||||
*
|
||||
* Covers AC-EXECUTORS-SSH-REACHABILITY-001.9, 001.11, 002.1, 003.1, 003.3.
|
||||
*/
|
||||
test.describe("ssh executor — reachability", () => {
|
||||
test("reflects reachable, then unreachable after the failure threshold, then clears on a single success", async ({
|
||||
testPage,
|
||||
seedData,
|
||||
}) => {
|
||||
test.setTimeout(150_000);
|
||||
const page = new SSHSettingsPage(testPage);
|
||||
await page.gotoExisting(seedData.sshExecutorId);
|
||||
|
||||
// A fresh probe against the live target starts reachable.
|
||||
await page.probeNow();
|
||||
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
|
||||
await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host);
|
||||
|
||||
dropTrafficToPort22(seedData.sshTarget);
|
||||
try {
|
||||
// First failed probe: below the two-probe failure threshold, so the
|
||||
// panel must not have flipped yet.
|
||||
await page.probeNow();
|
||||
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
|
||||
|
||||
// Second consecutive failure crosses the threshold.
|
||||
await page.probeNow();
|
||||
await expect(page.reachabilityState).toHaveText(/^unreachable$/i);
|
||||
await expect(page.reachabilityHost).toContainText(seedData.sshTarget.host);
|
||||
await expect(page.reachabilityReason).toBeVisible();
|
||||
} finally {
|
||||
restoreTraffic(seedData.sshTarget);
|
||||
}
|
||||
|
||||
// A single successful probe clears an unreachable state immediately;
|
||||
// no second confirming probe is required.
|
||||
await page.probeNow();
|
||||
await expect(page.reachabilityState).toHaveText(/^reachable$/i);
|
||||
});
|
||||
|
||||
test("a launch started while the host is unreachable is attempted and fails naming the host, not refused", async ({
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
test.setTimeout(240_000);
|
||||
dropTrafficToPort22(seedData.sshTarget);
|
||||
try {
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Reachability: launch while unreachable",
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: "/e2e:simple-message",
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
repository_ids: [seedData.repositoryId],
|
||||
executor_profile_id: seedData.sshExecutorProfileId,
|
||||
},
|
||||
);
|
||||
|
||||
// The task and its session are created rather than refused — the
|
||||
// launch is attempted and only fails once the real connection attempt
|
||||
// gives up. `dropTrafficToPort22` drops inbound packets on the
|
||||
// container's own port 22, but Docker's own port-publishing plumbing
|
||||
// (docker-proxy on some hosts) can still complete the client's TCP
|
||||
// handshake before its own upstream dial to the container stalls, so
|
||||
// the failure surfaces only once that stalls out — well past the
|
||||
// client-side `sshDialTimeout` (30s) alone. Give this real, variable
|
||||
// network path a wide budget rather than one tuned to a fast local
|
||||
// failure (e.g. a missing remote binary).
|
||||
let errMsg = "";
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const { sessions } = await apiClient.listTaskSessions(task.id);
|
||||
const session = sessions[0];
|
||||
errMsg = session?.error_message ?? "";
|
||||
return session?.state ?? "";
|
||||
},
|
||||
{ timeout: 200_000, message: "session reaches FAILED against the unreachable host" },
|
||||
)
|
||||
.toBe("FAILED");
|
||||
expect(errMsg).toContain(seedData.sshTarget.host);
|
||||
} finally {
|
||||
restoreTraffic(seedData.sshTarget);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useAppStore, useAppStoreApi } from "@/components/state-provider";
|
||||
import {
|
||||
getSSHExecutorReachability,
|
||||
probeSSHExecutorReachability,
|
||||
} from "@/lib/api/domains/ssh-api";
|
||||
|
||||
/**
|
||||
* Owns the settings-domain fetch, refresh, and immediate-probe lifecycle for
|
||||
* one SSH executor. The store remains the source of truth for the record, so
|
||||
* WebSocket updates and HTTP responses use the same reconciliation path.
|
||||
*/
|
||||
export function useSSHReachability(executorId: string) {
|
||||
const record = useAppStore((state) => state.sshReachability.byExecutorId[executorId]);
|
||||
const storeApi = useAppStoreApi();
|
||||
const [loadError, setLoadError] = useState(false);
|
||||
const [probing, setProbing] = useState(false);
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
const seqRef = useRef(0);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
const seq = ++seqRef.current;
|
||||
try {
|
||||
const response = await getSSHExecutorReachability(executorId);
|
||||
if (seq !== seqRef.current) return;
|
||||
setLoadError(false);
|
||||
storeApi.getState().setSSHReachability(response);
|
||||
} catch {
|
||||
if (seq !== seqRef.current) return;
|
||||
setLoadError(true);
|
||||
}
|
||||
}, [executorId, storeApi]);
|
||||
|
||||
useEffect(() => {
|
||||
seqRef.current = 0;
|
||||
setLoadError(false);
|
||||
setNow(Date.now());
|
||||
void load();
|
||||
return () => {
|
||||
seqRef.current = -1;
|
||||
};
|
||||
// The executor identity is the lifecycle boundary. `load` is stable for
|
||||
// that identity and must not restart the request on every store update.
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [executorId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!record || !record.probing_enabled || record.probe_interval_seconds <= 0) return;
|
||||
const interval = window.setInterval(() => void load(), record.probe_interval_seconds * 1000);
|
||||
return () => window.clearInterval(interval);
|
||||
}, [record, load]);
|
||||
|
||||
// A failed refresh can leave the record object unchanged. Keep the stale
|
||||
// badge live in that case instead of waiting for a successful fetch to
|
||||
// trigger another render.
|
||||
useEffect(() => {
|
||||
if (!record?.probing_enabled || !record.checked_at || record.probe_interval_seconds <= 0) {
|
||||
return;
|
||||
}
|
||||
const checkedAt = Date.parse(record.checked_at);
|
||||
if (Number.isNaN(checkedAt)) return;
|
||||
const staleAt = checkedAt + record.probe_interval_seconds * 1000 * 3;
|
||||
const delay = Math.max(0, staleAt - Date.now() + 1);
|
||||
const timer = window.setTimeout(() => setNow(Date.now()), delay);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [record]);
|
||||
|
||||
const probeNow = useCallback(async () => {
|
||||
setProbing(true);
|
||||
try {
|
||||
const response = await probeSSHExecutorReachability(executorId);
|
||||
setLoadError(false);
|
||||
storeApi.getState().setSSHReachability(response);
|
||||
} catch {
|
||||
setLoadError(true);
|
||||
} finally {
|
||||
setProbing(false);
|
||||
}
|
||||
}, [executorId, storeApi]);
|
||||
|
||||
return { record, loadError, probing, probeNow, now };
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
|
||||
vi.mock("@/lib/config", () => ({
|
||||
getBackendConfig: () => ({ apiBaseUrl: "http://api.test" }),
|
||||
}));
|
||||
|
||||
import {
|
||||
getSSHReachability,
|
||||
getSSHExecutorReachability,
|
||||
probeSSHExecutorReachability,
|
||||
} from "./ssh-api";
|
||||
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
|
||||
|
||||
const BASE = "http://api.test/api/v1/ssh";
|
||||
|
||||
type FetchInput = Parameters<typeof fetch>[0];
|
||||
type FetchInit = Parameters<typeof fetch>[1];
|
||||
|
||||
const fetchSpy = vi.fn<(...args: [FetchInput, FetchInit?]) => Promise<Response>>();
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy.mockReset();
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function jsonResponse(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function lastCall(): { url: string; init: FetchInit | undefined } {
|
||||
const call = fetchSpy.mock.calls.at(-1);
|
||||
if (!call) throw new Error("expected fetch to have been called");
|
||||
return { url: String(call[0]), init: call[1] };
|
||||
}
|
||||
|
||||
function record(overrides: Partial<SSHReachabilityRecord> = {}): SSHReachabilityRecord {
|
||||
return {
|
||||
executor_id: "exec-1",
|
||||
state: "reachable",
|
||||
reason: "",
|
||||
consecutive_failures: 0,
|
||||
host: "build.example",
|
||||
checked_at: "2026-09-17T00:00:00Z",
|
||||
last_success_at: "2026-09-17T00:00:00Z",
|
||||
updated_at: "2026-09-17T00:00:00Z",
|
||||
probing_enabled: true,
|
||||
probe_interval_seconds: 60,
|
||||
persisted: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("getSSHReachability", () => {
|
||||
it("GETs the list route", async () => {
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse([record()]));
|
||||
const result = await getSSHReachability();
|
||||
expect(result).toEqual([record()]);
|
||||
const { url, init } = lastCall();
|
||||
expect(url).toBe(`${BASE}/reachability`);
|
||||
expect(init?.method ?? "GET").toBe("GET");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getSSHExecutorReachability", () => {
|
||||
it("GETs the single-executor route", async () => {
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse(record()));
|
||||
const result = await getSSHExecutorReachability("exec-1");
|
||||
expect(result).toEqual(record());
|
||||
const { url } = lastCall();
|
||||
expect(url).toBe(`${BASE}/executors/exec-1/reachability`);
|
||||
});
|
||||
|
||||
it("encodes the executor id", async () => {
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse(record({ executor_id: "exec/1" })));
|
||||
await getSSHExecutorReachability("exec/1");
|
||||
const { url } = lastCall();
|
||||
expect(url).toBe(`${BASE}/executors/exec%2F1/reachability`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("probeSSHExecutorReachability", () => {
|
||||
it("POSTs the probe-now route", async () => {
|
||||
fetchSpy.mockResolvedValueOnce(jsonResponse(record()));
|
||||
const result = await probeSSHExecutorReachability("exec-1");
|
||||
expect(result).toEqual(record());
|
||||
const { url, init } = lastCall();
|
||||
expect(url).toBe(`${BASE}/executors/exec-1/reachability/probe`);
|
||||
expect(init?.method).toBe("POST");
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
SSHSession,
|
||||
SSHAgentReadinessResponse,
|
||||
SSHProbeShellsResponse,
|
||||
SSHReachabilityRecord,
|
||||
} from "@/lib/types/http-ssh";
|
||||
|
||||
export async function testSSHConnection(
|
||||
@@ -72,3 +73,36 @@ export async function probeSSHShells(
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function getSSHReachability(
|
||||
options?: ApiRequestOptions,
|
||||
): Promise<SSHReachabilityRecord[]> {
|
||||
return fetchJson<SSHReachabilityRecord[]>("/api/v1/ssh/reachability", options);
|
||||
}
|
||||
|
||||
export async function getSSHExecutorReachability(
|
||||
executorId: string,
|
||||
options?: ApiRequestOptions,
|
||||
): Promise<SSHReachabilityRecord> {
|
||||
return fetchJson<SSHReachabilityRecord>(
|
||||
`/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability`,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
export async function probeSSHExecutorReachability(
|
||||
executorId: string,
|
||||
options?: ApiRequestOptions,
|
||||
): Promise<SSHReachabilityRecord> {
|
||||
return fetchJson<SSHReachabilityRecord>(
|
||||
`/api/v1/ssh/executors/${encodeURIComponent(executorId)}/reachability/probe`,
|
||||
{
|
||||
...options,
|
||||
init: {
|
||||
...(options?.init ?? {}),
|
||||
method: "POST",
|
||||
headers: { ...(options?.init?.headers ?? {}), "Content-Type": "application/json" },
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
import type { SystemHealthResponse } from "@/lib/types/health";
|
||||
import type { AgentRuntimeAvailability } from "@/lib/types/agent-runtime";
|
||||
import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use";
|
||||
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
|
||||
import type { UISliceActions as UIA } from "./slices/ui/types";
|
||||
import type * as UISliceTypes from "./slices/ui/types";
|
||||
import type {
|
||||
@@ -121,6 +122,7 @@ export type AppState = KanbanSlice & {
|
||||
sleepInhibition: (typeof defaultSettingsState)["sleepInhibition"];
|
||||
userSettings: (typeof defaultSettingsState)["userSettings"];
|
||||
agentProfileRecentUse: (typeof defaultSettingsState)["agentProfileRecentUse"];
|
||||
sshReachability: (typeof defaultSettingsState)["sshReachability"];
|
||||
|
||||
// Session slice
|
||||
messages: (typeof defaultSessionState)["messages"];
|
||||
@@ -152,6 +154,7 @@ export type AppState = KanbanSlice & {
|
||||
sessionMode: (typeof defaultSessionRuntimeState)["sessionMode"];
|
||||
userShells: (typeof defaultSessionRuntimeState)["userShells"];
|
||||
prepareProgress: (typeof defaultSessionRuntimeState)["prepareProgress"];
|
||||
launchWarning: (typeof defaultSessionRuntimeState)["launchWarning"];
|
||||
sessionTodos: (typeof defaultSessionRuntimeState)["sessionTodos"];
|
||||
agentCapabilities: (typeof defaultSessionRuntimeState)["agentCapabilities"];
|
||||
sessionModels: (typeof defaultSessionRuntimeState)["sessionModels"];
|
||||
@@ -282,6 +285,7 @@ export type AppState = KanbanSlice & {
|
||||
upsertAgentUpdateJob: (job: AgentUpdateJob) => void;
|
||||
appendAgentUpdateOutput: (agentName: string, jobId: string, chunk: string) => void;
|
||||
clearAgentUpdateJob: (agentName: string) => void;
|
||||
setSSHReachability: (record: SSHReachabilityRecord) => void;
|
||||
setRepositories: (workspaceId: string, repositories: Repository[]) => void;
|
||||
upsertRepository: (workspaceId: string, repository: Repository) => void;
|
||||
setRepositoriesLoading: (workspaceId: string, loading: boolean) => void;
|
||||
@@ -599,6 +603,11 @@ export type AppState = KanbanSlice & {
|
||||
) => void;
|
||||
setSessionPollMode: (sessionId: string, mode: SessionPollMode) => void;
|
||||
setEmbeddedVscodeSupport: (sessionId: string, supported: boolean) => void;
|
||||
setLaunchWarning: (
|
||||
sessionId: string,
|
||||
entry: import("./slices/session-runtime/types").LaunchWarningEntry,
|
||||
) => void;
|
||||
clearLaunchWarning: (sessionId: string) => void;
|
||||
/* prettier-ignore */ setSidebarActiveView: UIA["setSidebarActiveView"];
|
||||
createSidebarView: UIA["createSidebarView"];
|
||||
updateSidebarDraft: UIA["updateSidebarDraft"];
|
||||
|
||||
@@ -101,6 +101,7 @@ export const defaultState = {
|
||||
sessionMode: defaultSessionRuntimeState.sessionMode,
|
||||
userShells: defaultSessionRuntimeState.userShells,
|
||||
prepareProgress: defaultSessionRuntimeState.prepareProgress,
|
||||
launchWarning: defaultSessionRuntimeState.launchWarning,
|
||||
sessionTodos: defaultSessionRuntimeState.sessionTodos,
|
||||
agentCapabilities: defaultSessionRuntimeState.agentCapabilities,
|
||||
sessionModels: defaultSessionRuntimeState.sessionModels,
|
||||
@@ -515,6 +516,7 @@ export function mergeInitialState(initialState?: HydrationState): DefaultState {
|
||||
sessionMode: { ...defaultState.sessionMode, ...initialState.sessionMode },
|
||||
userShells: { ...defaultState.userShells, ...initialState.userShells },
|
||||
prepareProgress: { ...defaultState.prepareProgress, ...initialState.prepareProgress },
|
||||
launchWarning: { ...defaultState.launchWarning, ...initialState.launchWarning },
|
||||
sessionTodos: { ...defaultState.sessionTodos, ...initialState.sessionTodos },
|
||||
agentCapabilities: { ...defaultState.agentCapabilities, ...initialState.agentCapabilities },
|
||||
sessionModels: { ...defaultState.sessionModels, ...initialState.sessionModels },
|
||||
|
||||
@@ -18,6 +18,20 @@ describe("purgeSessionRuntimeState", () => {
|
||||
store = makeStore();
|
||||
});
|
||||
|
||||
it("clears a launch warning when a relaunch reaches RUNNING", () => {
|
||||
const s = store.getState();
|
||||
s.setLaunchWarning(SESSION_ID, {
|
||||
executorId: "executor-1",
|
||||
host: "10.0.0.5",
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
});
|
||||
|
||||
s.clearLaunchWarning(SESSION_ID);
|
||||
|
||||
expect(store.getState().launchWarning.bySessionId[SESSION_ID]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops per-session maps, process output, and env-scoped buffers", () => {
|
||||
const s = store.getState();
|
||||
s.registerSessionEnvironment(SESSION_ID, "env-1");
|
||||
@@ -39,6 +53,12 @@ describe("purgeSessionRuntimeState", () => {
|
||||
status: "running",
|
||||
});
|
||||
s.appendProcessOutput("proc-1", "process noise");
|
||||
s.setLaunchWarning("session-1", {
|
||||
executorId: "executor-1",
|
||||
host: "10.0.0.5",
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
});
|
||||
|
||||
store.setState((draft) => {
|
||||
purgeSessionRuntimeState(draft, SESSION_ID);
|
||||
@@ -52,6 +72,7 @@ describe("purgeSessionRuntimeState", () => {
|
||||
expect(after.processes.devProcessBySessionId[SESSION_ID]).toBeUndefined();
|
||||
expect(after.processes.processesById["proc-1"]).toBeUndefined();
|
||||
expect(after.processes.outputsByProcessId["proc-1"]).toBeUndefined();
|
||||
expect(after.launchWarning.bySessionId["session-1"]).toBeUndefined();
|
||||
// env-scoped buffers gone because no other session references env-1.
|
||||
expect(after.shell.outputs["env-1"]).toBeUndefined();
|
||||
expect(after.shell.statuses["env-1"]).toBeUndefined();
|
||||
|
||||
@@ -50,6 +50,7 @@ function purgePerSessionRuntime(state: SessionRuntimeSliceState, sessionId: stri
|
||||
delete state.promptUsage.bySessionId[sessionId];
|
||||
delete state.sessionTodos.bySessionId[sessionId];
|
||||
delete state.prepareProgress.bySessionId[sessionId];
|
||||
delete state.launchWarning.bySessionId[sessionId];
|
||||
delete state.sessionPollMode.bySessionId[sessionId];
|
||||
delete state.embeddedVscodeSupport.bySessionId[sessionId];
|
||||
}
|
||||
@@ -121,6 +122,7 @@ export const defaultSessionRuntimeState: SessionRuntimeSliceState = {
|
||||
sessionTodos: { bySessionId: {} },
|
||||
userShells: { byEnvironmentId: {}, dismissedByEnvironmentId: {}, loading: {}, loaded: {} },
|
||||
prepareProgress: { bySessionId: {} },
|
||||
launchWarning: { bySessionId: {} },
|
||||
sessionPollMode: { bySessionId: {} },
|
||||
embeddedVscodeSupport: { bySessionId: {} },
|
||||
workspaceFilesRefresh: { bySessionId: {} },
|
||||
@@ -534,5 +536,13 @@ export const createSessionRuntimeSlice: StateCreator<
|
||||
set((draft) => {
|
||||
draft.sessionTodos.bySessionId[sessionId] = entries;
|
||||
}),
|
||||
setLaunchWarning: (sessionId, entry) =>
|
||||
set((draft) => {
|
||||
draft.launchWarning.bySessionId[sessionId] = entry;
|
||||
}),
|
||||
clearLaunchWarning: (sessionId) =>
|
||||
set((draft) => {
|
||||
delete draft.launchWarning.bySessionId[sessionId];
|
||||
}),
|
||||
...buildUserShellActions(set),
|
||||
});
|
||||
|
||||
@@ -445,6 +445,22 @@ export type EmbeddedVscodeSupportState = {
|
||||
bySessionId: Record<string, boolean>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Applied verbatim from a session.launch.warning event — no field is ever
|
||||
* re-derived from the reachability settings store. See launch-warning.tsx.
|
||||
*/
|
||||
export type LaunchWarningEntry = {
|
||||
executorId: string;
|
||||
host: string;
|
||||
state: string;
|
||||
reason: string;
|
||||
lastSuccessAt?: string;
|
||||
};
|
||||
|
||||
export type LaunchWarningState = {
|
||||
bySessionId: Record<string, LaunchWarningEntry>;
|
||||
};
|
||||
|
||||
export type SessionRuntimeSliceState = {
|
||||
terminal: TerminalState;
|
||||
shell: ShellState;
|
||||
@@ -465,6 +481,7 @@ export type SessionRuntimeSliceState = {
|
||||
sessionTodos: SessionTodosState;
|
||||
userShells: UserShellsState;
|
||||
prepareProgress: PrepareProgressState;
|
||||
launchWarning: LaunchWarningState;
|
||||
sessionPollMode: SessionPollModeState;
|
||||
embeddedVscodeSupport: EmbeddedVscodeSupportState;
|
||||
workspaceFilesRefresh: { bySessionId: Record<string, number> };
|
||||
@@ -558,6 +575,8 @@ export type SessionRuntimeSliceActions = {
|
||||
completeWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean;
|
||||
failWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt, details: string) => boolean;
|
||||
clearWorkspaceRestoration: (attempt: WorkspaceRestorationAttempt) => boolean;
|
||||
setLaunchWarning: (sessionId: string, entry: LaunchWarningEntry) => void;
|
||||
clearLaunchWarning: (sessionId: string) => void;
|
||||
};
|
||||
|
||||
export type SessionRuntimeSlice = SessionRuntimeSliceState & SessionRuntimeSliceActions;
|
||||
|
||||
@@ -366,6 +366,107 @@ describe("setAvailableAgents capability propagation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("ssh reachability", () => {
|
||||
const EXEC_ID = "exec-1";
|
||||
const BASE_TIMESTAMP = "2026-09-17T00:00:00Z";
|
||||
|
||||
function reachabilityRecord(
|
||||
overrides: Partial<import("@/lib/types/http-ssh").SSHReachabilityRecord> = {},
|
||||
): import("@/lib/types/http-ssh").SSHReachabilityRecord {
|
||||
return {
|
||||
executor_id: EXEC_ID,
|
||||
state: "reachable",
|
||||
reason: "",
|
||||
consecutive_failures: 0,
|
||||
host: "build.example",
|
||||
checked_at: BASE_TIMESTAMP,
|
||||
last_success_at: BASE_TIMESTAMP,
|
||||
updated_at: BASE_TIMESTAMP,
|
||||
probing_enabled: true,
|
||||
probe_interval_seconds: 60,
|
||||
persisted: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it("stores a record keyed by executor id", () => {
|
||||
const store = makeStore();
|
||||
store.getState().setSSHReachability(reachabilityRecord());
|
||||
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reachabilityRecord());
|
||||
});
|
||||
|
||||
it("keeps the record with the later updated_at and discards an older one", () => {
|
||||
const store = makeStore();
|
||||
const newer = reachabilityRecord({ state: "unreachable", updated_at: "2026-09-17T01:00:00Z" });
|
||||
const older = reachabilityRecord({ state: "reachable", updated_at: BASE_TIMESTAMP });
|
||||
|
||||
store.getState().setSSHReachability(newer);
|
||||
store.getState().setSSHReachability(older);
|
||||
|
||||
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(newer);
|
||||
});
|
||||
|
||||
it("applies a reset (null checked_at, newer updated_at) over a stored reachable/unreachable record", () => {
|
||||
// A connection-configuration reset clears checked_at but still advances
|
||||
// updated_at. Reconciling on checked_at would make this reset compare as
|
||||
// older than the record it just invalidated and get discarded, leaving a
|
||||
// stale reachable/unreachable state showing for a host the user just
|
||||
// re-pointed.
|
||||
const store = makeStore();
|
||||
const stale = reachabilityRecord({
|
||||
state: "unreachable",
|
||||
checked_at: "2026-09-17T02:00:00Z",
|
||||
updated_at: "2026-09-17T02:00:00Z",
|
||||
});
|
||||
const reset = reachabilityRecord({
|
||||
state: "unknown",
|
||||
checked_at: null,
|
||||
updated_at: "2026-09-17T03:00:00Z",
|
||||
persisted: false,
|
||||
});
|
||||
|
||||
store.getState().setSSHReachability(stale);
|
||||
store.getState().setSSHReachability(reset);
|
||||
|
||||
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(reset);
|
||||
});
|
||||
|
||||
it("discards a null-updated_at placeholder applied after a record with a real timestamp", () => {
|
||||
// The synthesized never-probed placeholder always loses.
|
||||
const store = makeStore();
|
||||
const real = reachabilityRecord({ updated_at: BASE_TIMESTAMP });
|
||||
const placeholder = reachabilityRecord({ state: "unknown", updated_at: null });
|
||||
|
||||
store.getState().setSSHReachability(real);
|
||||
store.getState().setSSHReachability(placeholder);
|
||||
|
||||
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(real);
|
||||
});
|
||||
|
||||
it("applies the first record for an executor even when its updated_at is null", () => {
|
||||
const store = makeStore();
|
||||
const placeholder = reachabilityRecord({ state: "unknown", updated_at: null });
|
||||
|
||||
store.getState().setSSHReachability(placeholder);
|
||||
|
||||
expect(store.getState().sshReachability.byExecutorId[EXEC_ID]).toEqual(placeholder);
|
||||
});
|
||||
|
||||
it("tracks distinct executors independently", () => {
|
||||
const store = makeStore();
|
||||
const first = reachabilityRecord({ executor_id: EXEC_ID });
|
||||
const second = reachabilityRecord({ executor_id: "exec-2", state: "unreachable" });
|
||||
|
||||
store.getState().setSSHReachability(first);
|
||||
store.getState().setSSHReachability(second);
|
||||
|
||||
expect(store.getState().sshReachability.byExecutorId).toEqual({
|
||||
[EXEC_ID]: first,
|
||||
"exec-2": second,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("folder opener discovery", () => {
|
||||
it("defaults unavailable and preserves capability through editor preference updates", () => {
|
||||
const store = makeStore();
|
||||
|
||||
@@ -38,6 +38,7 @@ export const defaultSettingsState: SettingsSliceState = {
|
||||
sleepInhibition: { response: null, loaded: false, loading: false, error: false },
|
||||
userSettings: createDefaultUserSettings(),
|
||||
agentProfileRecentUse: { records: {}, loaded: false },
|
||||
sshReachability: { byExecutorId: {} },
|
||||
};
|
||||
|
||||
type ImmerSet = Parameters<
|
||||
@@ -437,6 +438,29 @@ function createSecretAndSpriteActions(
|
||||
};
|
||||
}
|
||||
|
||||
// reachabilityUpdatedAtMs parses updated_at to epoch ms for reconciliation.
|
||||
// A null updated_at (the synthesized never-probed placeholder) sorts as
|
||||
// "always older" so any real record replaces it.
|
||||
function reachabilityUpdatedAtMs(updatedAt: string | null): number {
|
||||
return updatedAt ? Date.parse(updatedAt) : -Infinity;
|
||||
}
|
||||
|
||||
function createSSHReachabilityActions(set: ImmerSet): Pick<SettingsSlice, "setSSHReachability"> {
|
||||
return {
|
||||
setSSHReachability: (record) =>
|
||||
set((draft) => {
|
||||
const current = draft.sshReachability.byExecutorId[record.executor_id];
|
||||
if (
|
||||
current &&
|
||||
reachabilityUpdatedAtMs(current.updated_at) > reachabilityUpdatedAtMs(record.updated_at)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
draft.sshReachability.byExecutorId[record.executor_id] = record;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const createSettingsSlice: StateCreator<
|
||||
SettingsSlice,
|
||||
[["zustand/immer", never]],
|
||||
@@ -450,4 +474,5 @@ export const createSettingsSlice: StateCreator<
|
||||
...createInstallJobActions(set),
|
||||
...createAgentUpdateJobActions(set),
|
||||
...createSecretAndSpriteActions(set),
|
||||
...createSSHReachabilityActions(set),
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@ import type {
|
||||
} from "@/lib/agent-profile-recent-use";
|
||||
import type { AgentProfileRecentUseContext } from "@/lib/types/http-agent-profile-recent-use";
|
||||
import type { TaskColor } from "@/lib/task-colors";
|
||||
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
|
||||
|
||||
export type {
|
||||
AgentProfileRecentUseRecord,
|
||||
@@ -44,6 +45,15 @@ export type ExecutorsState = {
|
||||
items: Executor[];
|
||||
};
|
||||
|
||||
/**
|
||||
* SSH reachability records keyed by executor id. Populated by a per-card
|
||||
* fetch (task 06's SSHReachabilityCard) and kept live via the
|
||||
* executor.reachability.changed WS event.
|
||||
*/
|
||||
export type SSHReachabilityStoreState = {
|
||||
byExecutorId: Record<string, SSHReachabilityRecord>;
|
||||
};
|
||||
|
||||
export type SettingsAgentsState = {
|
||||
items: Agent[];
|
||||
};
|
||||
@@ -520,6 +530,7 @@ export type SettingsSliceState = {
|
||||
sleepInhibition: SleepInhibitionStoreState;
|
||||
userSettings: UserSettingsState;
|
||||
agentProfileRecentUse: AgentProfileRecentUseState;
|
||||
sshReachability: SSHReachabilityStoreState;
|
||||
};
|
||||
|
||||
export type SettingsSliceActions = {
|
||||
@@ -568,6 +579,16 @@ export type SettingsSliceActions = {
|
||||
record: AgentProfileRecentUseRecord,
|
||||
) => void;
|
||||
bumpAgentProfilesVersion: () => void;
|
||||
/**
|
||||
* Applies a reachability record (a fetch response or a pushed
|
||||
* executor.reachability.changed event). Reconciles on updated_at, never
|
||||
* checked_at: a connection-configuration reset clears checked_at (null)
|
||||
* while still advancing updated_at, so comparing on checked_at would make
|
||||
* the reset compare as older than the record it just invalidated and get
|
||||
* discarded. A null updated_at (the synthesized never-probed placeholder)
|
||||
* always loses to a record that has one.
|
||||
*/
|
||||
setSSHReachability: (record: SSHReachabilityRecord) => void;
|
||||
};
|
||||
|
||||
export type SettingsSlice = SettingsSliceState & SettingsSliceActions;
|
||||
|
||||
@@ -53,6 +53,7 @@ export function buildStateOverrides(m: DefaultState) {
|
||||
sessionMode: m.sessionMode,
|
||||
userShells: m.userShells,
|
||||
prepareProgress: m.prepareProgress,
|
||||
launchWarning: m.launchWarning,
|
||||
sessionTodos: m.sessionTodos,
|
||||
agentCapabilities: m.agentCapabilities,
|
||||
sessionModels: m.sessionModels,
|
||||
|
||||
@@ -44,6 +44,7 @@ import type {
|
||||
} from "@/lib/types/github";
|
||||
import type { TaskMR, TaskMRDeletedEvent, TaskMRAutomationOptions } from "@/lib/types/gitlab";
|
||||
import type { TaskStatusSummary } from "@/lib/types/task-status-summary";
|
||||
import type { SSHReachabilityRecord } from "@/lib/types/http-ssh";
|
||||
import type { AgentProfileRecentUseApiRecord } from "@/lib/types/http-agent-profile-recent-use";
|
||||
import type { SystemMetricsSnapshot, StorageAnalysisUpdatedPayload } from "./system";
|
||||
import type { AgentRuntimeAvailability } from "./agent-runtime";
|
||||
@@ -52,6 +53,7 @@ import type {
|
||||
ExecutorProfilePayload,
|
||||
PrepareProgressPayload,
|
||||
PrepareCompletedPayload,
|
||||
LaunchWarningPayload,
|
||||
EnvironmentPayload,
|
||||
} from "./executor-payloads";
|
||||
|
||||
@@ -358,6 +360,7 @@ export {
|
||||
type ExecutorProfilePayload,
|
||||
type PrepareProgressPayload,
|
||||
type PrepareCompletedPayload,
|
||||
type LaunchWarningPayload,
|
||||
type EnvironmentPayload,
|
||||
} from "./executor-payloads";
|
||||
|
||||
@@ -552,6 +555,11 @@ export type BackendMessageMap = SessionBackendMessageMap &
|
||||
"executor.prepare.completed",
|
||||
PrepareCompletedPayload
|
||||
>;
|
||||
"executor.reachability.changed": BackendMessage<
|
||||
"executor.reachability.changed",
|
||||
SSHReachabilityRecord
|
||||
>;
|
||||
"session.launch.warning": BackendMessage<"session.launch.warning", LaunchWarningPayload>;
|
||||
"environment.created": BackendMessage<"environment.created", EnvironmentPayload>;
|
||||
"environment.updated": BackendMessage<"environment.updated", EnvironmentPayload>;
|
||||
"environment.deleted": BackendMessage<"environment.deleted", EnvironmentPayload>;
|
||||
|
||||
@@ -65,6 +65,17 @@ export type PrepareCompletedPayload = {
|
||||
timestamp: string;
|
||||
};
|
||||
|
||||
export type LaunchWarningPayload = {
|
||||
task_id: string;
|
||||
session_id: string;
|
||||
executor_id: string;
|
||||
host: string;
|
||||
state: string;
|
||||
reason: string;
|
||||
last_success_at?: string;
|
||||
timestamp: string;
|
||||
};
|
||||
|
||||
export type EnvironmentPayload = {
|
||||
id: string;
|
||||
name: string;
|
||||
|
||||
@@ -70,3 +70,32 @@ export interface SSHProbeShellsResponse {
|
||||
duration_ms: number;
|
||||
available: string[];
|
||||
}
|
||||
|
||||
export type SSHReachabilityState = "unknown" | "reachable" | "unreachable";
|
||||
|
||||
export type SSHReachabilityReason =
|
||||
| ""
|
||||
| "config"
|
||||
| "timeout"
|
||||
| "host_key"
|
||||
| "auth"
|
||||
| "network"
|
||||
| "unknown";
|
||||
|
||||
/** Mirrors the backend's reachability.RecordDTO wire shape. */
|
||||
export interface SSHReachabilityRecord {
|
||||
executor_id: string;
|
||||
state: SSHReachabilityState;
|
||||
reason: SSHReachabilityReason;
|
||||
message?: string;
|
||||
consecutive_failures: number;
|
||||
host?: string;
|
||||
checked_at: string | null;
|
||||
last_success_at: string | null;
|
||||
/** Null on the synthesized "never probed" placeholder. */
|
||||
updated_at: string | null;
|
||||
probing_enabled: boolean;
|
||||
/** The effective, clamped probe interval in seconds (0 when disabled). */
|
||||
probe_interval_seconds: number;
|
||||
persisted: boolean;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ function makeStore(overrides: Record<string, unknown> = {}) {
|
||||
setActiveSessionAuto: vi.fn(),
|
||||
setSessionAgentctlStatus: vi.fn(),
|
||||
setResumeSkipped: vi.fn(),
|
||||
clearLaunchWarning: vi.fn(),
|
||||
setSessionFailureNotification: vi.fn(),
|
||||
setContextWindow: vi.fn(),
|
||||
clearContextWindow: vi.fn(),
|
||||
@@ -2079,6 +2080,7 @@ describe("session.state_changed resume-skipped clearing", () => {
|
||||
it("clears the resume-skipped marker on a RUNNING transition", () => {
|
||||
handler(makeMessage({ task_id: "t-1", session_id: "s-1", new_state: "RUNNING" }));
|
||||
expect(store.getState().setResumeSkipped).toHaveBeenCalledWith("s-1", false);
|
||||
expect(store.getState().clearLaunchWarning).toHaveBeenCalledWith("s-1");
|
||||
});
|
||||
|
||||
it("does NOT clear the marker on a STARTING transition (failed resumes keep the retry button)", () => {
|
||||
|
||||
@@ -853,6 +853,12 @@ function handleQueueStatusChangedMessage(
|
||||
state.setQueueEntries(payload.session_id, entries, meta);
|
||||
}
|
||||
|
||||
function clearResumeAndLaunchWarnings(store: StoreApi<AppState>, sessionId: string) {
|
||||
const state = store.getState();
|
||||
state.setResumeSkipped(sessionId, false);
|
||||
state.clearLaunchWarning?.(sessionId);
|
||||
}
|
||||
|
||||
/** Registers the task-session WebSocket handlers (state, messages, workspace sources, queue). */
|
||||
// eslint-disable-next-line max-lines-per-function -- session events remain one ordered registry.
|
||||
export function registerTaskSessionHandlers(store: StoreApi<AppState>): WsHandlers {
|
||||
@@ -912,7 +918,7 @@ export function registerTaskSessionHandlers(store: StoreApi<AppState>): WsHandle
|
||||
// it: a failed manual resume emits STARTING before the launch fails,
|
||||
// and clearing there would drop the Start agent retry affordance.
|
||||
if (newState === "RUNNING") {
|
||||
store.getState().setResumeSkipped(sessionId, false);
|
||||
clearResumeAndLaunchWarnings(store, sessionId);
|
||||
}
|
||||
|
||||
maybeAdoptSessionOnTransition(
|
||||
|
||||
@@ -42,5 +42,8 @@ export function registerExecutorsHandlers(store: StoreApi<AppState>): WsHandlers
|
||||
},
|
||||
}));
|
||||
},
|
||||
"executor.reachability.changed": (message) => {
|
||||
store.getState().setSSHReachability(message.payload);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { StoreApi } from "zustand";
|
||||
import type { AppState } from "@/lib/state/store";
|
||||
import type { BackendMessageMap, LaunchWarningPayload } from "@/lib/types/backend";
|
||||
import { registerSSHLaunchWarningHandlers } from "./ssh-launch-warning";
|
||||
|
||||
const SESSION_ID = "session-1";
|
||||
const TASK_ID = "task-1";
|
||||
|
||||
function makeStore() {
|
||||
const state = {
|
||||
launchWarning: { bySessionId: {} as Record<string, unknown> },
|
||||
setLaunchWarning: vi.fn(),
|
||||
} as unknown as AppState;
|
||||
|
||||
return {
|
||||
getState: () => state,
|
||||
setState: vi.fn(),
|
||||
subscribe: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
getInitialState: vi.fn(),
|
||||
} as unknown as StoreApi<AppState>;
|
||||
}
|
||||
|
||||
function makePayload(overrides: Partial<LaunchWarningPayload> = {}): LaunchWarningPayload {
|
||||
return {
|
||||
task_id: TASK_ID,
|
||||
session_id: SESSION_ID,
|
||||
executor_id: "executor-1",
|
||||
host: "10.0.0.5",
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
last_success_at: "2026-06-10T00:00:00.000Z",
|
||||
timestamp: "2026-06-11T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeMessage(payload: LaunchWarningPayload): BackendMessageMap["session.launch.warning"] {
|
||||
return {
|
||||
id: "message-1",
|
||||
type: "notification",
|
||||
action: "session.launch.warning",
|
||||
payload,
|
||||
timestamp: "2026-06-11T00:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
describe("session.launch.warning handler", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("stores a normalized entry keyed by session id", () => {
|
||||
const store = makeStore();
|
||||
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
|
||||
|
||||
handler(makeMessage(makePayload()));
|
||||
|
||||
expect(store.getState().setLaunchWarning).toHaveBeenCalledWith(SESSION_ID, {
|
||||
executorId: "executor-1",
|
||||
host: "10.0.0.5",
|
||||
state: "unreachable",
|
||||
reason: "timeout",
|
||||
lastSuccessAt: "2026-06-10T00:00:00.000Z",
|
||||
});
|
||||
});
|
||||
|
||||
it("omits lastSuccessAt when the record has never succeeded", () => {
|
||||
const store = makeStore();
|
||||
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
|
||||
|
||||
handler(makeMessage(makePayload({ last_success_at: undefined })));
|
||||
|
||||
expect(store.getState().setLaunchWarning).toHaveBeenCalledWith(
|
||||
SESSION_ID,
|
||||
expect.objectContaining({ lastSuccessAt: undefined }),
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores payloads without a session id", () => {
|
||||
const store = makeStore();
|
||||
const handler = registerSSHLaunchWarningHandlers(store)["session.launch.warning"]!;
|
||||
|
||||
handler(makeMessage(makePayload({ session_id: "" })));
|
||||
|
||||
expect(store.getState().setLaunchWarning).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { StoreApi } from "zustand";
|
||||
import type { AppState } from "@/lib/state/store";
|
||||
import type { WsHandlers } from "@/lib/ws/handlers/types";
|
||||
import type { LaunchWarningPayload } from "@/lib/types/backend";
|
||||
|
||||
export function registerSSHLaunchWarningHandlers(store: StoreApi<AppState>): WsHandlers {
|
||||
return {
|
||||
"session.launch.warning": (message) => {
|
||||
const payload = message.payload as LaunchWarningPayload;
|
||||
if (!payload.session_id) return;
|
||||
store.getState().setLaunchWarning(payload.session_id, {
|
||||
executorId: payload.executor_id,
|
||||
host: payload.host,
|
||||
state: payload.state,
|
||||
reason: payload.reason,
|
||||
lastSuccessAt: payload.last_success_at,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import { registerDiffsHandlers } from "@/lib/ws/handlers/diffs";
|
||||
import { registerExecutorsHandlers } from "@/lib/ws/handlers/executors";
|
||||
import { registerExecutorProfileHandlers } from "@/lib/ws/handlers/executor-profiles";
|
||||
import { registerExecutorPrepareHandlers } from "@/lib/ws/handlers/executor-prepare";
|
||||
import { registerSSHLaunchWarningHandlers } from "@/lib/ws/handlers/ssh-launch-warning";
|
||||
import { registerGitStatusHandlers } from "@/lib/ws/handlers/git-status";
|
||||
import { registerKanbanHandlers } from "@/lib/ws/handlers/kanban";
|
||||
import { registerSystemEventsHandlers } from "@/lib/ws/handlers/system-events";
|
||||
@@ -59,6 +60,7 @@ export function registerWsHandlers(store: StoreApi<AppState>) {
|
||||
...registerExecutorsHandlers(store),
|
||||
...registerExecutorProfileHandlers(store),
|
||||
...registerExecutorPrepareHandlers(store),
|
||||
...registerSSHLaunchWarningHandlers(store),
|
||||
...registerAgentsHandlers(store),
|
||||
...registerTaskSessionHandlers(store),
|
||||
...registerAvailableCommandsHandlers(store),
|
||||
|
||||
@@ -386,6 +386,28 @@
|
||||
"sshProbeHint": "Click \"{{button}}\" to check which agents are installed on the remote.",
|
||||
"sshProxyJumpHint": "Single bastion hop. Chained jumps are not supported.",
|
||||
"sshProxyJumpLabel": "ProxyJump (optional)",
|
||||
"sshReachabilityAgeLastProbe": "Last probe: {{age}}",
|
||||
"sshReachabilityAgeLastSuccess": "Last success: {{age}}",
|
||||
"sshReachabilityConsecutiveFailures_one": "{{count}} consecutive failure",
|
||||
"sshReachabilityConsecutiveFailures_other": "{{count}} consecutive failures",
|
||||
"sshReachabilityDescription": "The most recent SSH probe result for this host, checked automatically in the background.",
|
||||
"sshReachabilityHost": "Probed host: {{host}}",
|
||||
"sshReachabilityLoadFailed": "Reachability status could not be loaded. It is not known.",
|
||||
"sshReachabilityNeverProbed": "Not probed yet.",
|
||||
"sshReachabilityNoSuccess": "No successful probe recorded.",
|
||||
"sshReachabilityProbeNow": "Probe now",
|
||||
"sshReachabilityProbingOff": "Periodic probing is off. This shows the last recorded result, not the current state.",
|
||||
"sshReachabilityReasonAuth": "Authentication failed",
|
||||
"sshReachabilityReasonConfig": "The connection configuration is invalid",
|
||||
"sshReachabilityReasonHostKey": "The host key does not match the pinned fingerprint",
|
||||
"sshReachabilityReasonNetwork": "A network error occurred",
|
||||
"sshReachabilityReasonTimeout": "The connection attempt timed out",
|
||||
"sshReachabilityReasonUnknown": "An unknown error occurred",
|
||||
"sshReachabilityStale": "Stale: no probe within the last three check intervals",
|
||||
"sshReachabilityStateReachable": "Reachable",
|
||||
"sshReachabilityStateUnknown": "Unknown",
|
||||
"sshReachabilityStateUnreachable": "Unreachable",
|
||||
"sshReachabilityTitle": "Reachability",
|
||||
"sshReclaimTaskDirDescription": "Every task that runs on {{host}} keeps its checkout under {{path}} after the task ends, so a resumed or unarchived task finds its work where it left it.",
|
||||
"sshReclaimTaskDirLabel": "Delete a task's directory when the task is archived or deleted",
|
||||
"sshReclaimTaskDirTitle": "Remote task directories",
|
||||
|
||||
@@ -2245,6 +2245,16 @@
|
||||
"launchErrorSessionPreview": "The task launch needs attention.",
|
||||
"launchNeedsAttention": "Launch needs attention",
|
||||
"launchErrorNoChanges": "No task or branch settings were changed.",
|
||||
"launchWarningTitle": "SSH host may be unreachable",
|
||||
"launchWarningHost": "Probed host: {{host}}",
|
||||
"launchWarningLastSuccess": "Last success: {{age}}",
|
||||
"launchWarningNeverSucceeded": "No successful probe has ever been recorded for this host.",
|
||||
"launchWarningReasonConfig": "The last attempt failed because the connection configuration is invalid.",
|
||||
"launchWarningReasonTimeout": "The last attempt timed out.",
|
||||
"launchWarningReasonHostKey": "The last attempt failed because the host key did not match the pinned fingerprint.",
|
||||
"launchWarningReasonAuth": "The last attempt failed authentication.",
|
||||
"launchWarningReasonNetwork": "The last attempt failed with a network error.",
|
||||
"launchWarningReasonUnknown": "The last attempt failed for an unknown reason.",
|
||||
"launchPromptPreview": "Launch prompt preview",
|
||||
"retryWithDefaultBranch": "Retry with default branch",
|
||||
"pickBaseBranch": "Pick a base branch",
|
||||
|
||||
@@ -386,6 +386,28 @@
|
||||
"sshProbeHint": "Ćĺĩćķ \"{{button}}\" ţō ćĥēćķ ŵĥĩćĥ àĝēńţś àŕē ĩńśţàĺĺēď ōń ţĥē ŕēḿōţē.",
|
||||
"sshProxyJumpHint": "Śĩńĝĺē ƀàśţĩōń ĥōƥ. Ćĥàĩńēď ĵũḿƥś àŕē ńōţ śũƥƥōŕţēď.",
|
||||
"sshProxyJumpLabel": "ƤŕōxŷĴũḿƥ (ōƥţĩōńàĺ)",
|
||||
"sshReachabilityAgeLastProbe": "Ĺàśţ ƥŕōƀē: {{age}}",
|
||||
"sshReachabilityAgeLastSuccess": "Ĺàśţ śũććēśś: {{age}}",
|
||||
"sshReachabilityConsecutiveFailures_one": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕē",
|
||||
"sshReachabilityConsecutiveFailures_other": "{{count}} ćōńśēćũţĩvē ƒàĩĺũŕēś",
|
||||
"sshReachabilityDescription": "Ţĥē ḿōśţ ŕēćēńţ ŚŚĤ ƥŕōƀē ŕēśũĺţ ƒōŕ ţĥĩś ĥōśţ, ćĥēćķēď àũţōḿàţĩćàĺĺŷ ĩń ţĥē ƀàćķĝŕōũńď.",
|
||||
"sshReachabilityHost": "Ƥŕōƀēď ĥōśţ: {{host}}",
|
||||
"sshReachabilityLoadFailed": "Ŕēàćĥàƀĩĺĩţŷ śţàţũś ćōũĺď ńōţ ƀē ĺōàďēď. Ĩţ ĩś ńōţ ķńōŵń.",
|
||||
"sshReachabilityNeverProbed": "Ńōţ ƥŕōƀēď ŷēţ.",
|
||||
"sshReachabilityNoSuccess": "Ńō śũććēśśƒũĺ ƥŕōƀē ŕēćōŕďēď.",
|
||||
"sshReachabilityProbeNow": "Ƥŕōƀē ńōŵ",
|
||||
"sshReachabilityProbingOff": "Ƥēŕĩōďĩć ƥŕōƀĩńĝ ĩś ōƒƒ. Ţĥĩś śĥōŵś ţĥē ĺàśţ ŕēćōŕďēď ŕēśũĺţ, ńōţ ţĥē ćũŕŕēńţ śţàţē.",
|
||||
"sshReachabilityReasonAuth": "Àũţĥēńţĩćàţĩōń ƒàĩĺēď",
|
||||
"sshReachabilityReasonConfig": "Ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď",
|
||||
"sshReachabilityReasonHostKey": "Ţĥē ĥōśţ ķēŷ ďōēś ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ",
|
||||
"sshReachabilityReasonNetwork": "À ńēţŵōŕķ ēŕŕōŕ ōććũŕŕēď",
|
||||
"sshReachabilityReasonTimeout": "Ţĥē ćōńńēćţĩōń àţţēḿƥţ ţĩḿēď ōũţ",
|
||||
"sshReachabilityReasonUnknown": "Àń ũńķńōŵń ēŕŕōŕ ōććũŕŕēď",
|
||||
"sshReachabilityStale": "Śţàĺē: ńō ƥŕōƀē ŵĩţĥĩń ţĥē ĺàśţ ţĥŕēē ćĥēćķ ĩńţēŕvàĺś",
|
||||
"sshReachabilityStateReachable": "Ŕēàćĥàƀĺē",
|
||||
"sshReachabilityStateUnknown": "Ũńķńōŵń",
|
||||
"sshReachabilityStateUnreachable": "Ũńŕēàćĥàƀĺē",
|
||||
"sshReachabilityTitle": "Ŕēàćĥàƀĩĺĩţŷ",
|
||||
"sshReclaimTaskDirDescription": "Ēvēŕŷ ţàśķ ţĥàţ ŕũńś ōń {{host}} ķēēƥś ĩţś ćĥēćķōũţ ũńďēŕ {{path}} àƒţēŕ ţĥē ţàśķ ēńďś, śō à ŕēśũḿēď ōŕ ũńàŕćĥĩvēď ţàśķ ƒĩńďś ĩţś ŵōŕķ ŵĥēŕē ĩţ ĺēƒţ ĩţ.",
|
||||
"sshReclaimTaskDirLabel": "Ďēĺēţē à ţàśķ'ś ďĩŕēćţōŕŷ ŵĥēń ţĥē ţàśķ ĩś àŕćĥĩvēď ōŕ ďēĺēţēď",
|
||||
"sshReclaimTaskDirTitle": "Ŕēḿōţē ţàśķ ďĩŕēćţōŕĩēś",
|
||||
|
||||
@@ -2245,6 +2245,16 @@
|
||||
"launchErrorSessionPreview": "Ţĥē ţàśķ ĺàũńćĥ ńēēďś àţţēńţĩōń.",
|
||||
"launchNeedsAttention": "Ĺàũńćĥ ńēēďś àţţēńţĩōń",
|
||||
"launchErrorNoChanges": "Ńō ţàśķ ōŕ ƀŕàńćĥ śēţţĩńĝś ŵēŕē ćĥàńĝēď.",
|
||||
"launchWarningTitle": "ŚŚĤ ĥōśţ ḿàŷ ƀē ũńŕēàćĥàƀĺē",
|
||||
"launchWarningHost": "Ƥŕōƀēď ĥōśţ: {{host}}",
|
||||
"launchWarningLastSuccess": "Ĺàśţ śũććēśś: {{age}}",
|
||||
"launchWarningNeverSucceeded": "Ńō śũććēśśƒũĺ ƥŕōƀē ĥàś ēvēŕ ƀēēń ŕēćōŕďēď ƒōŕ ţĥĩś ĥōśţ.",
|
||||
"launchWarningReasonConfig": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ćōńńēćţĩōń ćōńƒĩĝũŕàţĩōń ĩś ĩńvàĺĩď.",
|
||||
"launchWarningReasonTimeout": "Ţĥē ĺàśţ àţţēḿƥţ ţĩḿēď ōũţ.",
|
||||
"launchWarningReasonHostKey": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƀēćàũśē ţĥē ĥōśţ ķēŷ ďĩď ńōţ ḿàţćĥ ţĥē ƥĩńńēď ƒĩńĝēŕƥŕĩńţ.",
|
||||
"launchWarningReasonAuth": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď àũţĥēńţĩćàţĩōń.",
|
||||
"launchWarningReasonNetwork": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ŵĩţĥ à ńēţŵōŕķ ēŕŕōŕ.",
|
||||
"launchWarningReasonUnknown": "Ţĥē ĺàśţ àţţēḿƥţ ƒàĩĺēď ƒōŕ àń ũńķńōŵń ŕēàśōń.",
|
||||
"launchPromptPreview": "Ĺàũńćĥ ƥŕōḿƥţ ƥŕēvĩēŵ",
|
||||
"retryWithDefaultBranch": "Ŕēţŕŷ ŵĩţĥ ďēƒàũĺţ ƀŕàńćĥ",
|
||||
"pickBaseBranch": "Ƥĩćķ à ƀàśē ƀŕàńćĥ",
|
||||
|
||||
@@ -370,6 +370,28 @@
|
||||
"sshProbeHint": "Clique em \"{{button}}\" para verificar que agentes estão instalados no anfitrião remoto.",
|
||||
"sshProxyJumpHint": "Um único salto por bastião. Saltos encadeados não são suportados.",
|
||||
"sshProxyJumpLabel": "ProxyJump (opcional)",
|
||||
"sshReachabilityAgeLastProbe": "Última verificação: {{age}}",
|
||||
"sshReachabilityAgeLastSuccess": "Último sucesso: {{age}}",
|
||||
"sshReachabilityConsecutiveFailures_one": "{{count}} falha consecutiva",
|
||||
"sshReachabilityConsecutiveFailures_other": "{{count}} falhas consecutivas",
|
||||
"sshReachabilityDescription": "O resultado mais recente da verificação SSH deste anfitrião, verificado automaticamente em segundo plano.",
|
||||
"sshReachabilityHost": "Anfitrião verificado: {{host}}",
|
||||
"sshReachabilityLoadFailed": "Não foi possível carregar o estado de alcançabilidade. Não é conhecido.",
|
||||
"sshReachabilityNeverProbed": "Ainda não verificado.",
|
||||
"sshReachabilityNoSuccess": "Nenhuma verificação bem-sucedida registada.",
|
||||
"sshReachabilityProbeNow": "Verificar agora",
|
||||
"sshReachabilityProbingOff": "A verificação periódica está desativada. Isto mostra o último resultado registado, não o estado atual.",
|
||||
"sshReachabilityReasonAuth": "Falha na autenticação",
|
||||
"sshReachabilityReasonConfig": "A configuração da ligação é inválida",
|
||||
"sshReachabilityReasonHostKey": "A chave do anfitrião não corresponde à impressão digital fixada",
|
||||
"sshReachabilityReasonNetwork": "Ocorreu um erro de rede",
|
||||
"sshReachabilityReasonTimeout": "A tentativa de ligação expirou",
|
||||
"sshReachabilityReasonUnknown": "Ocorreu um erro desconhecido",
|
||||
"sshReachabilityStale": "Desatualizado: sem verificação nos últimos três intervalos",
|
||||
"sshReachabilityStateReachable": "Alcançável",
|
||||
"sshReachabilityStateUnknown": "Desconhecido",
|
||||
"sshReachabilityStateUnreachable": "Inalcançável",
|
||||
"sshReachabilityTitle": "Alcançabilidade",
|
||||
"sshReclaimTaskDirDescription": "Cada tarefa executada em {{host}} mantém a sua cópia de trabalho em {{path}} depois de terminar, para que uma tarefa retomada ou desarquivada encontre o trabalho onde o deixou.",
|
||||
"sshReclaimTaskDirLabel": "Eliminar o diretório de uma tarefa quando a tarefa for arquivada ou eliminada",
|
||||
"sshReclaimTaskDirTitle": "Diretórios de tarefas remotos",
|
||||
|
||||
@@ -2245,6 +2245,16 @@
|
||||
"launchErrorSessionPreview": "O início da tarefa requer atenção.",
|
||||
"launchNeedsAttention": "O início requer atenção",
|
||||
"launchErrorNoChanges": "As definições da tarefa e do branch não foram alteradas.",
|
||||
"launchWarningTitle": "O anfitrião SSH pode estar inacessível",
|
||||
"launchWarningHost": "Anfitrião testado: {{host}}",
|
||||
"launchWarningLastSuccess": "Último sucesso: {{age}}",
|
||||
"launchWarningNeverSucceeded": "Nunca foi registada uma sondagem bem-sucedida para este anfitrião.",
|
||||
"launchWarningReasonConfig": "A última tentativa falhou porque a configuração da ligação é inválida.",
|
||||
"launchWarningReasonTimeout": "A última tentativa expirou.",
|
||||
"launchWarningReasonHostKey": "A última tentativa falhou porque a chave do anfitrião não correspondeu à impressão digital fixada.",
|
||||
"launchWarningReasonAuth": "A última tentativa falhou na autenticação.",
|
||||
"launchWarningReasonNetwork": "A última tentativa falhou com um erro de rede.",
|
||||
"launchWarningReasonUnknown": "A última tentativa falhou por um motivo desconhecido.",
|
||||
"launchPromptPreview": "Pré-visualização do prompt de início",
|
||||
"retryWithDefaultBranch": "Tentar novamente com o branch predefinido",
|
||||
"pickBaseBranch": "Escolher um branch base",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user