refactor(ssh): drop the unused conn pool + remote curl dependency

CodeRabbit and Greptile both flagged the SSHConnPool: the executor never
took the pool path (CreateInstance always dialDirect'd a fresh client and
parked it on sshSessionState), so the pool's keepalive goroutines and
refcount logic were dead code. Get() also overwrote map entries on every
call, which would have leaked keepalive goroutines if the pool *had* been
used. Delete the entire SSHConnPool + sshConnKey + pooledConn + keepalive
plumbing. Export DialSSH so the SSH test endpoint can dial directly and
close the client when its single-shot probe is done.

While here:
  * pickRemoteAgentctlPort: switch from time.Now().UnixNano()%20000 to
    math/rand/v2.IntN. Two concurrent CreateInstance calls on a fast
    machine were getting the same port (UnixNano cycles in ~20µs through
    the 20k range); randomized picks avoid the collision.
  * createRemoteAgentInstance: POST to the remote control port over a
    direct-tcpip dial via net/http instead of shelling out to `curl`. The
    sshd container no longer needs curl, the kandev test endpoint no
    longer needs to probe for it, and the call inherits Go's standard
    timeout + error handling. Drop `curl` from the e2e Alpine image.
  * maybeUploadCredentials: was probing `printf %s "$HOME"` and
    discarding the output, then allocating an unused sshFileUploader.
    SSH v1 doesn't ship credential upload — just log a warning when a
    user sets remote_credentials so they know it won't take effect, and
    delete the dead uploader file.
This commit is contained in:
José Almeida
2026-05-17 20:06:53 +01:00
parent 09b8f8511f
commit a30833b336
6 changed files with 92 additions and 287 deletions
@@ -42,12 +42,11 @@ type sshSessionState struct {
// SSHExecutor implements ExecutorBackend for SSH-reachable Linux hosts.
//
// One SSH connection is shared per (host, user, identity) tuple across all
// sessions on that host. Each session gets its own agentctl process + remote
// port + local-forwarded port. See docs/specs/ssh-executor/spec.md for the
// full design.
// Each session owns its own *ssh.Client (no shared pool). One SSH connection
// per session keeps teardown simple — closing the executor instance closes
// the client — at the cost of an extra TCP+handshake per session on the same
// host. See docs/specs/ssh-executor/spec.md for the full design.
type SSHExecutor struct {
pool *SSHConnPool
agentctlResolver *AgentctlResolver
secretStore secrets.SecretStore
agentList RemoteAuthAgentLister
@@ -66,7 +65,6 @@ func NewSSHExecutor(
log *logger.Logger,
) *SSHExecutor {
return &SSHExecutor{
pool: NewSSHConnPool(log),
agentctlResolver: resolver,
secretStore: secretStore,
agentList: agentList,
@@ -84,11 +82,10 @@ func (r *SSHExecutor) HealthCheck(_ context.Context) error {
return nil
}
// Close terminates pooled SSH connections at shutdown.
func (r *SSHExecutor) Close() error {
r.pool.CloseAll()
return nil
}
// Close is a no-op: per-session SSH clients are owned by sshSessionState and
// torn down by StopInstance. Kept on the type so the executor satisfies
// io.Closer alongside the other backends.
func (r *SSHExecutor) Close() error { return nil }
// targetFromMetadata builds an SSHTarget from the executor metadata
// propagated via buildLaunchMetadata (req.ExecutorConfig keys merged in).
@@ -461,34 +458,21 @@ func (r *SSHExecutor) IsAlwaysResumable() bool { return true }
// creation. SSH-specific tweaks: remote auth target dir defaults to the SSH
// user's home (resolved at runtime via a tiny `pwd` probe), and the file
// uploader writes via SFTP.
func (r *SSHExecutor) maybeUploadCredentials(ctx context.Context, client *ssh.Client, req *ExecutorCreateRequest) {
credsJSON := getMetadataString(req.Metadata, "remote_credentials")
if credsJSON == "" {
//
// v1 ships without remote-credentials upload — the field is accepted on the
// executor profile (parity with Sprites/Docker) but the upload pipeline lives
// in the Sprites executor and is not yet ported over. A non-empty
// remote_credentials log is the only side effect so users notice they
// configured a feature that isn't active yet.
func (r *SSHExecutor) maybeUploadCredentials(_ context.Context, _ *ssh.Client, req *ExecutorCreateRequest) {
if getMetadataString(req.Metadata, "remote_credentials") == "" {
return
}
// Resolve remote $HOME for the target auth dir, unless the profile pinned one.
if getMetadataString(req.Metadata, MetadataKeyRemoteAuthHome) == "" {
if _, _, err := runSSHCommand(ctx, client, "printf %s \"$HOME\""); err != nil {
r.logger.Warn("ssh credentials: could not resolve remote $HOME, skipping upload", zap.Error(err))
return
}
}
// Reuse the catalog + selection logic from the credential-uploader.
// We intentionally skip the env-secret resolution + auth setup scripts here
// (those live in the Sprites executor and depend on agent install scripts);
// v1 SSH just pushes the configured credential files.
if r.agentList == nil {
return
}
// The credential-upload pipeline is exercised end-to-end by Sprites; for
// SSH v1 we wire the FileUploader and let the caller select methods via
// the existing profile-config "remote_credentials" key. UploadCredentialFiles
// itself reads the local files and writes them through SFTP.
uploader := newSSHFileUploader(client)
// We do not parse the methods here; the more elaborate selection is the
// SpritesExecutor's job. For now, write a marker file to confirm SFTP
// works and surface the credentials JSON for follow-up integration.
_ = uploader // referenced for future expansion; intentional no-op in v1
r.logger.Warn(
"ssh executor: remote_credentials is configured but credential upload is not yet implemented for the SSH runtime; see docs/specs/ssh-executor/spec.md",
zap.String("task_id", req.TaskID),
zap.String("session_id", req.SessionID),
)
}
// sshTaskDirName builds a stable per-task remote directory name. Prefers an
@@ -9,22 +9,14 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/kevinburke/ssh_config"
"go.uber.org/zap"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"github.com/kandev/kandev/internal/common/logger"
)
const (
sshDialTimeout = 30 * time.Second
sshKeepaliveEvery = 30 * time.Second
sshKeepaliveTimeout = 15 * time.Second
)
const sshDialTimeout = 30 * time.Second
// SSHIdentitySource enumerates how kandev obtains an SSH credential.
type SSHIdentitySource string
@@ -307,6 +299,15 @@ func buildClientConfig(target *SSHTarget) (*ssh.ClientConfig, error) {
}, nil
}
// DialSSH opens an SSH connection to target, optionally via a bastion if
// ProxyJump is set. Returns the live *ssh.Client and the observed fingerprint
// (recorded on target.ObservedFingerprint via the host-key callback).
//
// Callers own the client and must Close it when done.
func DialSSH(ctx context.Context, target *SSHTarget) (*ssh.Client, error) {
return dialSSH(ctx, target)
}
// dial opens an SSH connection to target, optionally via a bastion if ProxyJump
// is set. Returns the live *ssh.Client and the observed fingerprint.
func dialSSH(ctx context.Context, target *SSHTarget) (*ssh.Client, error) {
@@ -437,161 +438,8 @@ func parseLiteralProxyJump(s string) (user, host string, port int, ok bool) {
return user, host, port, true
}
// sshConnKey identifies a pooled SSH connection. Connections are not shared
// across identity sources or proxy-jump configurations to avoid surprising
// behavior when a user changes auth methods mid-session.
type sshConnKey struct {
host string
port int
user string
identitySource SSHIdentitySource
identityFile string
proxyJump string
}
func keyForTarget(target *SSHTarget) sshConnKey {
return sshConnKey{
host: target.Host,
port: target.Port,
user: target.User,
identitySource: target.IdentitySource,
identityFile: target.IdentityFile,
proxyJump: target.ProxyJump,
}
}
// pooledConn is a refcounted live SSH client shared across an executor's
// sessions on the same host.
type pooledConn struct {
client *ssh.Client
target *SSHTarget
refs int
keepalive context.CancelFunc
}
// SSHConnPool multiplexes a single SSH client per (host, user, identity) tuple
// across all of the executor's live sessions on that host. Sessions acquire a
// connection via Get; the pool releases the underlying client when the last
// session releases.
type SSHConnPool struct {
mu sync.Mutex
entries map[sshConnKey]*pooledConn
logger *logger.Logger
}
func NewSSHConnPool(log *logger.Logger) *SSHConnPool {
return &SSHConnPool{
entries: make(map[sshConnKey]*pooledConn),
logger: log.WithFields(zap.String("component", "ssh-conn-pool")),
}
}
// Get returns a live *ssh.Client for the target. The pool keeps the
// underlying ssh.Client per (host, user, identity, …) so multiple sessions
// against the same host share a single SSH transport — channels are cheap,
// connections are not.
//
// NOTE: golang.org/x/crypto/ssh's Client.Conn carries a mutex around the
// channel/mux state. Several short-lived Dial calls from concurrent goroutines
// against the same Client occasionally race in a way that surfaces as
// `io.EOF` on the OpenChannel response, with no corresponding rejection from
// the server. To keep the e2e + production paths reliable, the pool dials a
// fresh connection per executor call and releases it back to the OS at the
// end of the session. Reusing one Client across N concurrent direct-tcpip
// channels remains a perf improvement we can revisit when we have a clean
// repro for the race.
func (p *SSHConnPool) Get(ctx context.Context, target *SSHTarget) (*ssh.Client, error) {
key := keyForTarget(target)
client, err := dialSSH(ctx, target)
if err != nil {
return nil, err
}
p.mu.Lock()
defer p.mu.Unlock()
kctx, kcancel := context.WithCancel(context.Background())
entry := &pooledConn{client: client, target: target, refs: 1, keepalive: kcancel}
// Multiple sessions against the same target get distinct entries; the map
// key still keys by tuple but acts as "any one matching" — Release walks
// the slice via refcount semantics, see Release().
p.entries[key] = entry
go p.runKeepalive(kctx, key, client)
return client, nil
}
// Release decrements the refcount; when it hits zero the underlying client is
// closed and removed from the pool.
func (p *SSHConnPool) Release(target *SSHTarget) {
key := keyForTarget(target)
p.mu.Lock()
defer p.mu.Unlock()
entry, ok := p.entries[key]
if !ok {
return
}
entry.refs--
if entry.refs > 0 {
return
}
entry.keepalive()
_ = entry.client.Close()
delete(p.entries, key)
}
// CloseAll terminates every pooled connection. Used at shutdown.
func (p *SSHConnPool) CloseAll() {
p.mu.Lock()
defer p.mu.Unlock()
for k, entry := range p.entries {
entry.keepalive()
_ = entry.client.Close()
delete(p.entries, k)
}
}
// runKeepalive sends an ssh keepalive every sshKeepaliveEvery so dropped
// connections surface within ~30s instead of waiting for the next operation.
// On any keepalive failure the pool entry is evicted so the next Get will
// re-dial.
func (p *SSHConnPool) runKeepalive(ctx context.Context, key sshConnKey, client *ssh.Client) {
t := time.NewTicker(sshKeepaliveEvery)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
done := make(chan error, 1)
go func() {
_, _, err := client.SendRequest("keepalive@kandev", true, nil)
done <- err
}()
select {
case err := <-done:
if err != nil {
p.evictOnFailure(key, err)
return
}
case <-time.After(sshKeepaliveTimeout):
p.evictOnFailure(key, fmt.Errorf("keepalive timed out"))
return
}
}
}
}
func (p *SSHConnPool) evictOnFailure(key sshConnKey, err error) {
p.mu.Lock()
defer p.mu.Unlock()
entry, ok := p.entries[key]
if !ok {
return
}
p.logger.Warn("ssh connection lost — evicting from pool",
zap.String("host", key.host),
zap.String("user", key.user),
zap.Error(err))
_ = entry.client.Close()
delete(p.entries, key)
}
// SSH connections are owned per-session (no shared pool): a session's client
// lives on sshSessionState and is Close()d by StopInstance. The earlier
// pool/refcount/keepalive plumbing was dead code — the executor never used it
// in production, and the orphaned keepalive goroutines surfaced under e2e
// fault-injection. See PR #927 for the removal rationale.
@@ -1,23 +0,0 @@
package lifecycle
import (
"context"
"os"
"golang.org/x/crypto/ssh"
)
// sshFileUploader implements FileUploader for an SSH client by writing through
// SFTP. Reused by the existing credential-upload pipeline so SSH executor gets
// gh_cli_token / claude / codex / etc. credential propagation for free.
type sshFileUploader struct {
client *ssh.Client
}
func newSSHFileUploader(client *ssh.Client) *sshFileUploader {
return &sshFileUploader{client: client}
}
func (u *sshFileUploader) WriteFile(_ context.Context, path string, data []byte, mode os.FileMode) error {
return sftpWriteFile(u.client, path, data, mode)
}
@@ -9,7 +9,9 @@ import (
"errors"
"fmt"
"io"
mrand "math/rand/v2"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
@@ -21,7 +23,7 @@ import (
"go.uber.org/zap"
"golang.org/x/crypto/ssh"
agentctl "github.com/kandev/kandev/internal/agentctl/client"
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/common/logger"
)
@@ -261,17 +263,6 @@ func sftpUploadBytes(client *ssh.Client, remotePath string, data []byte, mode os
return nil
}
// sftpWriteFile is the FileUploader.WriteFile implementation: ensures parent
// dirs exist and atomically writes the file.
func sftpWriteFile(client *ssh.Client, remotePath string, data []byte, mode os.FileMode) error {
if dir := filepath.Dir(remotePath); dir != "" && dir != "." {
if _, _, err := runSSHCommand(context.Background(), client, "mkdir -p "+shellQuote(dir)); err != nil {
return fmt.Errorf("ssh: mkdir %s: %w", dir, err)
}
}
return sftpUploadBytes(client, remotePath, data, mode)
}
// shellQuote is a minimal POSIX shell-safe single-quote wrapper.
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
@@ -391,11 +382,11 @@ func startRemoteAgentctl(
const sshAgentctlLogTailLines = 25
// createRemoteAgentInstance creates a per-session agent instance on the
// remote agentctl control server by POSTing to /api/v1/instances from inside
// the container (so we don't need a second SSH port forward just to call the
// control API). Returns the per-instance port the SSH executor should later
// forward + dial for ACP / workspace traffic. Mirrors what
// executor_sprites.go does inside its sprite.
// remote agentctl control server by POSTing to /api/v1/instances over a
// direct-tcpip channel through the existing SSH client — no second port
// forward, and no dependency on remote curl. Returns the per-instance port
// the SSH executor should later forward + dial for ACP / workspace traffic.
// Mirrors what executor_sprites.go does inside its sprite.
func createRemoteAgentInstance(
ctx context.Context,
client *ssh.Client,
@@ -416,39 +407,47 @@ func createRemoteAgentInstance(
if err != nil {
return 0, fmt.Errorf("ssh: marshal create-instance: %w", err)
}
// Pipe the JSON body via curl's --data-binary @- so the shell doesn't have
// to escape it. -sS to suppress progress but keep errors. --fail returns
// non-zero on HTTP >= 400 so we get a clear error code.
cmd := fmt.Sprintf(
"curl -sS --fail -X POST -H 'Content-Type: application/json' --data-binary @- http://127.0.0.1:%d/api/v1/instances",
controlPort,
)
session, err := client.NewSession()
// HTTP-over-direct-tcpip: every request dials a fresh SSH channel to the
// remote control port. Keep-alives are disabled so the channel closes
// after the response.
httpClient := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return client.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(controlPort)))
},
DisableKeepAlives: true,
},
}
url := fmt.Sprintf("http://127.0.0.1:%d/api/v1/instances", controlPort)
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return 0, fmt.Errorf("ssh: new session: %w", err)
return 0, fmt.Errorf("ssh: build create-instance request: %w", err)
}
defer func() { _ = session.Close() }()
session.Stdin = bytes.NewReader(body)
var stdout, stderr bytes.Buffer
session.Stdout = &stdout
session.Stderr = &stderr
done := make(chan error, 1)
go func() { done <- session.Run(cmd) }()
select {
case <-ctx.Done():
_ = session.Signal(ssh.SIGTERM)
return 0, ctx.Err()
case rerr := <-done:
if rerr != nil {
return 0, fmt.Errorf("ssh: create-instance curl: %w (stderr: %s)", rerr, strings.TrimSpace(stderr.String()))
}
httpReq.Header.Set("Content-Type", "application/json")
httpResp, err := httpClient.Do(httpReq)
if err != nil {
return 0, fmt.Errorf("ssh: create-instance dial: %w", err)
}
defer func() { _ = httpResp.Body.Close() }()
respBody, err := io.ReadAll(httpResp.Body)
if err != nil {
return 0, fmt.Errorf("ssh: read create-instance response: %w", err)
}
if httpResp.StatusCode >= http.StatusBadRequest {
return 0, fmt.Errorf("ssh: create-instance returned %d: %s", httpResp.StatusCode, strings.TrimSpace(string(respBody)))
}
var resp agentctl.CreateInstanceResponse
if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil {
return 0, fmt.Errorf("ssh: parse create-instance response: %w (body: %s)", err, stdout.String())
if err := json.Unmarshal(respBody, &resp); err != nil {
return 0, fmt.Errorf("ssh: parse create-instance response: %w (body: %s)", err, string(respBody))
}
if resp.Port == 0 {
return 0, fmt.Errorf("ssh: create-instance returned port 0 (body: %s)", stdout.String())
return 0, fmt.Errorf("ssh: create-instance returned port 0 (body: %s)", string(respBody))
}
log.Info("created remote agent instance",
zap.Int("control_port", controlPort),
@@ -467,14 +466,13 @@ func sshAgentTypeFromReq(req *ExecutorCreateRequest) string {
}
// pickRemoteAgentctlPort returns a port in [40000, 60000). The kandev backend
// picks per session; agentctl honors AGENTCTL_PORT. Collisions inside one sshd
// container would surface as a clear "failed to bind" error and a quick retry
// fixes it — in practice for e2e and a single SSH user, collisions are
// vanishingly unlikely.
//
//nolint:gosec // not used for cryptographic purposes
// picks per session; agentctl honors AGENTCTL_PORT. Uses math/rand/v2 so two
// concurrent CreateInstance calls don't collide (UnixNano%20000 cycles in
// ~20µs, which is well within the window between back-to-back launches on a
// fast machine). A residual collision still surfaces as a clear bind failure
// and the caller can retry.
func pickRemoteAgentctlPort() int {
return 40000 + int(time.Now().UnixNano())%20000
return 40000 + mrand.IntN(20000)
}
// stopRemoteAgentctl best-effort kills a remote agentctl by PID and removes
+7 -8
View File
@@ -15,7 +15,8 @@ import (
"go.uber.org/zap"
"golang.org/x/crypto/ssh"
"github.com/kandev/kandev/internal/agent/lifecycle"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/agentruntime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
ws "github.com/kandev/kandev/pkg/websocket"
@@ -192,13 +193,11 @@ func (h *Handler) runTest(ctx context.Context, req TestRequest) *TestResult {
return finalize()
}
pool := lifecycle.NewSSHConnPool(h.logger)
defer pool.CloseAll()
client, err := h.testHandshake(ctx, pool, target, result)
client, err := h.testHandshake(ctx, target, result)
if err != nil {
return finalize()
}
defer func() { _ = client.Close() }()
infoCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
@@ -237,10 +236,10 @@ func (h *Handler) testResolveTarget(req TestRequest, result *TestResult) (*lifec
}
func (h *Handler) testHandshake(
ctx context.Context, pool *lifecycle.SSHConnPool, target *lifecycle.SSHTarget, result *TestResult,
ctx context.Context, target *lifecycle.SSHTarget, result *TestResult,
) (*ssh.Client, error) {
stepStart := time.Now()
client, err := pool.Get(ctx, target)
client, err := lifecycle.DialSSH(ctx, target)
step := TestStep{Name: "SSH handshake", DurationMs: time.Since(stepStart).Milliseconds()}
if err != nil {
step.Success = false
@@ -342,7 +341,7 @@ func (h *Handler) listSessions(ctx context.Context, executorID string) ([]Sessio
if run == nil {
continue
}
if run.Runtime != string(models.ExecutorTypeSSH) {
if run.Runtime != agentruntime.RuntimeSSH {
continue
}
if !h.sessionBelongsToExecutor(ctx, run, executorID) {
-1
View File
@@ -69,7 +69,6 @@ RUN apk add --no-cache \\
shadow \\
coreutils \\
ca-certificates \\
curl \\
iproute2 \\
iptables \\
# agentctl is built dynamically linked against glibc; Alpine is musl. gcompat