diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go index 3f311f16a7..3e699cac26 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go @@ -42,12 +42,11 @@ type sshSessionState struct { // SSHExecutor implements ExecutorBackend for SSH-reachable Linux hosts. // -// One SSH connection is shared per (host, user, identity) tuple across all -// sessions on that host. Each session gets its own agentctl process + remote -// port + local-forwarded port. See docs/specs/ssh-executor/spec.md for the -// full design. +// Each session owns its own *ssh.Client (no shared pool). One SSH connection +// per session keeps teardown simple — closing the executor instance closes +// the client — at the cost of an extra TCP+handshake per session on the same +// host. See docs/specs/ssh-executor/spec.md for the full design. type SSHExecutor struct { - pool *SSHConnPool agentctlResolver *AgentctlResolver secretStore secrets.SecretStore agentList RemoteAuthAgentLister @@ -66,7 +65,6 @@ func NewSSHExecutor( log *logger.Logger, ) *SSHExecutor { return &SSHExecutor{ - pool: NewSSHConnPool(log), agentctlResolver: resolver, secretStore: secretStore, agentList: agentList, @@ -84,11 +82,10 @@ func (r *SSHExecutor) HealthCheck(_ context.Context) error { return nil } -// Close terminates pooled SSH connections at shutdown. -func (r *SSHExecutor) Close() error { - r.pool.CloseAll() - return nil -} +// Close is a no-op: per-session SSH clients are owned by sshSessionState and +// torn down by StopInstance. Kept on the type so the executor satisfies +// io.Closer alongside the other backends. +func (r *SSHExecutor) Close() error { return nil } // targetFromMetadata builds an SSHTarget from the executor metadata // propagated via buildLaunchMetadata (req.ExecutorConfig keys merged in). @@ -461,34 +458,21 @@ func (r *SSHExecutor) IsAlwaysResumable() bool { return true } // creation. SSH-specific tweaks: remote auth target dir defaults to the SSH // user's home (resolved at runtime via a tiny `pwd` probe), and the file // uploader writes via SFTP. -func (r *SSHExecutor) maybeUploadCredentials(ctx context.Context, client *ssh.Client, req *ExecutorCreateRequest) { - credsJSON := getMetadataString(req.Metadata, "remote_credentials") - if credsJSON == "" { +// +// v1 ships without remote-credentials upload — the field is accepted on the +// executor profile (parity with Sprites/Docker) but the upload pipeline lives +// in the Sprites executor and is not yet ported over. A non-empty +// remote_credentials log is the only side effect so users notice they +// configured a feature that isn't active yet. +func (r *SSHExecutor) maybeUploadCredentials(_ context.Context, _ *ssh.Client, req *ExecutorCreateRequest) { + if getMetadataString(req.Metadata, "remote_credentials") == "" { return } - // Resolve remote $HOME for the target auth dir, unless the profile pinned one. - if getMetadataString(req.Metadata, MetadataKeyRemoteAuthHome) == "" { - if _, _, err := runSSHCommand(ctx, client, "printf %s \"$HOME\""); err != nil { - r.logger.Warn("ssh credentials: could not resolve remote $HOME, skipping upload", zap.Error(err)) - return - } - } - // Reuse the catalog + selection logic from the credential-uploader. - // We intentionally skip the env-secret resolution + auth setup scripts here - // (those live in the Sprites executor and depend on agent install scripts); - // v1 SSH just pushes the configured credential files. - if r.agentList == nil { - return - } - // The credential-upload pipeline is exercised end-to-end by Sprites; for - // SSH v1 we wire the FileUploader and let the caller select methods via - // the existing profile-config "remote_credentials" key. UploadCredentialFiles - // itself reads the local files and writes them through SFTP. - uploader := newSSHFileUploader(client) - // We do not parse the methods here; the more elaborate selection is the - // SpritesExecutor's job. For now, write a marker file to confirm SFTP - // works and surface the credentials JSON for follow-up integration. - _ = uploader // referenced for future expansion; intentional no-op in v1 + r.logger.Warn( + "ssh executor: remote_credentials is configured but credential upload is not yet implemented for the SSH runtime; see docs/specs/ssh-executor/spec.md", + zap.String("task_id", req.TaskID), + zap.String("session_id", req.SessionID), + ) } // sshTaskDirName builds a stable per-task remote directory name. Prefers an diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go index b6ec8b1f57..0e71613813 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_connection.go @@ -9,22 +9,14 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/kevinburke/ssh_config" - "go.uber.org/zap" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - - "github.com/kandev/kandev/internal/common/logger" ) -const ( - sshDialTimeout = 30 * time.Second - sshKeepaliveEvery = 30 * time.Second - sshKeepaliveTimeout = 15 * time.Second -) +const sshDialTimeout = 30 * time.Second // SSHIdentitySource enumerates how kandev obtains an SSH credential. type SSHIdentitySource string @@ -307,6 +299,15 @@ func buildClientConfig(target *SSHTarget) (*ssh.ClientConfig, error) { }, nil } +// DialSSH opens an SSH connection to target, optionally via a bastion if +// ProxyJump is set. Returns the live *ssh.Client and the observed fingerprint +// (recorded on target.ObservedFingerprint via the host-key callback). +// +// Callers own the client and must Close it when done. +func DialSSH(ctx context.Context, target *SSHTarget) (*ssh.Client, error) { + return dialSSH(ctx, target) +} + // dial opens an SSH connection to target, optionally via a bastion if ProxyJump // is set. Returns the live *ssh.Client and the observed fingerprint. func dialSSH(ctx context.Context, target *SSHTarget) (*ssh.Client, error) { @@ -437,161 +438,8 @@ func parseLiteralProxyJump(s string) (user, host string, port int, ok bool) { return user, host, port, true } -// sshConnKey identifies a pooled SSH connection. Connections are not shared -// across identity sources or proxy-jump configurations to avoid surprising -// behavior when a user changes auth methods mid-session. -type sshConnKey struct { - host string - port int - user string - identitySource SSHIdentitySource - identityFile string - proxyJump string -} - -func keyForTarget(target *SSHTarget) sshConnKey { - return sshConnKey{ - host: target.Host, - port: target.Port, - user: target.User, - identitySource: target.IdentitySource, - identityFile: target.IdentityFile, - proxyJump: target.ProxyJump, - } -} - -// pooledConn is a refcounted live SSH client shared across an executor's -// sessions on the same host. -type pooledConn struct { - client *ssh.Client - target *SSHTarget - refs int - keepalive context.CancelFunc -} - -// SSHConnPool multiplexes a single SSH client per (host, user, identity) tuple -// across all of the executor's live sessions on that host. Sessions acquire a -// connection via Get; the pool releases the underlying client when the last -// session releases. -type SSHConnPool struct { - mu sync.Mutex - entries map[sshConnKey]*pooledConn - logger *logger.Logger -} - -func NewSSHConnPool(log *logger.Logger) *SSHConnPool { - return &SSHConnPool{ - entries: make(map[sshConnKey]*pooledConn), - logger: log.WithFields(zap.String("component", "ssh-conn-pool")), - } -} - -// Get returns a live *ssh.Client for the target. The pool keeps the -// underlying ssh.Client per (host, user, identity, …) so multiple sessions -// against the same host share a single SSH transport — channels are cheap, -// connections are not. -// -// NOTE: golang.org/x/crypto/ssh's Client.Conn carries a mutex around the -// channel/mux state. Several short-lived Dial calls from concurrent goroutines -// against the same Client occasionally race in a way that surfaces as -// `io.EOF` on the OpenChannel response, with no corresponding rejection from -// the server. To keep the e2e + production paths reliable, the pool dials a -// fresh connection per executor call and releases it back to the OS at the -// end of the session. Reusing one Client across N concurrent direct-tcpip -// channels remains a perf improvement we can revisit when we have a clean -// repro for the race. -func (p *SSHConnPool) Get(ctx context.Context, target *SSHTarget) (*ssh.Client, error) { - key := keyForTarget(target) - - client, err := dialSSH(ctx, target) - if err != nil { - return nil, err - } - - p.mu.Lock() - defer p.mu.Unlock() - kctx, kcancel := context.WithCancel(context.Background()) - entry := &pooledConn{client: client, target: target, refs: 1, keepalive: kcancel} - // Multiple sessions against the same target get distinct entries; the map - // key still keys by tuple but acts as "any one matching" — Release walks - // the slice via refcount semantics, see Release(). - p.entries[key] = entry - go p.runKeepalive(kctx, key, client) - return client, nil -} - -// Release decrements the refcount; when it hits zero the underlying client is -// closed and removed from the pool. -func (p *SSHConnPool) Release(target *SSHTarget) { - key := keyForTarget(target) - p.mu.Lock() - defer p.mu.Unlock() - entry, ok := p.entries[key] - if !ok { - return - } - entry.refs-- - if entry.refs > 0 { - return - } - entry.keepalive() - _ = entry.client.Close() - delete(p.entries, key) -} - -// CloseAll terminates every pooled connection. Used at shutdown. -func (p *SSHConnPool) CloseAll() { - p.mu.Lock() - defer p.mu.Unlock() - for k, entry := range p.entries { - entry.keepalive() - _ = entry.client.Close() - delete(p.entries, k) - } -} - -// runKeepalive sends an ssh keepalive every sshKeepaliveEvery so dropped -// connections surface within ~30s instead of waiting for the next operation. -// On any keepalive failure the pool entry is evicted so the next Get will -// re-dial. -func (p *SSHConnPool) runKeepalive(ctx context.Context, key sshConnKey, client *ssh.Client) { - t := time.NewTicker(sshKeepaliveEvery) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return - case <-t.C: - done := make(chan error, 1) - go func() { - _, _, err := client.SendRequest("keepalive@kandev", true, nil) - done <- err - }() - select { - case err := <-done: - if err != nil { - p.evictOnFailure(key, err) - return - } - case <-time.After(sshKeepaliveTimeout): - p.evictOnFailure(key, fmt.Errorf("keepalive timed out")) - return - } - } - } -} - -func (p *SSHConnPool) evictOnFailure(key sshConnKey, err error) { - p.mu.Lock() - defer p.mu.Unlock() - entry, ok := p.entries[key] - if !ok { - return - } - p.logger.Warn("ssh connection lost — evicting from pool", - zap.String("host", key.host), - zap.String("user", key.user), - zap.Error(err)) - _ = entry.client.Close() - delete(p.entries, key) -} +// SSH connections are owned per-session (no shared pool): a session's client +// lives on sshSessionState and is Close()d by StopInstance. The earlier +// pool/refcount/keepalive plumbing was dead code — the executor never used it +// in production, and the orphaned keepalive goroutines surfaced under e2e +// fault-injection. See PR #927 for the removal rationale. diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials.go deleted file mode 100644 index a864d2b70c..0000000000 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials.go +++ /dev/null @@ -1,23 +0,0 @@ -package lifecycle - -import ( - "context" - "os" - - "golang.org/x/crypto/ssh" -) - -// sshFileUploader implements FileUploader for an SSH client by writing through -// SFTP. Reused by the existing credential-upload pipeline so SSH executor gets -// gh_cli_token / claude / codex / etc. credential propagation for free. -type sshFileUploader struct { - client *ssh.Client -} - -func newSSHFileUploader(client *ssh.Client) *sshFileUploader { - return &sshFileUploader{client: client} -} - -func (u *sshFileUploader) WriteFile(_ context.Context, path string, data []byte, mode os.FileMode) error { - return sftpWriteFile(u.client, path, data, mode) -} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go index dc2b646b6f..e5eeba2fb8 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go @@ -9,7 +9,9 @@ import ( "errors" "fmt" "io" + mrand "math/rand/v2" "net" + "net/http" "os" "path/filepath" "strconv" @@ -21,7 +23,7 @@ import ( "go.uber.org/zap" "golang.org/x/crypto/ssh" - agentctl "github.com/kandev/kandev/internal/agentctl/client" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/common/logger" ) @@ -261,17 +263,6 @@ func sftpUploadBytes(client *ssh.Client, remotePath string, data []byte, mode os return nil } -// sftpWriteFile is the FileUploader.WriteFile implementation: ensures parent -// dirs exist and atomically writes the file. -func sftpWriteFile(client *ssh.Client, remotePath string, data []byte, mode os.FileMode) error { - if dir := filepath.Dir(remotePath); dir != "" && dir != "." { - if _, _, err := runSSHCommand(context.Background(), client, "mkdir -p "+shellQuote(dir)); err != nil { - return fmt.Errorf("ssh: mkdir %s: %w", dir, err) - } - } - return sftpUploadBytes(client, remotePath, data, mode) -} - // shellQuote is a minimal POSIX shell-safe single-quote wrapper. func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" @@ -391,11 +382,11 @@ func startRemoteAgentctl( const sshAgentctlLogTailLines = 25 // createRemoteAgentInstance creates a per-session agent instance on the -// remote agentctl control server by POSTing to /api/v1/instances from inside -// the container (so we don't need a second SSH port forward just to call the -// control API). Returns the per-instance port the SSH executor should later -// forward + dial for ACP / workspace traffic. Mirrors what -// executor_sprites.go does inside its sprite. +// remote agentctl control server by POSTing to /api/v1/instances over a +// direct-tcpip channel through the existing SSH client — no second port +// forward, and no dependency on remote curl. Returns the per-instance port +// the SSH executor should later forward + dial for ACP / workspace traffic. +// Mirrors what executor_sprites.go does inside its sprite. func createRemoteAgentInstance( ctx context.Context, client *ssh.Client, @@ -416,39 +407,47 @@ func createRemoteAgentInstance( if err != nil { return 0, fmt.Errorf("ssh: marshal create-instance: %w", err) } - // Pipe the JSON body via curl's --data-binary @- so the shell doesn't have - // to escape it. -sS to suppress progress but keep errors. --fail returns - // non-zero on HTTP >= 400 so we get a clear error code. - cmd := fmt.Sprintf( - "curl -sS --fail -X POST -H 'Content-Type: application/json' --data-binary @- http://127.0.0.1:%d/api/v1/instances", - controlPort, - ) - session, err := client.NewSession() + + // HTTP-over-direct-tcpip: every request dials a fresh SSH channel to the + // remote control port. Keep-alives are disabled so the channel closes + // after the response. + httpClient := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return client.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(controlPort))) + }, + DisableKeepAlives: true, + }, + } + + url := fmt.Sprintf("http://127.0.0.1:%d/api/v1/instances", controlPort) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) if err != nil { - return 0, fmt.Errorf("ssh: new session: %w", err) + return 0, fmt.Errorf("ssh: build create-instance request: %w", err) } - defer func() { _ = session.Close() }() - session.Stdin = bytes.NewReader(body) - var stdout, stderr bytes.Buffer - session.Stdout = &stdout - session.Stderr = &stderr - done := make(chan error, 1) - go func() { done <- session.Run(cmd) }() - select { - case <-ctx.Done(): - _ = session.Signal(ssh.SIGTERM) - return 0, ctx.Err() - case rerr := <-done: - if rerr != nil { - return 0, fmt.Errorf("ssh: create-instance curl: %w (stderr: %s)", rerr, strings.TrimSpace(stderr.String())) - } + httpReq.Header.Set("Content-Type", "application/json") + + httpResp, err := httpClient.Do(httpReq) + if err != nil { + return 0, fmt.Errorf("ssh: create-instance dial: %w", err) } + defer func() { _ = httpResp.Body.Close() }() + + respBody, err := io.ReadAll(httpResp.Body) + if err != nil { + return 0, fmt.Errorf("ssh: read create-instance response: %w", err) + } + if httpResp.StatusCode >= http.StatusBadRequest { + return 0, fmt.Errorf("ssh: create-instance returned %d: %s", httpResp.StatusCode, strings.TrimSpace(string(respBody))) + } + var resp agentctl.CreateInstanceResponse - if err := json.Unmarshal(stdout.Bytes(), &resp); err != nil { - return 0, fmt.Errorf("ssh: parse create-instance response: %w (body: %s)", err, stdout.String()) + if err := json.Unmarshal(respBody, &resp); err != nil { + return 0, fmt.Errorf("ssh: parse create-instance response: %w (body: %s)", err, string(respBody)) } if resp.Port == 0 { - return 0, fmt.Errorf("ssh: create-instance returned port 0 (body: %s)", stdout.String()) + return 0, fmt.Errorf("ssh: create-instance returned port 0 (body: %s)", string(respBody)) } log.Info("created remote agent instance", zap.Int("control_port", controlPort), @@ -467,14 +466,13 @@ func sshAgentTypeFromReq(req *ExecutorCreateRequest) string { } // pickRemoteAgentctlPort returns a port in [40000, 60000). The kandev backend -// picks per session; agentctl honors AGENTCTL_PORT. Collisions inside one sshd -// container would surface as a clear "failed to bind" error and a quick retry -// fixes it — in practice for e2e and a single SSH user, collisions are -// vanishingly unlikely. -// -//nolint:gosec // not used for cryptographic purposes +// picks per session; agentctl honors AGENTCTL_PORT. Uses math/rand/v2 so two +// concurrent CreateInstance calls don't collide (UnixNano%20000 cycles in +// ~20µs, which is well within the window between back-to-back launches on a +// fast machine). A residual collision still surfaces as a clear bind failure +// and the caller can retry. func pickRemoteAgentctlPort() int { - return 40000 + int(time.Now().UnixNano())%20000 + return 40000 + mrand.IntN(20000) } // stopRemoteAgentctl best-effort kills a remote agentctl by PID and removes diff --git a/apps/backend/internal/ssh/handlers.go b/apps/backend/internal/ssh/handlers.go index 87e0e0bf2e..356b2b3684 100644 --- a/apps/backend/internal/ssh/handlers.go +++ b/apps/backend/internal/ssh/handlers.go @@ -15,7 +15,8 @@ import ( "go.uber.org/zap" "golang.org/x/crypto/ssh" - "github.com/kandev/kandev/internal/agent/lifecycle" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/task/models" ws "github.com/kandev/kandev/pkg/websocket" @@ -192,13 +193,11 @@ func (h *Handler) runTest(ctx context.Context, req TestRequest) *TestResult { return finalize() } - pool := lifecycle.NewSSHConnPool(h.logger) - defer pool.CloseAll() - - client, err := h.testHandshake(ctx, pool, target, result) + client, err := h.testHandshake(ctx, target, result) if err != nil { return finalize() } + defer func() { _ = client.Close() }() infoCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -237,10 +236,10 @@ func (h *Handler) testResolveTarget(req TestRequest, result *TestResult) (*lifec } func (h *Handler) testHandshake( - ctx context.Context, pool *lifecycle.SSHConnPool, target *lifecycle.SSHTarget, result *TestResult, + ctx context.Context, target *lifecycle.SSHTarget, result *TestResult, ) (*ssh.Client, error) { stepStart := time.Now() - client, err := pool.Get(ctx, target) + client, err := lifecycle.DialSSH(ctx, target) step := TestStep{Name: "SSH handshake", DurationMs: time.Since(stepStart).Milliseconds()} if err != nil { step.Success = false @@ -342,7 +341,7 @@ func (h *Handler) listSessions(ctx context.Context, executorID string) ([]Sessio if run == nil { continue } - if run.Runtime != string(models.ExecutorTypeSSH) { + if run.Runtime != agentruntime.RuntimeSSH { continue } if !h.sessionBelongsToExecutor(ctx, run, executorID) { diff --git a/apps/web/e2e/fixtures/ssh-image.ts b/apps/web/e2e/fixtures/ssh-image.ts index f6af015e3a..d561c9f2b0 100644 --- a/apps/web/e2e/fixtures/ssh-image.ts +++ b/apps/web/e2e/fixtures/ssh-image.ts @@ -69,7 +69,6 @@ RUN apk add --no-cache \\ shadow \\ coreutils \\ ca-certificates \\ - curl \\ iproute2 \\ iptables \\ # agentctl is built dynamically linked against glibc; Alpine is musl. gcompat