6a684211fc
* fix(agentctl): reap disconnected ACP sessions + MCP child trees (#1247) Two independent leaks let opencode acp sessions accumulate indefinitely: 1. instance.Manager never reaped disconnected sessions. StopInstance only ran via the kandev DELETE route, so any session whose backend went away (UI closed mid-run, WS dropped, opencode failed to emit complete) was orphaned forever. 2. Even when DELETE fired, the ACP adapter unconditionally returned RequiresProcessKill=false. opencode acp keeps an HTTP server + MCP child tree alive after stdin EOF, so closing stdin was a no-op and the children re-parented to init. Fix both: - Plumb RuntimeConfig.RequiresProcessKill from agents.OpenCodeACP through the ExecutorCreateRequest -> CreateInstanceRequest -> InstanceConfig -> adapter.Config -> shared.Config chain. acp.Adapter now reads cfg.RequiresProcessKill instead of hardcoding false. - Belt-and-braces: process.Manager.waitForProcessExit timeout fallback now killProcessGroup first (only falls back to single-process kill if that errors), so any future ACP CLI that ignores stdin EOF still gets reaped. - Add an idle-instance reaper to instance.Manager. Per-instance activity middleware tracks inflight HTTP requests + lastActivity; a background goroutine sweeps every minute and StopInstance's any instance idle past KANDEV_ACP_IDLE_TIMEOUT (default 1h, set to 0 to disable). Long-lived /agent/stream WS connections keep the inflight counter above zero so active sessions are never reaped. - Validate stdio MCP commands at CreateInstance time via exec.LookPath; missing entries are dropped with a warn instead of spawning a permanently broken child every session (fixes the stale /snap/bin/brave MCP repro in the issue). Regression tests: process-group kill (parent+child fixture), idle reaper end-to-end, MCP validation, opencode-acp opt-in vs other ACP agents' default. * fix: address PR review feedback on #1247 fix - idle_reaper: extract reapSingleIdleInstance helper with defer cancel(), add reaperStop polling between sweep iterations so Shutdown drains in one StopInstance window instead of N * timeout (cubic, claude, greptile). - instance.Manager: use sync.Once instead of m.mu to gate reaper-stop channel close (claude). - mcpStdioValidationError: tolerate compound Command strings by resolving only the first whitespace-delimited token; reject entries with neither URL nor Command outright; document agentctl-vs-agent PATH skew in Docker/SSH executor modes (claude, greptile, cubic). - manager_killgroup_test: treat zombie state as dead via /proc/<pid>/stat on Linux + bump Eventually window to 15s — CI containers' PID 1 can delay harvesting orphans long enough to flake signal(0) probes. - instance package: add goleak TestMain for the new reaper goroutine (coderabbit). - AGENTS.md: correct SIGKILL vs SIGTERM wording; clarify the timeout fallback only fires on ctx.Done (coderabbit, cubic). --------- Co-authored-by: Kandev Agent <agent@kandev.dev>
143 lines
4.9 KiB
Go
143 lines
4.9 KiB
Go
//go:build !windows
|
|
|
|
package process
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestWaitForProcessExit_KillsProcessGroupOnTimeout is the regression guard
|
|
// for GH issue #1247: when an agent subprocess does not exit on stdin close
|
|
// (opencode acp), the timeout fallback in waitForProcessExit used to call
|
|
// only cmd.Process.Kill(), leaving MCP children re-parented to init.
|
|
// After the fix, killProcessGroup is delivered to the leader's pgid so the
|
|
// whole tree dies.
|
|
func TestWaitForProcessExit_KillsProcessGroupOnTimeout(t *testing.T) {
|
|
log := newTestLogger(t)
|
|
pidFile := filepath.Join(t.TempDir(), "child.pid")
|
|
|
|
m := &Manager{
|
|
logger: log,
|
|
}
|
|
m.cmd = fixtureCmd("sleep-with-child " + pidFile + " 30")
|
|
setProcGroup(m.cmd)
|
|
require.NoError(t, m.cmd.Start())
|
|
t.Cleanup(func() {
|
|
// Best-effort: nuke the leader's pgid in case the test left the
|
|
// fixture alive (e.g. assertion failed before reaping).
|
|
_ = killProcessGroup(m.cmd.Process.Pid)
|
|
_, _ = m.cmd.Process.Wait()
|
|
})
|
|
|
|
// Wait for the child to be spawned and the pidfile to land. The fixture
|
|
// writes the PID before sleeping; bound the wait so a broken fixture
|
|
// can't hang the test.
|
|
childPID := waitForChildPID(t, pidFile, 5*time.Second)
|
|
|
|
// Sanity: the parent is in its own process group, and the child should
|
|
// have inherited it (no setpgid call inside the fixture).
|
|
parentPGID, err := syscall.Getpgid(m.cmd.Process.Pid)
|
|
require.NoError(t, err)
|
|
childPGID, err := syscall.Getpgid(childPID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, parentPGID, childPGID,
|
|
"child must inherit parent's pgid for the group-kill assumption to hold")
|
|
|
|
// Keep waitForProcessExit's internal wg.Wait() blocked so the select
|
|
// hits the ctx.Done branch (the path we're regression-testing). The
|
|
// real Start() path Add()s for readStderr/waitForExit; this test
|
|
// bypasses Start so we Add(1) ourselves and Done at the very end so
|
|
// goleak in TestMain stays clean.
|
|
m.wg.Add(1)
|
|
defer m.wg.Done()
|
|
|
|
// Drive the timeout fallback path: context is already done, so
|
|
// waitForProcessExit jumps straight to the kill branch.
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
m.waitForProcessExit(ctx)
|
|
|
|
// Both parent and child must be reaped within a short window. We waited
|
|
// the parent ourselves; for the child we treat both "process gone" and
|
|
// "process exists in zombie state" as success — in CI the orphan
|
|
// re-parents to a container's PID 1 (tini-style) that may take longer
|
|
// than the kill itself to harvest the zombie, but for our purposes the
|
|
// SIGKILL has already done its job once the kernel marks the task as
|
|
// dead. 15s window covers slow CI runners on Linux.
|
|
require.Eventually(t, func() bool {
|
|
return !processAlive(childPID)
|
|
}, 15*time.Second, 50*time.Millisecond,
|
|
"child process %d should be killed by process-group reap", childPID)
|
|
}
|
|
|
|
// waitForChildPID polls pidFile until it contains a valid PID or timeout
|
|
// expires. Returns the parsed PID. Fails the test on timeout.
|
|
func waitForChildPID(t *testing.T, pidFile string, timeout time.Duration) int {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for {
|
|
raw, err := os.ReadFile(pidFile)
|
|
if err == nil {
|
|
pid, perr := strconv.Atoi(strings.TrimSpace(string(raw)))
|
|
if perr == nil && pid > 0 {
|
|
return pid
|
|
}
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("timed out waiting for fixture to write child pid to %s", pidFile)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// processAlive reports whether a PID exists, is not a zombie, and can
|
|
// receive a signal. A zombie is treated as dead — the task has already
|
|
// been killed; we're just waiting for the parent (or init) to harvest the
|
|
// exit status, which is irrelevant to the regression we're testing for.
|
|
//
|
|
// On Linux we consult /proc/<pid>/stat to detect zombie state (field 3,
|
|
// `R` running, `S` sleeping, `D` uninterruptible, `Z` zombie, `T` stopped).
|
|
// On other Unix platforms we fall back to signal(0), which still catches
|
|
// the leak case in the bug report — opencode acp processes live with
|
|
// state 'S' on Linux until killed.
|
|
func processAlive(pid int) bool {
|
|
if pid <= 0 {
|
|
return false
|
|
}
|
|
if runtime.GOOS == "linux" {
|
|
raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
|
|
if err != nil {
|
|
return false // /proc entry gone => process gone
|
|
}
|
|
// Format: `<pid> (<comm>) <state> ...`. comm may contain spaces,
|
|
// so anchor on the trailing ")" before the state field.
|
|
s := string(raw)
|
|
idx := strings.LastIndex(s, ") ")
|
|
if idx == -1 || idx+2 >= len(s) {
|
|
return false
|
|
}
|
|
state := s[idx+2]
|
|
return state != 'Z'
|
|
}
|
|
proc, err := os.FindProcess(pid)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
// signal 0 doesn't deliver anything; it just checks whether the kernel
|
|
// could have delivered a signal — i.e. the process exists.
|
|
err = proc.Signal(syscall.Signal(0))
|
|
return err == nil
|
|
}
|