Files
kandev/apps/backend/internal/agentctl/server/process/manager_killgroup_test.go
T
João Salavisa 6a684211fc fix(agentctl): reap disconnected ACP sessions + MCP child trees (#1249)
* fix(agentctl): reap disconnected ACP sessions + MCP child trees (#1247)

Two independent leaks let opencode acp sessions accumulate indefinitely:

1. instance.Manager never reaped disconnected sessions. StopInstance
   only ran via the kandev DELETE route, so any session whose backend
   went away (UI closed mid-run, WS dropped, opencode failed to emit
   complete) was orphaned forever.
2. Even when DELETE fired, the ACP adapter unconditionally returned
   RequiresProcessKill=false. opencode acp keeps an HTTP server + MCP
   child tree alive after stdin EOF, so closing stdin was a no-op and
   the children re-parented to init.

Fix both:

- Plumb RuntimeConfig.RequiresProcessKill from agents.OpenCodeACP
  through the ExecutorCreateRequest -> CreateInstanceRequest ->
  InstanceConfig -> adapter.Config -> shared.Config chain. acp.Adapter
  now reads cfg.RequiresProcessKill instead of hardcoding false.
- Belt-and-braces: process.Manager.waitForProcessExit timeout fallback
  now killProcessGroup first (only falls back to single-process kill
  if that errors), so any future ACP CLI that ignores stdin EOF still
  gets reaped.
- Add an idle-instance reaper to instance.Manager. Per-instance
  activity middleware tracks inflight HTTP requests + lastActivity;
  a background goroutine sweeps every minute and StopInstance's any
  instance idle past KANDEV_ACP_IDLE_TIMEOUT (default 1h, set to 0
  to disable). Long-lived /agent/stream WS connections keep the
  inflight counter above zero so active sessions are never reaped.
- Validate stdio MCP commands at CreateInstance time via
  exec.LookPath; missing entries are dropped with a warn instead of
  spawning a permanently broken child every session (fixes the
  stale /snap/bin/brave MCP repro in the issue).

Regression tests: process-group kill (parent+child fixture), idle
reaper end-to-end, MCP validation, opencode-acp opt-in vs other
ACP agents' default.

* fix: address PR review feedback on #1247 fix

- idle_reaper: extract reapSingleIdleInstance helper with defer cancel(),
  add reaperStop polling between sweep iterations so Shutdown drains in
  one StopInstance window instead of N * timeout (cubic, claude, greptile).
- instance.Manager: use sync.Once instead of m.mu to gate reaper-stop
  channel close (claude).
- mcpStdioValidationError: tolerate compound Command strings by
  resolving only the first whitespace-delimited token; reject entries
  with neither URL nor Command outright; document agentctl-vs-agent
  PATH skew in Docker/SSH executor modes (claude, greptile, cubic).
- manager_killgroup_test: treat zombie state as dead via /proc/<pid>/stat
  on Linux + bump Eventually window to 15s — CI containers' PID 1 can
  delay harvesting orphans long enough to flake signal(0) probes.
- instance package: add goleak TestMain for the new reaper goroutine
  (coderabbit).
- AGENTS.md: correct SIGKILL vs SIGTERM wording; clarify the timeout
  fallback only fires on ctx.Done (coderabbit, cubic).

---------

Co-authored-by: Kandev Agent <agent@kandev.dev>
2026-06-02 21:46:49 +01:00

143 lines
4.9 KiB
Go

//go:build !windows
package process
import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestWaitForProcessExit_KillsProcessGroupOnTimeout is the regression guard
// for GH issue #1247: when an agent subprocess does not exit on stdin close
// (opencode acp), the timeout fallback in waitForProcessExit used to call
// only cmd.Process.Kill(), leaving MCP children re-parented to init.
// After the fix, killProcessGroup is delivered to the leader's pgid so the
// whole tree dies.
func TestWaitForProcessExit_KillsProcessGroupOnTimeout(t *testing.T) {
log := newTestLogger(t)
pidFile := filepath.Join(t.TempDir(), "child.pid")
m := &Manager{
logger: log,
}
m.cmd = fixtureCmd("sleep-with-child " + pidFile + " 30")
setProcGroup(m.cmd)
require.NoError(t, m.cmd.Start())
t.Cleanup(func() {
// Best-effort: nuke the leader's pgid in case the test left the
// fixture alive (e.g. assertion failed before reaping).
_ = killProcessGroup(m.cmd.Process.Pid)
_, _ = m.cmd.Process.Wait()
})
// Wait for the child to be spawned and the pidfile to land. The fixture
// writes the PID before sleeping; bound the wait so a broken fixture
// can't hang the test.
childPID := waitForChildPID(t, pidFile, 5*time.Second)
// Sanity: the parent is in its own process group, and the child should
// have inherited it (no setpgid call inside the fixture).
parentPGID, err := syscall.Getpgid(m.cmd.Process.Pid)
require.NoError(t, err)
childPGID, err := syscall.Getpgid(childPID)
require.NoError(t, err)
require.Equal(t, parentPGID, childPGID,
"child must inherit parent's pgid for the group-kill assumption to hold")
// Keep waitForProcessExit's internal wg.Wait() blocked so the select
// hits the ctx.Done branch (the path we're regression-testing). The
// real Start() path Add()s for readStderr/waitForExit; this test
// bypasses Start so we Add(1) ourselves and Done at the very end so
// goleak in TestMain stays clean.
m.wg.Add(1)
defer m.wg.Done()
// Drive the timeout fallback path: context is already done, so
// waitForProcessExit jumps straight to the kill branch.
ctx, cancel := context.WithCancel(context.Background())
cancel()
m.waitForProcessExit(ctx)
// Both parent and child must be reaped within a short window. We waited
// the parent ourselves; for the child we treat both "process gone" and
// "process exists in zombie state" as success — in CI the orphan
// re-parents to a container's PID 1 (tini-style) that may take longer
// than the kill itself to harvest the zombie, but for our purposes the
// SIGKILL has already done its job once the kernel marks the task as
// dead. 15s window covers slow CI runners on Linux.
require.Eventually(t, func() bool {
return !processAlive(childPID)
}, 15*time.Second, 50*time.Millisecond,
"child process %d should be killed by process-group reap", childPID)
}
// waitForChildPID polls pidFile until it contains a valid PID or timeout
// expires. Returns the parsed PID. Fails the test on timeout.
func waitForChildPID(t *testing.T, pidFile string, timeout time.Duration) int {
t.Helper()
deadline := time.Now().Add(timeout)
for {
raw, err := os.ReadFile(pidFile)
if err == nil {
pid, perr := strconv.Atoi(strings.TrimSpace(string(raw)))
if perr == nil && pid > 0 {
return pid
}
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for fixture to write child pid to %s", pidFile)
}
time.Sleep(20 * time.Millisecond)
}
}
// processAlive reports whether a PID exists, is not a zombie, and can
// receive a signal. A zombie is treated as dead — the task has already
// been killed; we're just waiting for the parent (or init) to harvest the
// exit status, which is irrelevant to the regression we're testing for.
//
// On Linux we consult /proc/<pid>/stat to detect zombie state (field 3,
// `R` running, `S` sleeping, `D` uninterruptible, `Z` zombie, `T` stopped).
// On other Unix platforms we fall back to signal(0), which still catches
// the leak case in the bug report — opencode acp processes live with
// state 'S' on Linux until killed.
func processAlive(pid int) bool {
if pid <= 0 {
return false
}
if runtime.GOOS == "linux" {
raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
if err != nil {
return false // /proc entry gone => process gone
}
// Format: `<pid> (<comm>) <state> ...`. comm may contain spaces,
// so anchor on the trailing ")" before the state field.
s := string(raw)
idx := strings.LastIndex(s, ") ")
if idx == -1 || idx+2 >= len(s) {
return false
}
state := s[idx+2]
return state != 'Z'
}
proc, err := os.FindProcess(pid)
if err != nil {
return false
}
// signal 0 doesn't deliver anything; it just checks whether the kernel
// could have delivered a signal — i.e. the process exists.
err = proc.Signal(syscall.Signal(0))
return err == nil
}