package launcher import ( "errors" "fmt" "io" "os" "os/exec" "os/signal" "sync" "sync/atomic" "syscall" "time" ) const capturedOutputLimit = 64 * 1024 // Keep this above the backend's serial graceful-shutdown budgets so the // launcher does not preempt agent/session cleanup that is still in progress. const managedProcessShutdownGrace = 75 * time.Second const managedProcessForceKillWait = 2 * time.Second var launcherShutdownDebug atomic.Bool var launcherStatusOutput io.Writer = os.Stderr var launcherExit = os.Exit type processSupervisor struct { mu sync.Mutex children []*managedProcess shutdownOnce sync.Once } type managedProcess struct { label string cmd *exec.Cmd exitCode int exited bool mu sync.Mutex done chan struct{} } type managedProcessShutdownResult struct { label string pid int duration time.Duration graceful bool forceKilled bool err error } type shutdownSummary struct { graceful int forceKilled int failed int } func newSupervisor() *processSupervisor { return &processSupervisor{} } func setLauncherShutdownDebug(enabled bool) { launcherShutdownDebug.Store(enabled) } func shutdownDebugf(format string, args ...interface{}) { if !launcherShutdownDebug.Load() { return } fmt.Fprintf(os.Stderr, "[kandev] [SHUTDOWN-DEBUG] "+format+"\n", args...) } func launcherInfof(format string, args ...interface{}) { _, _ = fmt.Fprintf(launcherStatusOutput, "[kandev] "+format+"\n", args...) } func (s *processSupervisor) add(proc *managedProcess) { s.mu.Lock() defer s.mu.Unlock() s.children = append(s.children, proc) shutdownDebugf("supervisor add child pid=%d total_children=%d", proc.cmd.Process.Pid, len(s.children)) } func (s *processSupervisor) shutdown(reason string) { s.shutdownOnce.Do(func() { s.runShutdown(reason) }) } func (s *processSupervisor) runShutdown(reason string) { s.mu.Lock() children := append([]*managedProcess(nil), s.children...) s.mu.Unlock() start := time.Now() launcherInfof("graceful shutdown started (reason=%s, timeout=%s, processes=%d)", reason, managedProcessShutdownGrace, len(children)) shutdownDebugf("launcher shutdown begin reason=%q launcher_pid=%d children=%d", reason, os.Getpid(), len(children)) results := make([]managedProcessShutdownResult, len(children)) var wg sync.WaitGroup for i, child := range children { wg.Add(1) go func(i int, child *managedProcess) { defer wg.Done() results[i] = child.kill() }(i, child) } wg.Wait() logShutdownComplete(time.Since(start), results) shutdownDebugf("launcher shutdown complete reason=%q", reason) } func (s *processSupervisor) forceKillAll(reason string) []managedProcessShutdownResult { s.mu.Lock() children := append([]*managedProcess(nil), s.children...) s.mu.Unlock() shutdownDebugf("launcher force kill begin reason=%q launcher_pid=%d children=%d", reason, os.Getpid(), len(children)) results := make([]managedProcessShutdownResult, len(children)) var wg sync.WaitGroup for i, child := range children { wg.Add(1) go func(i int, child *managedProcess) { defer wg.Done() results[i] = child.forceKill(reason) }(i, child) } wg.Wait() shutdownDebugf("launcher force kill complete reason=%q", reason) return results } func (s *processSupervisor) attachSignals() { ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) shutdownDebugf("launcher signal handler armed launcher_pid=%d", os.Getpid()) go func() { sig := <-ch shutdownDebugf("launcher received signal=%s launcher_pid=%d", sig.String(), os.Getpid()) shutdownDone := make(chan struct{}) go func() { s.shutdown("signal " + sig.String()) close(shutdownDone) }() select { case nextSig := <-ch: shutdownDebugf("launcher received signal during shutdown=%s launcher_pid=%d", nextSig.String(), os.Getpid()) forceStart := time.Now() launcherInfof("forced shutdown after second signal (signal=%s)", nextSig.String()) results := s.forceKillAll("second signal " + nextSig.String()) logForcedShutdownComplete(time.Since(forceStart), results) signal.Stop(ch) launcherExit(1) case <-shutdownDone: signal.Stop(ch) launcherExit(0) } }() } func startProcess(command string, args []string, cwd string, env []string, quiet bool, label string, supervisor *processSupervisor) (*managedProcess, func(), error) { cmd := exec.Command(command, args...) cmd.Dir = cwd cmd.Env = env cmd.Stdin = nil configureManagedProcess(cmd) stdout := newLimitedBuffer(capturedOutputLimit) var stdoutSink io.Writer if !quiet { stdoutSink = os.Stdout } cmd.Stdout = newProcessOutput(stdout, stdoutSink, os.Stderr, label+".stdout") cmd.Stderr = newProcessOutput(nil, os.Stderr, nil, label+".stderr") supervisor.mu.Lock() if err := cmd.Start(); err != nil { supervisor.mu.Unlock() return nil, nil, err } proc := &managedProcess{label: label, cmd: cmd, done: make(chan struct{})} supervisor.children = append(supervisor.children, proc) childCount := len(supervisor.children) supervisor.mu.Unlock() shutdownDebugf("managed process started label=%q pid=%d command=%q args=%q cwd=%q", label, cmd.Process.Pid, command, args, cwd) shutdownDebugf("supervisor add child pid=%d total_children=%d", proc.cmd.Process.Pid, childCount) go func() { err := cmd.Wait() code := 0 if err != nil { code = 1 if exitErr, ok := err.(*exec.ExitError); ok { code = exitErr.ExitCode() } } state := "" if cmd.ProcessState != nil { state = cmd.ProcessState.String() } shutdownDebugf("managed process wait complete label=%q pid=%d code=%d state=%q err=%v", label, cmd.Process.Pid, code, state, err) proc.mu.Lock() proc.exitCode = code proc.exited = true proc.mu.Unlock() close(proc.done) if label != "" && code != 0 { fmt.Fprintf(os.Stderr, "[kandev] %s exited (code=%d)\n", label, code) } }() return proc, func() { snapshot := stdout.Bytes() if len(snapshot) == 0 { return } fmt.Fprintln(os.Stderr, "[kandev] --- backend stdout (last captured output) ---") _, _ = os.Stderr.Write(snapshot) fmt.Fprintln(os.Stderr, "[kandev] --- end backend stdout ---") }, nil } type limitedBuffer struct { mu sync.Mutex limit int buf []byte } type processOutput struct { mu sync.Mutex buffer *limitedBuffer sink io.Writer fallbackSink io.Writer label string sinkDisabled bool } func newProcessOutput(buffer *limitedBuffer, sink io.Writer, fallbackSink io.Writer, label string) *processOutput { return &processOutput{buffer: buffer, sink: sink, fallbackSink: fallbackSink, label: label} } func (w *processOutput) Write(p []byte) (int, error) { if w.buffer != nil { _, _ = w.buffer.Write(p) } w.mu.Lock() defer w.mu.Unlock() if w.sink == nil || w.sinkDisabled { return len(p), nil } if _, err := w.sink.Write(p); err != nil { if w.fallbackSink != nil && w.fallbackSink != w.sink { shutdownDebugf("switching broken output sink to fallback label=%q err=%v", w.label, err) w.sink = w.fallbackSink w.fallbackSink = nil var fallbackErr error if _, fallbackErr = w.sink.Write(p); fallbackErr == nil { return len(p), nil } err = fallbackErr } w.sinkDisabled = true shutdownDebugf("disabling broken output sink label=%q err=%v", w.label, err) launcherInfof("warning: disabling %s output sink after write failure: %v", w.label, err) } return len(p), nil } func newLimitedBuffer(limit int) *limitedBuffer { return &limitedBuffer{limit: limit} } func (b *limitedBuffer) Write(p []byte) (int, error) { b.mu.Lock() defer b.mu.Unlock() b.buf = append(b.buf, p...) if len(b.buf) > b.limit { b.buf = append([]byte(nil), b.buf[len(b.buf)-b.limit:]...) } return len(p), nil } func (b *limitedBuffer) Bytes() []byte { b.mu.Lock() defer b.mu.Unlock() return append([]byte(nil), b.buf...) } func (p *managedProcess) Exited() (bool, int) { p.mu.Lock() defer p.mu.Unlock() return p.exited, p.exitCode } func (p *managedProcess) kill() managedProcessShutdownResult { start := time.Now() result := managedProcessShutdownResult{label: p.label} if p.cmd.Process == nil { shutdownDebugf("managed process kill skipped: process nil") result.err = fmt.Errorf("process is nil") return result } pid := p.cmd.Process.Pid result.pid = pid select { case <-p.done: result.duration = time.Since(start) result.graceful = true shutdownDebugf("managed process kill skipped; already exited label=%q pid=%d", p.label, pid) return result default: } shutdownDebugf("managed process kill begin label=%q pid=%d grace=%s", p.label, pid, managedProcessShutdownGrace) shutdownDebugf("managed process group SIGTERM requested label=%q pgid=%d", p.label, pid) if err := terminateManagedProcessGroup(pid); err != nil { if errors.Is(err, syscall.ESRCH) { result.duration = time.Since(start) result.graceful = true shutdownDebugf("managed process group already gone label=%q pid=%d", p.label, pid) return result } shutdownDebugf("managed process group SIGTERM failed pid=%d err=%v; killing process", pid, err) shutdownDebugf("managed process SIGKILL requested label=%q pid=%d reason=%q", p.label, pid, "sigterm_failed") _ = p.cmd.Process.Kill() result.forceKilled = true result.err = err if waitForManagedProcessKillDone(p.done, managedProcessForceKillWait) { shutdownDebugf("managed process killed after SIGTERM failure pid=%d", pid) } else { result.err = errors.Join(result.err, fmt.Errorf("timed out waiting for process %d after SIGKILL", pid)) shutdownDebugf("managed process kill wait timed out after SIGTERM failure pid=%d", pid) } result.duration = time.Since(start) return result } shutdownDebugf("managed process group SIGTERM sent pgid=%d", pid) select { case <-p.done: result.duration = time.Since(start) result.graceful = true shutdownDebugf("managed process exited within grace pid=%d", pid) return result case <-time.After(managedProcessShutdownGrace): } shutdownDebugf("managed process grace expired; sending SIGKILL pgid=%d", pid) result.forceKilled = true if err := killManagedProcessGroup(pid); err != nil { if errors.Is(err, syscall.ESRCH) { result.forceKilled = false result.graceful = true result.duration = time.Since(start) shutdownDebugf("managed process group already gone before SIGKILL label=%q pid=%d", p.label, pid) return result } shutdownDebugf("managed process group SIGKILL failed pid=%d err=%v; killing process", pid, err) shutdownDebugf("managed process SIGKILL requested label=%q pid=%d reason=%q", p.label, pid, "process_group_kill_failed") _ = p.cmd.Process.Kill() result.err = err } if waitForManagedProcessKillDone(p.done, managedProcessForceKillWait) { shutdownDebugf("managed process killed after grace pid=%d", pid) } else { result.err = errors.Join(result.err, fmt.Errorf("timed out waiting for process %d after SIGKILL", pid)) shutdownDebugf("managed process kill wait timed out after SIGKILL pid=%d", pid) } result.duration = time.Since(start) return result } func (p *managedProcess) forceKill(reason string) managedProcessShutdownResult { start := time.Now() result := managedProcessShutdownResult{label: p.label} if p.cmd.Process == nil { shutdownDebugf("managed process force kill skipped: process nil") result.err = fmt.Errorf("process is nil") return result } pid := p.cmd.Process.Pid result.pid = pid select { case <-p.done: result.duration = time.Since(start) result.graceful = true shutdownDebugf("managed process force kill skipped; already exited label=%q pid=%d reason=%q", p.label, pid, reason) return result default: } shutdownDebugf("managed process group SIGKILL requested label=%q pgid=%d reason=%q", p.label, pid, reason) result.forceKilled = true if err := killManagedProcessGroup(pid); err != nil { if errors.Is(err, syscall.ESRCH) { result.forceKilled = false result.graceful = true result.duration = time.Since(start) shutdownDebugf("managed process group already gone before force kill label=%q pid=%d", p.label, pid) return result } shutdownDebugf("managed process group SIGKILL failed pid=%d err=%v; killing process", pid, err) shutdownDebugf("managed process SIGKILL requested label=%q pid=%d reason=%q", p.label, pid, "force_kill_group_failed") _ = p.cmd.Process.Kill() result.err = err } if waitForManagedProcessKillDone(p.done, managedProcessForceKillWait) { result.duration = time.Since(start) shutdownDebugf("managed process force killed pid=%d", pid) return result } result.duration = time.Since(start) result.err = errors.Join(result.err, fmt.Errorf("timed out waiting for process %d after SIGKILL", pid)) shutdownDebugf("managed process force kill wait timed out pid=%d", pid) return result } func waitForManagedProcessKillDone(done <-chan struct{}, timeout time.Duration) bool { select { case <-done: return true case <-time.After(timeout): return false } } func waitForAppExit(supervisor *processSupervisor, backend *restartableBackend) int { code := <-backend.exitCh supervisor.shutdown("backend exit") return code } func logForcedShutdownComplete(duration time.Duration, results []managedProcessShutdownResult) { summary := summarizeShutdown(results) launcherInfof("forced shutdown complete (duration=%s, graceful=%d, force_killed=%d, failed=%d)", duration.Round(time.Millisecond), summary.graceful, summary.forceKilled, summary.failed) } func logShutdownComplete(duration time.Duration, results []managedProcessShutdownResult) { summary := summarizeShutdown(results) launcherInfof("graceful shutdown complete (duration=%s, graceful=%d, force_killed=%d, failed=%d)", duration.Round(time.Millisecond), summary.graceful, summary.forceKilled, summary.failed) for _, result := range results { if result.forceKilled || result.err != nil { label := result.label if label == "" { label = "process" } if result.err != nil { launcherInfof("shutdown detail: %s pid=%d required force cleanup after %s: %v", label, result.pid, result.duration.Round(time.Millisecond), result.err) continue } launcherInfof("shutdown detail: %s pid=%d required SIGKILL after %s", label, result.pid, result.duration.Round(time.Millisecond)) } } } func summarizeShutdown(results []managedProcessShutdownResult) shutdownSummary { var summary shutdownSummary for _, result := range results { if result.forceKilled { summary.forceKilled++ } if result.err != nil { summary.failed++ } if result.graceful { summary.graceful++ } } return summary }