kandev/apps/backend/internal/plugins/runtime/process.go

package runtime

import (
	"fmt"
	"sync"
	"time"

	"go.uber.org/zap"

	"github.com/kandev/kandev/internal/common/logger"
	"github.com/kandev/kandev/pkg/pluginsdk"
)

// Default supervision tuning, per the task's frozen semantics: ping every
// 30s, 3 consecutive failures (or a detected process exit) triggers a
// restart, backed off 1s/2s/4s/8s/16s across up to 5 attempts before giving
// up.
const (
	defaultPingInterval           = 30 * time.Second
	defaultMaxConsecutiveFailures = 3
	defaultMaxRestartAttempts     = 5
)

// defaultRestartBackoff returns a fresh slice (callers may hold onto and
// mutate the one an Option sets, so Manager must not share a single
// package-level slice across instances).
func defaultRestartBackoff() []time.Duration {
	return []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
}

// spawnedProcess is the minimal surface the supervision loop needs from a
// running plugin subprocess. hcProcess (manager.go) implements this over a
// real *hcplugin.Client; tests inject fakes to unit-test supervision
// decisions without spawning a real process.
type spawnedProcess interface {
	// Ping checks that the plugin's control connection is still healthy.
	Ping() error
	// Exited reports whether the subprocess has already exited.
	Exited() bool
	// Kill terminates the subprocess if still running. Idempotent.
	Kill()
	// Remote returns the Go-native client for calling the plugin's RPCs.
	Remote() *pluginsdk.RemotePlugin
}

// process supervises one spawned plugin: periodic health pings, crash
// detection, and restart-with-backoff, up to maxRestartAttempts before
// giving up permanently (until Manager.Start is called again). Lifecycle is
// owned by Manager (start/stop), per apps/backend/AGENTS.md's
// goroutine-ownership convention: start registers the loop on wg, stop
// closes stopCh and waits for it to drain.
type process struct {
	id  string
	log *logger.Logger

	spawnFn        func() (spawnedProcess, error)
	onStatusChange func(id string, healthy bool, reason error)

	pingInterval        time.Duration
	maxConsecutiveFails int
	restartBackoff      []time.Duration
	maxRestartAttempts  int

	// sleepFn blocks for d or until stopCh fires, returning false if
	// stopped first. Overridable in tests so supervision-decision tests
	// never perform a real sleep.
	sleepFn func(stopCh <-chan struct{}, d time.Duration) bool

	// onTick, if set, runs once per supervision-loop iteration (after that
	// iteration's outcome has been fully handled), letting tests wait
	// deterministically for N cycles instead of racing the goroutine with
	// a sleep.
	onTick func()

	mu       sync.Mutex
	current  spawnedProcess
	failures int
	restarts int
	gaveUp   bool

	stopCh chan struct{}
	wg     sync.WaitGroup
}

// newProcess builds a process with default tuning. Callers may override
// pingInterval/maxConsecutiveFails/restartBackoff/maxRestartAttempts/
// sleepFn/onTick before calling start.
func newProcess(id string, log *logger.Logger, spawnFn func() (spawnedProcess, error), onStatusChange func(string, bool, error)) *process {
	return &process{
		id:                  id,
		log:                 log,
		spawnFn:             spawnFn,
		onStatusChange:      onStatusChange,
		pingInterval:        defaultPingInterval,
		maxConsecutiveFails: defaultMaxConsecutiveFailures,
		restartBackoff:      defaultRestartBackoff(),
		maxRestartAttempts:  defaultMaxRestartAttempts,
		sleepFn:             sleepOrStop,
		stopCh:              make(chan struct{}),
	}
}

// sleepOrStop is the production sleepFn: blocks for d (or returns
// immediately if d<=0) unless stopCh fires first, per the select-based
// backoff convention in apps/backend/AGENTS.md ("never time.Sleep in a
// retry/backoff loop").
func sleepOrStop(stopCh <-chan struct{}, d time.Duration) bool {
	if d <= 0 {
		select {
		case <-stopCh:
			return false
		default:
			return true
		}
	}
	timer := time.NewTimer(d)
	defer timer.Stop()
	select {
	case <-timer.C:
		return true
	case <-stopCh:
		return false
	}
}

// start spawns the process for the first time and launches the supervision
// loop. The initial spawn error is returned synchronously so a bad
// executable path/handshake failure fails Manager.Start fast rather than
// being discovered on the first supervision tick.
func (p *process) start() error {
	proc, err := p.spawnFn()
	if err != nil {
		return err
	}
	p.mu.Lock()
	p.current = proc
	p.mu.Unlock()

	p.wg.Add(1)
	go p.run()
	return nil
}

// stop signals the supervision loop to exit, waits for it to drain, and
// kills any still-running process. Idempotent-safe to call at most once per
// process (Manager.Stop removes the process from its map before calling
// this, so a second Stop(id) for the same id is a no-op at the Manager
// level).
func (p *process) stop() {
	close(p.stopCh)
	p.wg.Wait()
	p.mu.Lock()
	if p.current != nil {
		p.current.Kill()
		p.current = nil
	}
	p.mu.Unlock()
}

// remote returns the current live RemotePlugin, if any.
func (p *process) remote() (*pluginsdk.RemotePlugin, bool) {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.current == nil {
		return nil, false
	}
	return p.current.Remote(), true
}

// ping issues an on-demand health ping against the current process.
func (p *process) ping() error {
	p.mu.Lock()
	cur := p.current
	p.mu.Unlock()
	if cur == nil {
		return errNotRunning(p.id)
	}
	return cur.Ping()
}

// restartCount returns how many times this process has been successfully
// restarted since it was started.
func (p *process) restartCount() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.restarts
}

// isGaveUp reports whether this process has permanently exhausted its
// restart attempts (its supervision loop has already exited and current is
// nil). Manager.claimStart uses this to distinguish a genuinely live entry
// from a dead one that a fresh Start should be allowed to replace.
func (p *process) isGaveUp() bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.gaveUp
}

// run is the supervision loop: on every tick (pingInterval, or immediately
// if stopped) it health-checks the current process, driving restart on
// failure. It returns either when stopCh fires or the process permanently
// gives up after exhausting restart attempts.
func (p *process) run() {
	defer p.wg.Done()
	for {
		if !p.sleepFn(p.stopCh, p.pingInterval) {
			return
		}
		gaveUp := p.tick()
		if p.onTick != nil {
			p.onTick()
		}
		if gaveUp {
			return
		}
	}
}

// tick performs one supervision check. Returns true if the process just
// permanently gave up (restart attempts exhausted), signalling run to stop
// looping.
func (p *process) tick() bool {
	p.mu.Lock()
	cur := p.current
	p.mu.Unlock()
	if cur == nil {
		return false
	}

	if cur.Exited() {
		reason := fmt.Errorf("plugin process exited unexpectedly")
		p.log.Warn("plugin process exited unexpectedly", zap.String("plugin_id", p.id))
		return p.handleFailureAndRestart(reason)
	}

	if err := cur.Ping(); err != nil {
		p.mu.Lock()
		p.failures++
		reachedThreshold := p.failures >= p.maxConsecutiveFails
		p.mu.Unlock()
		if reachedThreshold {
			p.log.Warn("plugin health check failed repeatedly",
				zap.String("plugin_id", p.id), zap.Error(err))
			return p.handleFailureAndRestart(err)
		}
		return false
	}

	p.mu.Lock()
	hadFailures := p.failures > 0
	p.failures = 0
	p.mu.Unlock()
	if hadFailures {
		p.notify(true, nil)
	}
	return false
}

// handleFailureAndRestart marks the plugin unhealthy, kills whatever is
// left of the old process, and attempts to respawn it with backoff up to
// maxRestartAttempts times. Returns true if every attempt failed (the
// process has given up permanently).
func (p *process) handleFailureAndRestart(reason error) bool {
	p.notify(false, reason)

	p.mu.Lock()
	if p.current != nil {
		p.current.Kill()
		p.current = nil
	}
	p.failures = 0
	p.mu.Unlock()

	var lastErr error
	for attempt := 0; attempt < p.maxRestartAttempts; attempt++ {
		delay := time.Duration(0)
		if attempt < len(p.restartBackoff) {
			delay = p.restartBackoff[attempt]
		}
		if !p.sleepFn(p.stopCh, delay) {
			return false // stopping, not giving up
		}

		proc, err := p.spawnFn()
		if err != nil {
			lastErr = err
			p.log.Warn("plugin restart attempt failed",
				zap.String("plugin_id", p.id), zap.Int("attempt", attempt+1), zap.Error(err))
			continue
		}

		p.mu.Lock()
		p.current = proc
		p.restarts++
		p.mu.Unlock()
		p.notify(true, nil)
		return false
	}

	p.log.Error("plugin exhausted restart attempts, giving up",
		zap.String("plugin_id", p.id), zap.Int("max_attempts", p.maxRestartAttempts))
	p.mu.Lock()
	p.gaveUp = true
	p.mu.Unlock()
	if lastErr != nil {
		p.notify(false, fmt.Errorf("plugin restart attempts exhausted: %w", lastErr))
	} else {
		p.notify(false, fmt.Errorf("plugin restart attempts exhausted"))
	}
	return true
}

// notify invokes onStatusChange, if set.
func (p *process) notify(healthy bool, reason error) {
	if p.onStatusChange != nil {
		p.onStatusChange(p.id, healthy, reason)
	}
}