kandev/apps/backend/internal/plugins/runtime/manager.go

// Package runtime spawns and supervises kandev plugin backends as
// hashicorp/go-plugin subprocesses, per §2/§3 of
// docs/plans/plugins/GRPC-CONTRACT.md. Manager owns one *process (see
// process.go) per running plugin: Start resolves the host-platform
// executable declared in the plugin's manifest, spawns it, and performs the
// go-plugin gRPC handshake; the returned *process then supervises the
// subprocess in the background (periodic health pings, crash detection,
// restart with backoff) until Stop is called.
package runtime

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	goruntime "runtime"
	"sync"
	"sync/atomic"
	"time"

	hcplugin "github.com/hashicorp/go-plugin"
	"go.uber.org/zap"

	"github.com/kandev/kandev/internal/common/logger"
	"github.com/kandev/kandev/internal/plugins/manifest"
	"github.com/kandev/kandev/internal/plugins/store"
	"github.com/kandev/kandev/pkg/pluginsdk"
)

// pluginDataDirEnv is the environment variable kandev injects into every
// spawned plugin subprocess, per §2 of the frozen contract.
const pluginDataDirEnv = "KANDEV_PLUGIN_DATA_DIR"

// errNotRunning is returned by Manager.Ping (via process.ping) when no
// process is currently tracked/live for the given plugin id.
func errNotRunning(id string) error {
	return fmt.Errorf("plugins/runtime: plugin %q is not running", id)
}

// Option configures a Manager at construction time.
type Option func(*Manager)

// WithPingInterval overrides the supervision loop's health-check cadence
// (default 30s). Tests use a short interval instead of real sleeps.
func WithPingInterval(d time.Duration) Option {
	return func(m *Manager) { m.pingInterval = d }
}

// WithMaxConsecutiveFailures overrides how many consecutive failed pings
// trigger a restart (default 3).
func WithMaxConsecutiveFailures(n int) Option {
	return func(m *Manager) { m.maxConsecutiveFailures = n }
}

// WithRestartBackoff overrides the delay schedule between restart attempts
// (default 1s/2s/4s/8s/16s). len(delays) also bounds maxRestartAttempts
// unless WithMaxRestartAttempts is set separately.
func WithRestartBackoff(delays []time.Duration) Option {
	return func(m *Manager) { m.restartBackoff = delays }
}

// WithMaxRestartAttempts overrides how many restart attempts are made
// before giving up permanently (default 5).
func WithMaxRestartAttempts(n int) Option {
	return func(m *Manager) { m.maxRestartAttempts = n }
}

// defaultStartTimeout bounds how long a single spawn attempt (process start
// + go-plugin gRPC handshake) may take before hcplugin.Client.Client() gives
// up and returns an error. hashicorp/go-plugin defaults StartTimeout to a
// full minute when left unset; that default let a hung plugin binary block
// Service.activate (Enable/Install) for up to a minute per call. 30s is a
// generous bound for a well-behaved plugin's startup while still failing a
// hung binary fast with a clear error.
const defaultStartTimeout = 30 * time.Second

// WithStartTimeout overrides how long a single spawn attempt (process start
// + go-plugin gRPC handshake) may take before it is aborted as failed
// (default 30s). Passed through to hcplugin.ClientConfig.StartTimeout on
// every spawn, so it also bounds each individual restart attempt.
func WithStartTimeout(d time.Duration) Option {
	return func(m *Manager) { m.startTimeout = d }
}

// Manager owns the set of currently-spawned plugin subprocesses. One
// Manager is constructed for the whole backend process (see
// internal/plugins.Provide); Start/Stop are called per plugin as it is
// enabled/disabled/installed/uninstalled.
type Manager struct {
	pluginsDir     string
	log            *logger.Logger
	onStatusChange func(id string, healthy bool, reason error)

	pingInterval           time.Duration
	maxConsecutiveFailures int
	restartBackoff         []time.Duration
	maxRestartAttempts     int
	startTimeout           time.Duration

	mu        sync.Mutex
	processes map[string]*process
	// starting tracks plugin ids currently mid-Start (spawnFuncFor resolved,
	// process.start() spawning, not yet inserted into processes). Guards the
	// TOCTOU window between the "already running" check and the map
	// insertion: a concurrent Start for the same id sees the reservation and
	// fails fast instead of racing a second spawn. Held only across the fast
	// check-and-reserve/release bookkeeping in claimStart/releaseStart, never
	// across the slow spawn itself, so concurrent Starts for *different* ids
	// are never serialized against each other.
	starting map[string]struct{}
	// stopRequested tracks plugin ids Stop/StopAll was asked to stop while
	// they were still mid-Start (present in starting, not yet in
	// processes) — the Stop-during-Start orphan-process race: without this,
	// Stop/StopAll only consult processes, so a Disable/Uninstall racing an
	// in-flight spawn is a silent no-op, and the spawn then completes and
	// inserts a live supervised process for a plugin whose record/files may
	// already be gone. startProcess checks and consumes this flag
	// (consumeStopRequested) right after a successful spawn, before
	// inserting into processes; releaseStart always clears it afterward so
	// it never leaks onto a later, unrelated Start for the same id.
	stopRequested map[string]struct{}
}

// NewManager returns a Manager rooted at pluginsDir (the same directory
// store.FSStore persists records under — KANDEV_PLUGIN_DATA_DIR for plugin
// id "x" is pluginsDir/x/data). onStatusChange is invoked from the
// supervision loop's own goroutine whenever a running plugin's health
// transitions (degraded -> unhealthy, or a restart recovers it); reason is
// the triggering ping/process/restart error for unhealthy transitions and nil
// on recovery. The callback must not block. May be nil (e.g. tests that don't
// care about transitions).
func NewManager(pluginsDir string, onStatusChange func(id string, healthy bool, reason error), log *logger.Logger, opts ...Option) *Manager {
	m := &Manager{
		pluginsDir:             pluginsDir,
		log:                    log,
		onStatusChange:         onStatusChange,
		pingInterval:           defaultPingInterval,
		maxConsecutiveFailures: defaultMaxConsecutiveFailures,
		restartBackoff:         defaultRestartBackoff(),
		maxRestartAttempts:     defaultMaxRestartAttempts,
		startTimeout:           defaultStartTimeout,
		processes:              make(map[string]*process),
	}
	for _, opt := range opts {
		opt(m)
	}
	return m
}

// Start resolves the host-platform executable declared in rec's manifest
// under rec.InstallPath, spawns it as a go-plugin gRPC subprocess (AutoMTLS,
// KANDEV_PLUGIN_DATA_DIR set, cwd = rec.InstallPath), performs the
// handshake, and — on success — begins background supervision. hostFactory
// builds the Go-native Host implementation bound to this plugin's record;
// it is called once, synchronously, before the subprocess is spawned.
//
// Returns an error without registering anything if rec is not
// runtime-managed, a process is already tracked for rec.ID, the host
// platform has no declared executable, or the initial spawn/handshake
// fails. ctx is checked for early cancellation before spawning; the
// subprocess handshake itself is not context-aware (hashicorp/go-plugin's
// Client() call does not accept a context) — it is instead bounded by
// m.startTimeout (default 30s, see WithStartTimeout), passed through as
// hcplugin.ClientConfig.StartTimeout, so a hung plugin binary still fails
// fast rather than blocking the caller for go-plugin's own 1-minute default.
func (m *Manager) Start(ctx context.Context, rec *store.Record, hostFactory func(pluginID string) pluginsdk.Host) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	if !rec.IsManaged() {
		return fmt.Errorf("plugins/runtime: plugin %q is not runtime-managed", rec.ID)
	}

	recCopy := rec.Manifest
	spawnFn, err := m.spawnFuncFor(rec.ID, &recCopy, rec.InstallPath, hostFactory)
	if err != nil {
		return err
	}
	return m.startProcess(rec.ID, spawnFn)
}

// startProcess serializes start-up for a single plugin id and performs the
// actual spawn+supervise: claimStart reserves id (rejecting a concurrent
// Start for the same id, and clearing out any exhausted/gave-up previous
// entry so a fresh Start can respawn) before the slow spawn runs, closing
// the TOCTOU window between the "already running" check and the map
// insertion that let two concurrent Start calls for the same id both spawn
// a subprocess. claimStart/releaseStart only hold m.mu for fast bookkeeping
// — the spawn itself (p.start()) runs unlocked, so concurrent Starts for
// different ids are never serialized against each other.
func (m *Manager) startProcess(id string, spawnFn func() (spawnedProcess, error)) error {
	if err := m.claimStart(id); err != nil {
		return err
	}
	defer m.releaseStart(id)

	p := newProcess(id, m.log, spawnFn, m.onStatusChange)
	p.pingInterval = m.pingInterval
	p.maxConsecutiveFails = m.maxConsecutiveFailures
	p.restartBackoff = m.restartBackoff
	p.maxRestartAttempts = m.maxRestartAttempts

	if err := p.start(); err != nil {
		return fmt.Errorf("plugins/runtime: start plugin %q: %w", id, err)
	}

	// A Stop/StopAll racing this Start (see stopRequested's doc comment)
	// only had a chance to record the request while id was still in
	// m.starting — which it still is here, since releaseStart hasn't run
	// yet. Check and consume it before ever inserting into m.processes, so
	// the caller that asked to stop id is never silently ignored.
	if m.consumeStopRequested(id) {
		p.stop()
		return fmt.Errorf("plugins/runtime: plugin %q was stopped while starting, aborting", id)
	}

	m.mu.Lock()
	m.processes[id] = p
	m.mu.Unlock()
	return nil
}

// consumeStopRequested reports whether Stop/StopAll requested id be
// stopped while it was still starting, clearing the flag either way so it
// never leaks onto a later Start for the same id.
func (m *Manager) consumeStopRequested(id string) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	if _, ok := m.stopRequested[id]; ok {
		delete(m.stopRequested, id)
		return true
	}
	return false
}

// claimStart atomically checks that id has no live process and no Start
// already in flight, then reserves id in m.starting so a concurrent Start
// for the same id fails fast instead of racing a second spawn. If the
// tracked entry for id has already permanently given up (exhausted its
// restart attempts — current==nil, gaveUp==true), it is dropped so a fresh
// Start can respawn cleanly rather than being rejected as "already
// running" against a dead entry. The exhausted process is stopped after m.mu
// is released: p.stop() waits for the supervision goroutine, and its final
// status callback may call back into Manager and need the same mutex.
func (m *Manager) claimStart(id string) error {
	m.mu.Lock()

	if _, inFlight := m.starting[id]; inFlight {
		m.mu.Unlock()
		return fmt.Errorf("plugins/runtime: plugin %q is already starting", id)
	}
	var exhausted *process
	if p, exists := m.processes[id]; exists {
		if !p.isGaveUp() {
			m.mu.Unlock()
			return fmt.Errorf("plugins/runtime: plugin %q is already running", id)
		}
		delete(m.processes, id)
		exhausted = p
	}
	if m.starting == nil {
		m.starting = make(map[string]struct{})
	}
	m.starting[id] = struct{}{}
	m.mu.Unlock()

	if exhausted != nil {
		exhausted.stop()
	}
	return nil
}

// releaseStart clears id's reservation in m.starting, whether startProcess
// succeeded or failed. Also clears any stopRequested entry for id: if
// startProcess's own consumeStopRequested call didn't already consume one
// (e.g. Stop raced in just after m.processes[id] was set, so it took the
// normal already-running stop path instead), it must not leak onto a
// later, unrelated Start for the same id.
func (m *Manager) releaseStart(id string) {
	m.mu.Lock()
	delete(m.starting, id)
	delete(m.stopRequested, id)
	m.mu.Unlock()
}

// Stop kills and stops supervising the process for id, if any. A no-op if
// id is not currently running AND not currently starting. If id is
// currently mid-Start (spawnFuncFor resolved, spawning, not yet inserted
// into processes — see the starting/stopRequested doc comments), the
// request is recorded so startProcess aborts and kills the just-spawned
// process itself once the spawn completes, instead of silently leaving an
// orphan process registered.
func (m *Manager) Stop(id string) {
	m.mu.Lock()
	p, ok := m.processes[id]
	if ok {
		delete(m.processes, id)
	}
	if _, starting := m.starting[id]; starting {
		if m.stopRequested == nil {
			m.stopRequested = make(map[string]struct{})
		}
		m.stopRequested[id] = struct{}{}
	}
	m.mu.Unlock()
	if !ok {
		return
	}
	p.stop()
}

// StopAll stops every currently-running or currently-starting plugin.
// Intended for graceful backend shutdown (addCleanup).
func (m *Manager) StopAll() {
	m.mu.Lock()
	ids := make(map[string]struct{}, len(m.processes)+len(m.starting))
	for id := range m.processes {
		ids[id] = struct{}{}
	}
	for id := range m.starting {
		ids[id] = struct{}{}
	}
	m.mu.Unlock()
	for id := range ids {
		m.Stop(id)
	}
}

// Get returns the live RemotePlugin for id, for calling DeliverEvent/
// HandleWebhook, and whether one is currently available (false
// if the plugin was never started, was stopped, or is mid-restart).
func (m *Manager) Get(id string) (*pluginsdk.RemotePlugin, bool) {
	m.mu.Lock()
	p, ok := m.processes[id]
	m.mu.Unlock()
	if !ok {
		return nil, false
	}
	return p.remote()
}

// Ping issues an on-demand health check against id's current process.
func (m *Manager) Ping(id string) error {
	m.mu.Lock()
	p, ok := m.processes[id]
	m.mu.Unlock()
	if !ok {
		return errNotRunning(id)
	}
	return p.ping()
}

// Running reports whether id currently has a live process.
func (m *Manager) Running(id string) bool {
	_, ok := m.Get(id)
	return ok
}

// RestartCount returns how many times id's process has been automatically
// restarted since it was started, for Service to persist best-effort onto
// store.Record.RestartCount. Returns 0 if id is not tracked.
func (m *Manager) RestartCount(id string) int {
	m.mu.Lock()
	p, ok := m.processes[id]
	m.mu.Unlock()
	if !ok {
		return 0
	}
	return p.restartCount()
}

// spawnFuncFor builds the production spawnFn for a plugin: resolve the host
// platform's executable, prepare its data dir, and construct a fresh
// hcplugin.Client wired to hostFactory(id)'s Host implementation on every
// call (so a restart gets a fresh client/broker rather than reusing a dead
// one).
func (m *Manager) spawnFuncFor(id string, mf *manifest.Manifest, installPath string, hostFactory func(string) pluginsdk.Host) (func() (spawnedProcess, error), error) {
	execRelPath, ok := mf.ExecutableFor(goruntime.GOOS, goruntime.GOARCH)
	if !ok {
		return nil, fmt.Errorf("plugins/runtime: plugin %q has no executable for %s-%s", id, goruntime.GOOS, goruntime.GOARCH)
	}
	execPath := filepath.Join(installPath, execRelPath)
	dataDir := filepath.Join(m.pluginsDir, id, "data")
	startTimeout := m.startTimeout

	return func() (spawnedProcess, error) {
		if err := os.MkdirAll(dataDir, 0o755); err != nil {
			return nil, fmt.Errorf("plugins/runtime: create data dir for %q: %w", id, err)
		}

		cmd := exec.Command(execPath) //nolint:gosec // execPath is resolved from a verified, installed package manifest
		cmd.Dir = installPath
		cmd.Env = append(os.Environ(), pluginDataDirEnv+"="+dataDir)

		// Route go-plugin's internal hclog through zap and share a stopping
		// flag with the returned hcProcess so a deliberate Kill downgrades the
		// expected "signal: terminated" exit from ERROR to DEBUG.
		stopping := &atomic.Bool{}
		client := hcplugin.NewClient(&hcplugin.ClientConfig{
			HandshakeConfig:  pluginsdk.Handshake,
			Plugins:          map[string]hcplugin.Plugin{pluginsdk.PluginMapKey: &pluginsdk.GRPCPlugin{Host: hostFactory(id)}},
			AllowedProtocols: []hcplugin.Protocol{hcplugin.ProtocolGRPC},
			AutoMTLS:         true,
			Cmd:              cmd,
			StartTimeout:     startTimeout,
			Logger:           newHCLogAdapter(m.log.WithFields(zap.String("plugin_id", id)), stopping),
		})

		rpcClient, err := client.Client()
		if err != nil {
			client.Kill()
			return nil, fmt.Errorf("plugins/runtime: connect to plugin %q: %w", id, err)
		}
		raw, err := rpcClient.Dispense(pluginsdk.PluginMapKey)
		if err != nil {
			client.Kill()
			return nil, fmt.Errorf("plugins/runtime: dispense plugin %q: %w", id, err)
		}
		remote, ok := raw.(*pluginsdk.RemotePlugin)
		if !ok {
			client.Kill()
			return nil, fmt.Errorf("plugins/runtime: plugin %q dispensed %T, want *pluginsdk.RemotePlugin", id, raw)
		}
		return &hcProcess{client: client, proto: rpcClient, remote: remote, stopping: stopping}, nil
	}, nil
}

// hcProcess adapts a real *hcplugin.Client (plus its dispensed
// ClientProtocol/RemotePlugin) to the spawnedProcess interface process.go's
// supervision loop depends on.
type hcProcess struct {
	client *hcplugin.Client
	proto  hcplugin.ClientProtocol
	remote *pluginsdk.RemotePlugin
	// stopping is shared with the client's hclog adapter: setting it before
	// Kill marks the resulting subprocess exit as an expected teardown so the
	// adapter logs it at DEBUG rather than ERROR.
	stopping *atomic.Bool
}

func (h *hcProcess) Ping() error  { return h.proto.Ping() }
func (h *hcProcess) Exited() bool { return h.client.Exited() }
func (h *hcProcess) Kill() {
	if h.stopping != nil {
		h.stopping.Store(true)
	}
	h.client.Kill()
}
func (h *hcProcess) Remote() *pluginsdk.RemotePlugin { return h.remote }

var _ spawnedProcess = (*hcProcess)(nil)