kandev/apps/backend/internal/backendapp/agentctl.go

package backendapp

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"time"

	agentruntime "github.com/kandev/kandev/internal/agent/runtime"
	agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl"
	"github.com/kandev/kandev/internal/agent/runtime/agentctl/launcher"
	"github.com/kandev/kandev/internal/common/config"
	"github.com/kandev/kandev/internal/common/logger"
	"github.com/kandev/kandev/internal/common/ownershipperiod"
	"github.com/kandev/kandev/internal/secrets"
	"go.uber.org/zap"
)

// agentctlLauncherResult holds the outputs of provideAgentctlLauncher.
type agentctlLauncherResult struct {
	cleanup    func() error
	binaryPath string
	// recoveryDeadlineStart is the AC-EXECUTORS-SURVIVAL-003.7 recovery
	// deadline's clock start (the instant this launch first contacted a
	// recorded control endpoint). Zero when this launch never contacted one
	// (adoption disabled, no record, or a fresh spawn) -- the lifecycle
	// manager falls back to its own Start-time default in that case.
	recoveryDeadlineStart time.Time
	// inheritedRecordScope is what this launch found at the recorded control
	// endpoint. It governs how recovery classifies recovery-inventory records
	// left by an earlier launch, which cannot be judged against a server this
	// backend started itself.
	inheritedRecordScope agentruntime.InheritedRecordScope
}

// provideAgentctlLauncher starts or adopts the agentctl control server for
// standalone runtime. agentctl is a core service that always runs - it's used
// by the Standalone runtime for agent execution on the host machine.
//
// When the agent-survival capability is enabled (Layer 6.1 of
// agent-survival-across-restart), a surviving control server from a prior
// launch is tried first via agentruntime.AttemptAdoptControlServer; only when
// that fails or there is no record does this spawn a fresh one, same as
// when the capability is disabled. Either way, cfg.Agent.Standalone* is
// updated to point at whichever server this launch ends up using.
func provideAgentctlLauncher(
	ctx context.Context,
	cfg *config.Config,
	log *logger.Logger,
	availability *agentctlclient.Availability,
	store agentruntime.AdoptionRecordStore,
	secretStore secrets.SecretStore,
) (*agentctlLauncherResult, error) {
	result, scope := resolveSurvivingAgentctl(ctx, cfg, log, store, secretStore)
	if result != nil {
		availability.MarkAvailable()
		return result, nil
	}
	fresh, err := spawnFreshAgentctl(ctx, cfg, log, availability, store, secretStore)
	if fresh != nil {
		// The server this launch just started never launched the instances a
		// prior launch recorded, so recovery needs to know what was found at
		// the recorded endpoint rather than judging them against it.
		fresh.inheritedRecordScope = scope
	}
	return fresh, err
}

// resolveSurvivingAgentctl applies the agent-survival capability gate to a
// control server a prior launch left running, and returns non-nil only when
// this launch took that server over. With the capability disabled nothing is
// ever adopted, but a server an earlier survival-enabled launch detached is
// still running with no backend attached to it, so it is stopped here rather
// than left to its own unowned-shutdown timer. Either way a nil return means
// the caller spawns fresh.
func resolveSurvivingAgentctl(
	ctx context.Context,
	cfg *config.Config,
	log *logger.Logger,
	store agentruntime.AdoptionRecordStore,
	secretStore secrets.SecretStore,
) (*agentctlLauncherResult, agentruntime.InheritedRecordScope) {
	if !cfg.Features.AgentSurvival {
		agentruntime.ReclaimUnneededControlServer(ctx, store, secretStore, controlClientFactory(log), cfg.ResolvedHomeDir(),
			cfg.Agentctl.RecoveryReadTimeout, cfg.Agentctl.RecoveryReadRetries, log)
		return nil, agentruntime.InheritedRecordScopeNoServer
	}
	return adoptSurvivingAgentctl(ctx, cfg, log, store, secretStore)
}

// refusedAdoptionScope maps a refused adoption to what recovery may conclude
// about records an earlier launch left behind. A refusal that never reached
// the server -- nothing answered at the recorded endpoint, or no endpoint was
// recorded -- means no control server holds those instances, so they can be
// probed and repaired. Every refusal that did reach a reachable server leaves
// that server running, and an instance may still be alive on it, so nothing
// may be concluded from its absence from a server this launch started.
func refusedAdoptionScope(outcome agentruntime.AdoptionOutcome) agentruntime.InheritedRecordScope {
	if outcome.ContactedAt.IsZero() {
		return agentruntime.InheritedRecordScopeNoServer
	}
	return agentruntime.InheritedRecordScopeForeignServer
}

// adoptSurvivingAgentctl attempts to adopt a control server recorded by a
// prior launch. Returns nil when adoption did not complete for any reason
// (design 02's "Startup" steps 4-6): the caller then falls back to spawning
// a fresh server exactly as it would with the capability disabled.
func adoptSurvivingAgentctl(
	ctx context.Context,
	cfg *config.Config,
	log *logger.Logger,
	store agentruntime.AdoptionRecordStore,
	secretStore secrets.SecretStore,
) (*agentctlLauncherResult, agentruntime.InheritedRecordScope) {
	outcome := agentruntime.AttemptAdoptControlServer(ctx, store, secretStore, controlClientFactory(log),
		cfg.ResolvedHomeDir(), agentruntime.RequiredSurvivalCapabilities,
		cfg.Agentctl.RecoveryReadTimeout, cfg.Agentctl.RecoveryReadRetries, log)
	if !outcome.Adopted {
		log.Info("control server adoption did not complete; spawning a fresh one",
			zap.String("reason", string(outcome.Reason)))
		return nil, refusedAdoptionScope(outcome)
	}

	host, port, err := splitEndpoint(outcome.Endpoint)
	if err != nil {
		log.Warn("adopted control server endpoint is malformed; spawning fresh instead", zap.Error(err))
		return nil, refusedAdoptionScope(outcome)
	}

	log.Info("adopted a surviving control server", zap.String("endpoint", outcome.Endpoint))
	cfg.Agent.StandaloneHost = host
	cfg.Agent.StandalonePort = port
	cfg.Agent.StandaloneAuthToken = outcome.Credential
	// This process never spawned the adopted server, so it has no PID to
	// record here. Standalone liveness for an adopted record is judged by
	// enumeration against the adopted server (design 02, "Persistence"),
	// not by a stale PID probe -- a later layer of this feature.
	cfg.Agent.StandalonePID = 0

	// AC-EXECUTORS-CONTROL-OWNERSHIP-003.8: the rotation AttemptAdoptControlServer
	// just performed already counted as the first renewal, but nothing renews
	// ownership again after that without this loop. The renewal cadence is
	// computed from the adopted server's OWN reported unowned period
	// (outcome.UnownedPeriod), never this launch's local config: the two can
	// disagree across a restart that changed agentctl.unownedPeriod, and
	// renewing on a stale/mismatched cadence lets the server's own reaper
	// fire while this backend still believes it owns it (Review round 3,
	// finding 5).
	renewer := startOwnershipRenewal(ctx, log, outcome.Endpoint, outcome.Credential,
		resolveAdoptedRenewalPeriod(outcome.UnownedPeriod))

	return &agentctlLauncherResult{
		// Survival is enabled and this server was adopted, not spawned: the
		// registered cleanup must not stop it (AC-EXECUTORS-SURVIVAL-001.1),
		// and there is no local process for this launcher to own regardless.
		// It must still stop the renewal loop this backend started, or the
		// goroutine leaks past backend shutdown.
		cleanup: func() error {
			if renewer != nil {
				renewer.Stop()
			}
			return nil
		},
		binaryPath:            launcher.FindAgentctlBinary(),
		recoveryDeadlineStart: outcome.ContactedAt,
		inheritedRecordScope:  agentruntime.InheritedRecordScopeAdopted,
	}, agentruntime.InheritedRecordScopeAdopted
}

// spawnFreshAgentctl is today's unconditional launch path. If the
// agent-survival capability is enabled, the freshly spawned server is also
// durably recorded as the installation's control server (failure-table row
// "Own server started after a refused or failed adoption"); recording
// failure is logged and non-fatal to startup, since the next restart simply
// finds no record and spawns fresh again.
func spawnFreshAgentctl(
	ctx context.Context,
	cfg *config.Config,
	log *logger.Logger,
	availability *agentctlclient.Availability,
	store agentruntime.AdoptionRecordStore,
	secretStore secrets.SecretStore,
) (*agentctlLauncherResult, error) {
	l, cleanup, err := launcher.Provide(ctx, launcher.Config{
		Host:             cfg.Agent.StandaloneHost,
		Port:             cfg.Agent.StandalonePort,
		StartupConfig:    cfg.ManagedAgentctlStartupConfig(),
		OnUnexpectedExit: availability.MarkUnavailable,
	}, log)
	if err != nil {
		return nil, err
	}
	availability.MarkAvailable()
	// Update config with the actual port (may differ if fallback was used)
	if actualPort := l.Port(); actualPort != cfg.Agent.StandalonePort {
		log.Info("agentctl port changed from configured value",
			zap.Int("configured_port", cfg.Agent.StandalonePort),
			zap.Int("actual_port", actualPort))
		cfg.Agent.StandalonePort = actualPort
	}
	// Store the per-launch auth token so downstream clients can authenticate.
	cfg.Agent.StandaloneAuthToken = l.AuthToken()
	// Store the agentctl control-server PID so local/standalone executor rows can
	// carry a real host-local liveness handle (executors_running.local_pid).
	cfg.Agent.StandalonePID = l.Pid()

	var renewer *agentruntime.OwnershipRenewer
	if cfg.Features.AgentSurvival {
		endpoint := net.JoinHostPort(cfg.Agent.StandaloneHost, strconv.Itoa(cfg.Agent.StandalonePort))
		client, err := controlClientFactory(log)(endpoint)
		if err != nil {
			log.Warn("failed to build control client for the freshly spawned agentctl; adoption record not written", zap.Error(err))
		} else if err := agentruntime.RecordFreshControlServer(ctx, store, secretStore, client, endpoint, l.AuthToken()); err != nil {
			log.Warn("failed to record freshly spawned control server", zap.Error(err))
		}
		// AC-EXECUTORS-CONTROL-OWNERSHIP-003.8: the bootstrap handshake this
		// launch just completed already counted as the first renewal, but
		// nothing renews ownership again after that without this loop. This
		// launch spawned the server moments ago from this same config, so
		// (unlike the adopted path above) the local resolution IS the value
		// the server itself enforces.
		period, _ := ownershipperiod.Resolve(cfg.Agentctl.UnownedPeriod, cfg.Agentctl.IdleTimeout)
		renewer = startOwnershipRenewal(ctx, log, endpoint, l.AuthToken(), period)
	}

	return &agentctlLauncherResult{
		cleanup: func() error {
			if renewer != nil {
				renewer.Stop()
			}
			return cleanup()
		},
		binaryPath: l.BinaryPath(),
	}, nil
}

// startOwnershipRenewal builds a dedicated control client authenticated with
// the given credential and starts the periodic ownership-claim loop against
// it (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/.8), computing the renewal
// interval from the given already-resolved unowned period. The caller
// decides where that period comes from: this launch's own local config is
// only correct for a server this launch just spawned itself -- an adopted
// server may have been spawned by a prior launch with different config, so
// its renewal period must come from what that server itself reports (see
// resolveAdoptedRenewalPeriod). Returns nil (logged, not fatal) when the
// endpoint can't be parsed; startup already succeeded by this point, so
// refusing to run the surviving server over a renewal-loop failure would
// trade a smaller problem for a bigger one.
func startOwnershipRenewal(
	ctx context.Context,
	log *logger.Logger,
	endpoint string,
	credential string,
	period time.Duration,
) *agentruntime.OwnershipRenewer {
	host, port, err := splitEndpoint(endpoint)
	if err != nil {
		log.Warn("failed to start ownership renewal loop; endpoint malformed", zap.Error(err))
		return nil
	}
	client := agentctlclient.NewControlClient(host, port, log)
	client.SetAuthToken(credential)
	renewer := agentruntime.NewOwnershipRenewer(client, ownershipperiod.RenewalInterval(period), log)
	renewer.Start(ctx)
	return renewer
}

// resolveAdoptedRenewalPeriod picks the unowned period an adopted server's
// ownership-renewal loop renews against: the value that server itself
// reported on /identity during adoption (outcome.UnownedPeriod), never this
// launch's local config -- the two can disagree across a restart that
// changed agentctl.unownedPeriod, and renewing on this launch's own
// (possibly longer) cadence lets the adopted server's own reaper fire while
// this backend still believes it owns it. Falls back to the shared floor
// only for a legacy pre-upgrade server that answered /identity without an
// unowned_period_ms field at all (reported as zero).
func resolveAdoptedRenewalPeriod(reported time.Duration) time.Duration {
	if reported <= 0 {
		return ownershipperiod.MinPeriod
	}
	return reported
}

// controlClientFactory builds the real agentruntime.AdoptionControlClientFactory
// used in production: it parses "host:port" and returns a live
// agentctlclient.ControlClient targeting it.
func controlClientFactory(log *logger.Logger) agentruntime.AdoptionControlClientFactory {
	return func(endpoint string) (agentruntime.AdoptionControlClient, error) {
		host, port, err := splitEndpoint(endpoint)
		if err != nil {
			return nil, err
		}
		return agentctlclient.NewControlClient(host, port, log), nil
	}
}

func splitEndpoint(endpoint string) (host string, port int, err error) {
	host, portStr, err := net.SplitHostPort(endpoint)
	if err != nil {
		return "", 0, fmt.Errorf("split control server endpoint %q: %w", endpoint, err)
	}
	port, err = strconv.Atoi(portStr)
	if err != nil {
		return "", 0, fmt.Errorf("parse control server port from endpoint %q: %w", endpoint, err)
	}
	return host, port, nil
}

// waitForAgentctlControlHealthy waits for the agentctl control server to be healthy.
// This is called during startup to ensure agentctl is ready before accepting requests.
func waitForAgentctlControlHealthy(ctx context.Context, cfg *config.Config, log *logger.Logger) {
	client := agentctlclient.NewControlClient(cfg.Agent.StandaloneHost, cfg.Agent.StandalonePort, log)
	healthCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()

	ticker := time.NewTicker(250 * time.Millisecond)
	defer ticker.Stop()

	var lastErr error
	for {
		attemptCtx, attemptCancel := context.WithTimeout(healthCtx, 1*time.Second)
		err := client.Health(attemptCtx)
		attemptCancel()
		if err == nil {
			log.Info("agentctl control server is healthy")
			return
		}
		lastErr = err
		if healthCtx.Err() != nil {
			log.Warn("agentctl control server not ready; skipping resume wait", zap.Error(lastErr))
			return
		}
		<-ticker.C
	}
}