e75c8b37c8
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
562 lines
26 KiB
Go
562 lines
26 KiB
Go
package lifecycle
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
|
|
"github.com/kandev/kandev/internal/common/logger"
|
|
"github.com/kandev/kandev/internal/secrets"
|
|
"github.com/kandev/kandev/internal/task/models"
|
|
)
|
|
|
|
// RequiredSurvivalCapabilities is the capability set this backend requires
|
|
// of a control server before it will adopt it (design 01, "Capability
|
|
// compatibility": "The backend requires a set and adopts only when its
|
|
// required set is a subset"). Must stay a subset of
|
|
// api.SurvivalCapabilities -- kept as an independent literal here rather
|
|
// than imported, since agentctl/server/api is not a package the backend
|
|
// package layer depends on.
|
|
var RequiredSurvivalCapabilities = []string{"agent-survival.v1"}
|
|
|
|
// AdoptionRecordStore is the narrow persistence interface the adoption
|
|
// orchestration reads and writes the single installation-scoped
|
|
// control-server record through, satisfied by *sqlite.Repository.
|
|
type AdoptionRecordStore interface {
|
|
GetControlServerRecord(ctx context.Context) (*models.ControlServerRecord, error)
|
|
UpsertControlServerRecord(ctx context.Context, record *models.ControlServerRecord) error
|
|
}
|
|
|
|
// staleExecutionRepairStore is the optional persistence capability
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-004.3 and AC-EXECUTORS-SURVIVAL-005.5's
|
|
// immediate-repair step need: list this installation's live standalone
|
|
// recovery-inventory records and repair each in place. Satisfied by
|
|
// *sqlite.Repository alongside AdoptionRecordStore; type-asserted rather
|
|
// than folded into AdoptionRecordStore itself so RecordFreshControlServer's
|
|
// store parameter and every existing AdoptionRecordStore test double stay
|
|
// unaffected, matching this package's turnOutcomeApplier/recoveryStopper
|
|
// optional-capability convention.
|
|
type staleExecutionRepairStore interface {
|
|
ListExecutorsRunningLiveStandalone(ctx context.Context) ([]*models.ExecutorRunning, error)
|
|
RepairExecutorRunningDead(ctx context.Context, sessionID string) error
|
|
}
|
|
|
|
// repairLiveStandaloneRecordsAfterOwnServerStopped implements
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-004.3's "repair their recovery-inventory
|
|
// records itself... performed immediately" for a control server this
|
|
// backend has just proved is its own and has itself successfully stopped:
|
|
// unlike the deferred unowned-shutdown repair of AC-EXECUTORS-CONTROL-
|
|
// OWNERSHIP-003.5 (no backend attached at kill time), a backend is attached
|
|
// here and can write the durable store right away. Since exactly one
|
|
// control server is ever recorded per installation, stopping it makes every
|
|
// currently-live standalone recovery-inventory record definitively dead --
|
|
// there is no live enumeration to correlate against, unlike an ordinary
|
|
// recovery pass. Repair preserves resume token and worktree identity
|
|
// (RepairExecutorRunningDead), never deletes. Best-effort: a listing or
|
|
// per-record repair failure is logged and does not block the caller, since
|
|
// the stopped server is gone regardless and a later reconciliation pass
|
|
// will eventually repair any row this call missed.
|
|
func repairLiveStandaloneRecordsAfterOwnServerStopped(ctx context.Context, store staleExecutionRepairStore, log *logger.Logger) {
|
|
records, err := store.ListExecutorsRunningLiveStandalone(ctx)
|
|
if err != nil {
|
|
log.Warn("failed to list live standalone recovery-inventory records for immediate repair after stopping this installation's own control server", zap.Error(err))
|
|
return
|
|
}
|
|
for _, record := range records {
|
|
if record == nil || record.SessionID == "" {
|
|
continue
|
|
}
|
|
if err := store.RepairExecutorRunningDead(ctx, record.SessionID); err != nil {
|
|
log.Warn("failed to repair recovery-inventory record after stopping this installation's own control server",
|
|
zap.String("session_id", record.SessionID), zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
// shutdownControlServerWithRetry issues the ownership-shutdown operation
|
|
// within the same bounded per-attempt timeout and retry count as
|
|
// AC-EXECUTORS-SURVIVAL-002.13 (AC-EXECUTORS-CONTROL-OWNERSHIP-004.7), using
|
|
// the operator-configured override when the caller supplies one (a zero
|
|
// timeout or negative retry count falls back to the AC's own two-second/
|
|
// two-retry default, mirroring StandaloneExecutor.stopWithRetry). A
|
|
// dial-level connection-refused error counts as success without consuming a
|
|
// retry: AC-004.7's explicit "already stopping or already absent" carve-out,
|
|
// reached when the target has already exited on its own (e.g. via
|
|
// runUnownedReaper) before this call was ever issued.
|
|
func shutdownControlServerWithRetry(ctx context.Context, client AdoptionControlClient, timeout time.Duration, retries int) error {
|
|
if timeout <= 0 {
|
|
timeout = defaultRecoveryReadTimeout
|
|
}
|
|
if retries < 0 {
|
|
retries = defaultRecoveryReadRetries
|
|
}
|
|
|
|
var lastErr error
|
|
for attempt := 0; attempt <= retries; attempt++ {
|
|
attemptCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
lastErr = client.ShutdownControlServer(attemptCtx)
|
|
cancel()
|
|
if lastErr == nil || isAlreadyAbsentShutdownError(lastErr) {
|
|
return nil
|
|
}
|
|
}
|
|
return lastErr
|
|
}
|
|
|
|
// isAlreadyAbsentShutdownError reports whether err is the dial-level
|
|
// connection-refused shape produced when the target control server has
|
|
// already exited before this call reached it.
|
|
func isAlreadyAbsentShutdownError(err error) bool {
|
|
return err != nil && strings.Contains(err.Error(), "connection refused")
|
|
}
|
|
|
|
// withAdoptionRetry executes fn up to retries+1 times, each attempt bounded
|
|
// by its own per-attempt timeout derived from ctx rather than ctx's own
|
|
// (possibly absent) deadline -- the same bounded-per-attempt shape as
|
|
// shutdownControlServerWithRetry and StandaloneExecutor.listInstancesWithRetry
|
|
// (AC-EXECUTORS-SURVIVAL-002.13). A zero timeout or negative retry count
|
|
// falls back to the AC's own two-second/two-retry default.
|
|
func withAdoptionRetry[T any](ctx context.Context, timeout time.Duration, retries int, fn func(context.Context) (T, error)) (T, error) {
|
|
if timeout <= 0 {
|
|
timeout = defaultRecoveryReadTimeout
|
|
}
|
|
if retries < 0 {
|
|
retries = defaultRecoveryReadRetries
|
|
}
|
|
|
|
var lastErr error
|
|
var zero T
|
|
for attempt := 0; attempt <= retries; attempt++ {
|
|
attemptCtx, cancel := context.WithTimeout(ctx, timeout)
|
|
result, err := fn(attemptCtx)
|
|
cancel()
|
|
if err == nil {
|
|
return result, nil
|
|
}
|
|
lastErr = err
|
|
}
|
|
return zero, lastErr
|
|
}
|
|
|
|
// AdoptionControlClient is the subset of agentctl.ControlClient's ownership
|
|
// surface the adoption orchestration drives, narrowed so the decision below
|
|
// is testable without a real HTTP round trip.
|
|
type AdoptionControlClient interface {
|
|
SetAuthToken(token string)
|
|
GetIdentity(ctx context.Context) (*agentctl.IdentityInfo, error)
|
|
ProveOwnership(ctx context.Context, challenge string) ([]string, error)
|
|
GetServerDetails(ctx context.Context) (*agentctl.ServerDetails, error)
|
|
RotateCredential(ctx context.Context) (*agentctl.CredentialRotationResult, error)
|
|
ConfirmCredentialRotation(ctx context.Context, rotationID int64) error
|
|
ShutdownControlServer(ctx context.Context) error
|
|
}
|
|
|
|
// AdoptionControlClientFactory builds a client targeting the recorded
|
|
// control endpoint. Injected so tests never need a real listener.
|
|
type AdoptionControlClientFactory func(endpoint string) (AdoptionControlClient, error)
|
|
|
|
// AdoptionReason is a fixed, stable adoption-outcome label for observability
|
|
// (design 02: "adoption attempted, adopted, and refused with a reason drawn
|
|
// from a fixed set"). AC-EXECUTORS-CONTROL-OWNERSHIP-001.8's "no recorded
|
|
// endpoint, or nothing answers, or already shutting down" bucket is
|
|
// deliberately NOT a refusal reason (nothing about it was refused), so it
|
|
// gets its own value outside the AC-001.6 refusal-reason precedence list.
|
|
//
|
|
// AC-001.6's own refusal-reason list also names "no answer" as one of its six
|
|
// enumerated reasons, ahead of identity mismatch in its precedence order --
|
|
// which reads as a refusal, in tension with AC-001.8 treating the identical
|
|
// case ("an endpoint that nothing answers") as no refusal at all, and with
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-003.9 explicitly mapping its own
|
|
// shutting-down outcome onto AC-001.8's bucket rather than AC-001.6's. Two
|
|
// ACs describing this case as "not a refusal" against one phrase in a third
|
|
// naming it as one: AC-001.8 (reinforced by 003.9) is treated as the
|
|
// authoritative behavior, and AC-001.6's "no answer" is read as the same
|
|
// no_server bucket by another name, not a distinct recorded refusal reason.
|
|
type AdoptionReason string
|
|
|
|
const (
|
|
// AdoptionReasonNoServer covers "no recorded control endpoint", "the
|
|
// recorded endpoint answers nothing", and "the recorded endpoint reports
|
|
// it has already begun an unowned shutdown" alike (AC-001.8): none of
|
|
// these is a refusal, so none carries a refusal reason, and all three
|
|
// result in the same action, spawn fresh and report nothing recovered.
|
|
AdoptionReasonNoServer AdoptionReason = "no_server"
|
|
// AdoptionReasonIdentityMismatch covers a server that advertises no
|
|
// identity or no capability set, one whose identity differs from the
|
|
// recorded one, and one that cannot prove it holds the recorded
|
|
// credential (AC-001.4, AC-001.5) -- refused, left running untouched,
|
|
// never contacted again by this adoption attempt, and never sent the
|
|
// credential.
|
|
AdoptionReasonIdentityMismatch AdoptionReason = "identity_mismatch"
|
|
// AdoptionReasonCredentialUnavailable is the stored credential could not
|
|
// be retrieved (AC-001.9): refused before any authentication was
|
|
// attempted, left running untouched, its own unowned shutdown reclaims it.
|
|
AdoptionReasonCredentialUnavailable AdoptionReason = "credential_unavailable"
|
|
// AdoptionReasonAuthenticationFailed is an attempted rotation the server
|
|
// rejected -- including a rotate refused because the server has already
|
|
// begun an unowned shutdown after this attempt's identity read succeeded,
|
|
// which is intentionally not distinguished from a wrong credential here:
|
|
// both refuse without issuing a stop and take the identical spawn-fresh
|
|
// path, and only the logged reason label would differ.
|
|
AdoptionReasonAuthenticationFailed AdoptionReason = "authentication_failed"
|
|
// AdoptionReasonCapabilityIncompatible is an authenticated, identified,
|
|
// own server missing a required capability (AC-004.3/004.4): refused
|
|
// AND stopped, because only an own authenticated server may ever be
|
|
// stopped.
|
|
AdoptionReasonCapabilityIncompatible AdoptionReason = "capability_incompatible"
|
|
// AdoptionReasonCredentialRotationFailed is a rotation the server
|
|
// accepted whose durable storage (secret write or record write) did not
|
|
// complete (AC-002.5): adoption is incomplete, nothing further is issued
|
|
// to that server, and it is left exactly as it was left by the
|
|
// two-phase window -- still holding both the superseded and the new
|
|
// credential, unconfirmed.
|
|
AdoptionReasonCredentialRotationFailed AdoptionReason = "credential_rotation_failed"
|
|
)
|
|
|
|
// AdoptionOutcome is the result of attempting to adopt a previously-detached
|
|
// control server. Endpoint and Credential are populated only when Adopted.
|
|
// ContactedAt is the instant this attempt first reached the recorded control
|
|
// endpoint (the GetIdentity call) -- the AC-EXECUTORS-SURVIVAL-003.7 recovery
|
|
// deadline's clock start. It is left zero when no live server was ever
|
|
// reached (no record, or the client/transport itself could not be built),
|
|
// since nothing recovery-relevant is running in that case.
|
|
type AdoptionOutcome struct {
|
|
Adopted bool
|
|
Reason AdoptionReason
|
|
Endpoint string
|
|
// Credential is the single credential that authenticates both
|
|
// control-plane and per-instance operations and streams (design 01
|
|
// "Single driver", AC-EXECUTORS-CONTROL-OWNERSHIP-002.6): use it for
|
|
// every client built against this control server or any instance it
|
|
// supervises.
|
|
Credential string
|
|
ContactedAt time.Time
|
|
// UnownedPeriod is the adopted server's own resolved unowned period, read
|
|
// back from its /identity response (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/
|
|
// .7). Only this value is safe to compute an ownership-renewal cadence
|
|
// from: this backend's local config can disagree with what the adopted
|
|
// process actually enforces across a restart that changed
|
|
// agentctl.unownedPeriod. Zero when the server answered without the
|
|
// field at all (a legacy pre-upgrade server) -- callers apply their own
|
|
// floor in that case, never a locally-resolved value.
|
|
UnownedPeriod time.Duration
|
|
}
|
|
|
|
// AttemptAdoptControlServer implements startup steps 4 through 6 of design
|
|
// 02's "Startup" control flow: read the recorded control endpoint and
|
|
// identity, evaluate the gates in the required order -- home match, then
|
|
// authentication, then capability subset (AC-EXECUTORS-CONTROL-OWNERSHIP-001.3)
|
|
// -- and on a match rotate the ownership credential and persist the new
|
|
// record. It never spawns a process: the caller decides what to do with a
|
|
// refusal, which is always "spawn a fresh server on a free port and record
|
|
// it" regardless of which reason produced the refusal.
|
|
func AttemptAdoptControlServer(
|
|
ctx context.Context,
|
|
store AdoptionRecordStore,
|
|
secretStore secrets.SecretStore,
|
|
newClient AdoptionControlClientFactory,
|
|
homeDir string,
|
|
requiredCapabilities []string,
|
|
recoveryReadTimeout time.Duration,
|
|
recoveryReadRetries int,
|
|
log *logger.Logger,
|
|
) AdoptionOutcome {
|
|
record, err := store.GetControlServerRecord(ctx)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonNoServer}
|
|
}
|
|
|
|
client, err := newClient(record.Endpoint)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonNoServer}
|
|
}
|
|
|
|
// The AC-EXECUTORS-SURVIVAL-003.7 recovery deadline's clock starts here,
|
|
// at the first real contact with the recorded control endpoint --
|
|
// regardless of which gate below ultimately refuses or accepts adoption.
|
|
contactedAt := time.Now()
|
|
identity, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.GetIdentity)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonNoServer}
|
|
}
|
|
|
|
if !identityMatchesRecordedServer(identity, record) || len(identity.Capabilities) == 0 {
|
|
return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt}
|
|
}
|
|
|
|
credential, err := revealControlServerCredential(ctx, secretStore, record.CredentialSecretID)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonCredentialUnavailable, ContactedAt: contactedAt}
|
|
}
|
|
|
|
if !controlServerHoldsCredential(ctx, client, credential, homeDir, recoveryReadTimeout, recoveryReadRetries) {
|
|
return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt}
|
|
}
|
|
|
|
client.SetAuthToken(credential)
|
|
rotated, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.RotateCredential)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonAuthenticationFailed, ContactedAt: contactedAt}
|
|
}
|
|
|
|
if !capabilitiesSatisfy(requiredCapabilities, identity.Capabilities) {
|
|
client.SetAuthToken(rotated.Credential)
|
|
return stopIncompatibleControlServer(ctx, client, store, record, recoveryReadTimeout, recoveryReadRetries, contactedAt, log)
|
|
}
|
|
|
|
client.SetAuthToken(rotated.Credential)
|
|
return finalizeControlServerAdoption(ctx, finalizeAdoptionArgs{
|
|
store: store,
|
|
secretStore: secretStore,
|
|
client: client,
|
|
record: record,
|
|
identity: identity,
|
|
rotated: rotated,
|
|
contactedAt: contactedAt,
|
|
recoveryReadTimeout: recoveryReadTimeout,
|
|
recoveryReadRetries: recoveryReadRetries,
|
|
log: log,
|
|
})
|
|
}
|
|
|
|
// finalizeAdoptionArgs carries the state the two durable writes and the
|
|
// confirmation call need, so the adoption gates above stay readable as a
|
|
// sequence of refusals.
|
|
type finalizeAdoptionArgs struct {
|
|
store AdoptionRecordStore
|
|
secretStore secrets.SecretStore
|
|
client AdoptionControlClient
|
|
record *models.ControlServerRecord
|
|
identity *agentctl.IdentityInfo
|
|
rotated *agentctl.CredentialRotationResult
|
|
contactedAt time.Time
|
|
recoveryReadTimeout time.Duration
|
|
recoveryReadRetries int
|
|
log *logger.Logger
|
|
}
|
|
|
|
// finalizeControlServerAdoption performs the two durable writes adoption
|
|
// depends on -- the rotated credential and the updated record -- and only
|
|
// then tells the server the rotation is confirmed. Either write failing
|
|
// leaves adoption incomplete, because confirming a credential this backend
|
|
// did not durably record would tell the server to drop the one credential
|
|
// still recoverable from durable state.
|
|
func finalizeControlServerAdoption(ctx context.Context, a finalizeAdoptionArgs) AdoptionOutcome {
|
|
secretID, err := storeControlServerCredential(ctx, a.secretStore, a.record.CredentialSecretID, a.rotated.Credential)
|
|
if err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt}
|
|
}
|
|
|
|
updated := &models.ControlServerRecord{
|
|
Endpoint: a.record.Endpoint,
|
|
ServerIdentity: a.identity.ServerIdentity,
|
|
CredentialSecretID: secretID,
|
|
Capabilities: a.identity.Capabilities,
|
|
DiagnosticLogPath: recordedDiagnosticLogPath(ctx, a.client, a.recoveryReadTimeout, a.recoveryReadRetries),
|
|
CreatedAt: a.record.CreatedAt,
|
|
}
|
|
if err := a.store.UpsertControlServerRecord(ctx, updated); err != nil {
|
|
return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt}
|
|
}
|
|
|
|
// Sent only after both durable writes above completed
|
|
// (AC-EXECUTORS-CONTROL-OWNERSHIP-002.7). A failure here is not treated
|
|
// as fatal to adoption: the durable state already names the rotated
|
|
// credential as current, so a future retry or the server's own
|
|
// idempotent replay (AC-002.10) recovers it, and the superseded
|
|
// credential merely stays adoption-only-acceptable a little longer than
|
|
// necessary in the meantime.
|
|
if _, err := withAdoptionRetry(ctx, a.recoveryReadTimeout, a.recoveryReadRetries, func(c context.Context) (struct{}, error) {
|
|
return struct{}{}, a.client.ConfirmCredentialRotation(c, a.rotated.RotationID)
|
|
}); err != nil {
|
|
a.log.Warn("control server credential rotation stored durably but confirmation call failed",
|
|
zap.String("endpoint", a.record.Endpoint), zap.Error(err))
|
|
}
|
|
|
|
return AdoptionOutcome{
|
|
Adopted: true,
|
|
Endpoint: a.record.Endpoint,
|
|
Credential: a.rotated.Credential,
|
|
ContactedAt: a.contactedAt,
|
|
UnownedPeriod: time.Duration(a.identity.UnownedPeriodMS) * time.Millisecond,
|
|
}
|
|
}
|
|
|
|
// RecordFreshControlServer durably records a freshly spawned (not adopted)
|
|
// control server as the new installation-scoped record. Per the failure
|
|
// table's "Own server started after a refused or failed adoption" row, the
|
|
// single control-server record is always rewritten to name the new server:
|
|
// the record cannot keep pointing at a server that is about to reap itself
|
|
// through its own unowned shutdown. credential is the server's initial
|
|
// ownership credential (the bootstrap auth token minted at launch). A
|
|
// prior record's CredentialSecretID, when one is readable, is reused so a
|
|
// spawn-after-refused-adoption does not leave an orphaned secret row behind.
|
|
func RecordFreshControlServer(
|
|
ctx context.Context,
|
|
store AdoptionRecordStore,
|
|
secretStore secrets.SecretStore,
|
|
client AdoptionControlClient,
|
|
endpoint string,
|
|
credential string,
|
|
) error {
|
|
identity, err := client.GetIdentity(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("read freshly spawned control server identity: %w", err)
|
|
}
|
|
|
|
var priorSecretID string
|
|
if prior, err := store.GetControlServerRecord(ctx); err == nil {
|
|
priorSecretID = prior.CredentialSecretID
|
|
}
|
|
|
|
secretID, err := storeControlServerCredential(ctx, secretStore, priorSecretID, credential)
|
|
if err != nil {
|
|
return fmt.Errorf("store freshly spawned control server credential: %w", err)
|
|
}
|
|
|
|
record := &models.ControlServerRecord{
|
|
Endpoint: endpoint,
|
|
ServerIdentity: identity.ServerIdentity,
|
|
CredentialSecretID: secretID,
|
|
Capabilities: identity.Capabilities,
|
|
DiagnosticLogPath: recordedDiagnosticLogPath(ctx, client, defaultRecoveryReadTimeout, defaultRecoveryReadRetries),
|
|
}
|
|
return store.UpsertControlServerRecord(ctx, record)
|
|
}
|
|
|
|
// ReclaimUnneededControlServer implements AC-EXECUTORS-SURVIVAL-005.5: when
|
|
// the agent-survival capability is disabled, a detached control server left
|
|
// running by an earlier survival-enabled launch is stopped rather than left
|
|
// running unowned -- but only when this backend can prove that server is
|
|
// its own by the same identity+credential test AttemptAdoptControlServer
|
|
// applies to adoption. A server this backend cannot prove is its own is
|
|
// left untouched, and so is one that answers nothing (nothing left to
|
|
// reclaim) or has no recorded endpoint at all.
|
|
//
|
|
// This never adopts: it does not rotate the credential, so a backend
|
|
// holding only a superseded credential can still perform it, which
|
|
// AC-EXECUTORS-SURVIVAL-005.1 forbids adoption itself from doing. The stop
|
|
// is issued as the ownership-shutdown operation of
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-002.9, which requires no prior adoption or
|
|
// rotation for exactly this reason. Every failure mode here is logged and
|
|
// non-fatal to startup: the caller always falls through to spawning a fresh
|
|
// server regardless of what this reclaim attempt did.
|
|
func ReclaimUnneededControlServer(
|
|
ctx context.Context,
|
|
store AdoptionRecordStore,
|
|
secretStore secrets.SecretStore,
|
|
newClient AdoptionControlClientFactory,
|
|
homeDir string,
|
|
recoveryReadTimeout time.Duration,
|
|
recoveryReadRetries int,
|
|
log *logger.Logger,
|
|
) {
|
|
record, err := store.GetControlServerRecord(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
client, err := newClient(record.Endpoint)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
identity, err := client.GetIdentity(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if !identityMatchesRecordedServer(identity, record) {
|
|
return
|
|
}
|
|
|
|
credential, err := revealControlServerCredential(ctx, secretStore, record.CredentialSecretID)
|
|
if err != nil {
|
|
log.Warn("cannot reclaim detached control server: stored credential unavailable",
|
|
zap.String("endpoint", record.Endpoint), zap.Error(err))
|
|
return
|
|
}
|
|
|
|
if !controlServerHoldsCredential(ctx, client, credential, homeDir, recoveryReadTimeout, recoveryReadRetries) {
|
|
log.Warn("cannot reclaim detached control server: it could not prove it holds this installation's credential",
|
|
zap.String("endpoint", record.Endpoint))
|
|
return
|
|
}
|
|
|
|
client.SetAuthToken(credential)
|
|
if err := shutdownControlServerWithRetry(ctx, client, recoveryReadTimeout, recoveryReadRetries); err != nil {
|
|
log.Warn("failed to stop detached control server left by an earlier survival-enabled launch after exhausting retries; leaving it to its own unowned shutdown",
|
|
zap.String("endpoint", record.Endpoint), zap.Error(err))
|
|
return
|
|
}
|
|
|
|
log.Info("stopped a detached control server left by an earlier survival-enabled launch now that the capability is disabled",
|
|
zap.String("endpoint", record.Endpoint))
|
|
if repairStore, ok := store.(staleExecutionRepairStore); ok {
|
|
repairLiveStandaloneRecordsAfterOwnServerStopped(ctx, repairStore, log)
|
|
}
|
|
}
|
|
|
|
// stopIncompatibleControlServer stops an authenticated, identified, own
|
|
// server that is missing a required capability and, on success, repairs its
|
|
// live standalone records immediately (AC-EXECUTORS-CONTROL-OWNERSHIP-004.3).
|
|
// Always returns the AdoptionReasonCapabilityIncompatible refusal regardless
|
|
// of whether the stop itself succeeded.
|
|
func stopIncompatibleControlServer(
|
|
ctx context.Context,
|
|
client AdoptionControlClient,
|
|
store AdoptionRecordStore,
|
|
record *models.ControlServerRecord,
|
|
recoveryReadTimeout time.Duration,
|
|
recoveryReadRetries int,
|
|
contactedAt time.Time,
|
|
log *logger.Logger,
|
|
) AdoptionOutcome {
|
|
if stopErr := shutdownControlServerWithRetry(ctx, client, recoveryReadTimeout, recoveryReadRetries); stopErr != nil {
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-004.7: retries exhausted -- record no
|
|
// repair, report no recovered instances (already true, Adopted stays
|
|
// false below), and leave the server to its own unowned shutdown; a
|
|
// later backend repairs through the deferred
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-003.5 path once it stops itself.
|
|
log.Warn("failed to stop incompatible control server after exhausting retries; leaving it to its own unowned shutdown",
|
|
zap.String("endpoint", record.Endpoint), zap.Error(stopErr))
|
|
} else if repairStore, ok := store.(staleExecutionRepairStore); ok {
|
|
// AC-EXECUTORS-CONTROL-OWNERSHIP-004.3: the stop succeeded and a
|
|
// backend is attached, so repair every live standalone record
|
|
// immediately rather than deferring to the next start.
|
|
repairLiveStandaloneRecordsAfterOwnServerStopped(ctx, repairStore, log)
|
|
}
|
|
return AdoptionOutcome{Reason: AdoptionReasonCapabilityIncompatible, ContactedAt: contactedAt}
|
|
}
|
|
|
|
// identityMatchesRecordedServer reports whether a live server's per-launch
|
|
// identity nonce is the exact one the control-server record carries, which is
|
|
// what proves the live server IS the recorded process rather than merely
|
|
// another launch. Installation identity (the home directory) is a separate
|
|
// question, proved over the authenticated ownership-details exchange rather
|
|
// than here, because /identity deliberately discloses no filesystem path
|
|
// (AC-EXECUTORS-CONTROL-OWNERSHIP-001.11). An identity either side cannot
|
|
// produce leaves the match unprovable and is refused rather than assumed.
|
|
func identityMatchesRecordedServer(identity *agentctl.IdentityInfo, record *models.ControlServerRecord) bool {
|
|
if record.ServerIdentity == "" || identity.ServerIdentity != record.ServerIdentity {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func capabilitiesSatisfy(required, advertised []string) bool {
|
|
have := make(map[string]struct{}, len(advertised))
|
|
for _, c := range advertised {
|
|
have[c] = struct{}{}
|
|
}
|
|
for _, r := range required {
|
|
if _, ok := have[r]; !ok {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|