137336de86
* fix: drain queued chat messages after cancel * fix: address queue drain review feedback * fix: keep queued messages parked after cancel * fix: stabilize queue drain errors * fix: keep queue parked during cancel events * fix: wait for lifecycle stream goroutines * fix: address lifecycle stream review feedback
604 lines
24 KiB
Go
604 lines
24 KiB
Go
package lifecycle
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/kandev/kandev/internal/agentctl/tracing"
|
|
"github.com/kandev/kandev/internal/common/appctx"
|
|
"github.com/kandev/kandev/internal/events"
|
|
"github.com/kandev/kandev/internal/secrets"
|
|
)
|
|
|
|
// ErrSessionWorkspaceNotReady indicates the task session exists but does not yet
|
|
// have a resolved workspace path (typically while worktree preparation is in progress).
|
|
var ErrSessionWorkspaceNotReady = errors.New("session workspace not ready")
|
|
|
|
// GetOrEnsureExecution returns an existing execution or creates one on-demand.
|
|
// Use this for workspace-oriented operations (files, shell, inference, ports, vscode, LSP)
|
|
// that should survive backend restarts. For operations requiring a running agent
|
|
// process (prompt, cancel, mode), use GetExecutionBySessionID instead.
|
|
//
|
|
// Concurrent calls for the same sessionID are deduplicated via singleflight.
|
|
func (m *Manager) GetOrEnsureExecution(ctx context.Context, sessionID string) (*AgentExecution, error) {
|
|
if sessionID == "" {
|
|
return nil, fmt.Errorf("session_id is required")
|
|
}
|
|
|
|
// Fast path: execution already in memory
|
|
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
|
|
return execution, nil
|
|
}
|
|
|
|
// Slow path: create on-demand, deduplicated by sessionID-keyed singleflight.
|
|
// Use ensureWorkspaceExecutionLocked (not EnsureWorkspaceExecutionForSession)
|
|
// to avoid recursing into the same singleflight slot we already hold.
|
|
v, err, _ := m.ensureExecutionGroup.Do(sessionID, func() (interface{}, error) {
|
|
return m.ensureWorkspaceExecutionLocked(ctx, "", sessionID)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return v.(*AgentExecution), nil
|
|
}
|
|
|
|
// GetOrEnsureExecutionForEnvironment returns an execution for a task environment,
|
|
// creating one on-demand from the workspace info provider when needed.
|
|
//
|
|
// Important: this MUST share the session-keyed singleflight bucket with
|
|
// GetOrEnsureExecution(sessionID) and EnsureWorkspaceExecutionForSession.
|
|
// A previous version keyed by `"env:" + envID`, which let a concurrent
|
|
// session-keyed call race past it (each path observed "no execution" for its
|
|
// own key, both called createExecution, both ExecutionStore.Add, the second
|
|
// silently overwrote the bySession index, and the first execution's
|
|
// agent subprocess was orphaned). See `ErrExecutionAlreadyExistsForSession`.
|
|
func (m *Manager) GetOrEnsureExecutionForEnvironment(ctx context.Context, taskEnvironmentID string) (*AgentExecution, error) {
|
|
if taskEnvironmentID == "" {
|
|
return nil, fmt.Errorf("task_environment_id is required")
|
|
}
|
|
|
|
if execution, exists := m.executionStore.GetByTaskEnvironmentID(taskEnvironmentID); exists {
|
|
return execution, nil
|
|
}
|
|
|
|
if m.workspaceInfoProvider == nil {
|
|
return nil, fmt.Errorf("workspace info provider not configured")
|
|
}
|
|
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForEnvironment(ctx, taskEnvironmentID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get workspace info for environment %s: %w", taskEnvironmentID, err)
|
|
}
|
|
if info == nil {
|
|
return nil, fmt.Errorf("task environment %s not found", taskEnvironmentID)
|
|
}
|
|
if info.TaskEnvironmentID == "" {
|
|
return nil, fmt.Errorf("task environment %s has no task_environment_id", taskEnvironmentID)
|
|
}
|
|
if info.TaskEnvironmentID != taskEnvironmentID {
|
|
return nil, fmt.Errorf("workspace info resolved environment %s, want %s", info.TaskEnvironmentID, taskEnvironmentID)
|
|
}
|
|
if info.WorkspacePath == "" {
|
|
return nil, fmt.Errorf("%w: task environment %s has no workspace path yet", ErrSessionWorkspaceNotReady, taskEnvironmentID)
|
|
}
|
|
if info.SessionID == "" {
|
|
return nil, fmt.Errorf("task environment %s has no task session", taskEnvironmentID)
|
|
}
|
|
|
|
// Share the sessionID-keyed bucket so we deduplicate against any concurrent
|
|
// GetOrEnsureExecution(sessionID) / EnsureWorkspaceExecutionForSession for
|
|
// the same session.
|
|
v, err, _ := m.ensureExecutionGroup.Do(info.SessionID, func() (interface{}, error) {
|
|
if execution, exists := m.executionStore.GetBySessionID(info.SessionID); exists {
|
|
return execution, nil
|
|
}
|
|
if execution, exists := m.executionStore.GetByTaskEnvironmentID(taskEnvironmentID); exists {
|
|
return execution, nil
|
|
}
|
|
// createExecution publishes AgentctlStarting before spawning the
|
|
// waitForAgentctlReady goroutine, so frontend gates flip out of
|
|
// `undefined` even on this lazy-create path.
|
|
execution, err := m.createExecution(ctx, info.TaskID, info)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return execution, nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return v.(*AgentExecution), nil
|
|
}
|
|
|
|
// EnsureWorkspaceExecutionForSession ensures an agentctl execution exists for a specific task session.
|
|
// This is used when the frontend provides a session ID (e.g., from URL path /task/[id]/[sessionId]).
|
|
// If an execution already exists for the session, it returns it. Otherwise, it creates a new execution
|
|
// using the session's workspace configuration from the database.
|
|
//
|
|
// Concurrent calls (including from GetOrEnsureExecution and
|
|
// GetOrEnsureExecutionForEnvironment) are deduplicated via the same
|
|
// sessionID-keyed singleflight bucket so they cannot race past their
|
|
// individual check-then-act guards and create duplicate executions.
|
|
func (m *Manager) EnsureWorkspaceExecutionForSession(ctx context.Context, taskID, sessionID string) (*AgentExecution, error) {
|
|
if sessionID == "" {
|
|
return nil, fmt.Errorf("session_id is required")
|
|
}
|
|
|
|
// Fast path: execution already in memory
|
|
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
|
|
return execution, nil
|
|
}
|
|
|
|
v, err, _ := m.ensureExecutionGroup.Do(sessionID, func() (interface{}, error) {
|
|
return m.ensureWorkspaceExecutionLocked(ctx, taskID, sessionID)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return v.(*AgentExecution), nil
|
|
}
|
|
|
|
// ensureWorkspaceExecutionLocked is the body of EnsureWorkspaceExecutionForSession
|
|
// run inside the sessionID-keyed singleflight bucket. Callers other than
|
|
// EnsureWorkspaceExecutionForSession must already hold the singleflight slot.
|
|
func (m *Manager) ensureWorkspaceExecutionLocked(ctx context.Context, taskID, sessionID string) (*AgentExecution, error) {
|
|
// Double-check after acquiring the slot — a peer in the same group may have
|
|
// finished while we were waiting.
|
|
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
|
|
return execution, nil
|
|
}
|
|
|
|
if m.workspaceInfoProvider == nil {
|
|
return nil, fmt.Errorf("workspace info provider not configured")
|
|
}
|
|
|
|
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, taskID, sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get workspace info for session %s: %w", sessionID, err)
|
|
}
|
|
if info == nil {
|
|
return nil, fmt.Errorf("session %s not found", sessionID)
|
|
}
|
|
|
|
// Resolve taskID from provider when caller doesn't have it (e.g., GetOrEnsureExecution)
|
|
if taskID == "" {
|
|
taskID = info.TaskID
|
|
}
|
|
|
|
if info.TaskEnvironmentID != "" {
|
|
if execution, exists := m.executionStore.GetByTaskEnvironmentID(info.TaskEnvironmentID); exists {
|
|
m.logger.Info("reusing existing execution for task environment",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("task_environment_id", info.TaskEnvironmentID),
|
|
zap.String("execution_id", execution.ID))
|
|
return execution, nil
|
|
}
|
|
}
|
|
|
|
if info.WorkspacePath == "" {
|
|
return nil, fmt.Errorf("%w: session %s has no workspace path yet", ErrSessionWorkspaceNotReady, sessionID)
|
|
}
|
|
|
|
m.logger.Info("creating execution for task session",
|
|
zap.String("task_id", taskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("workspace_path", info.WorkspacePath),
|
|
zap.String("acp_session_id", info.ACPSessionID))
|
|
|
|
// createExecution publishes AgentctlStarting before spawning the
|
|
// waitForAgentctlReady goroutine, so workspace-only executions also
|
|
// notify the frontend without racing the readiness event.
|
|
execution, err := m.createExecution(ctx, taskID, info)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// For workspace-only executions (no agent), wait for agentctl to be ready
|
|
// then connect the workspace stream so process output can be received.
|
|
// Note: AgentctlReady/Error events are already handled by waitForAgentctlReady
|
|
// (started by createExecution), so this goroutine only connects the stream.
|
|
go func() {
|
|
// Use detached context that respects stopCh for graceful shutdown
|
|
waitCtx, cancel := appctx.Detached(ctx, m.stopCh, 60*time.Second)
|
|
defer cancel()
|
|
|
|
if err := execution.agentctl.WaitForReady(waitCtx, 60*time.Second); err != nil {
|
|
m.logger.Error("agentctl not ready for workspace stream connection",
|
|
zap.String("execution_id", execution.ID),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
|
|
// Connect workspace stream for process output (agent stream not needed for workspace-only)
|
|
if m.streamManager != nil {
|
|
m.logger.Info("connecting workspace stream for workspace-only execution",
|
|
zap.String("execution_id", execution.ID))
|
|
m.streamManager.ConnectWorkspaceStream(execution, nil)
|
|
}
|
|
}()
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
// GetExecutionIDForSession returns the execution ID for a session from the in-memory
|
|
// execution store. Returns empty string and error if no execution is found.
|
|
func (m *Manager) GetExecutionIDForSession(_ context.Context, sessionID string) (string, error) {
|
|
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
|
|
return execution.ID, nil
|
|
}
|
|
return "", fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID)
|
|
}
|
|
|
|
// EnsurePassthroughExecution ensures an execution exists for a passthrough session
|
|
// and starts the passthrough process if needed. This is called when the terminal
|
|
// handler receives a connection for a session that might need recovery after backend restart.
|
|
//
|
|
// The sessionID is required. If taskID is empty, it will be looked up from:
|
|
// 1. The existing execution (if any)
|
|
// 2. The workspace info provider
|
|
//
|
|
// Returns the execution with a running passthrough process, or an error.
|
|
func (m *Manager) EnsurePassthroughExecution(ctx context.Context, sessionID string) (*AgentExecution, error) {
|
|
// Check if execution already exists with a running passthrough process.
|
|
// PassthroughProcessID is not cleared on exit, so a stale ID can point at
|
|
// a dead process; verify the runner still has it before short-circuiting,
|
|
// otherwise a fast-failed resume launch would keep returning the dead ID
|
|
// and the WS handler's IsProcessReadyOrPending check would 503 forever.
|
|
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
|
|
if execution.PassthroughProcessID != "" {
|
|
if runner := m.GetInteractiveRunner(); runner != nil && runner.IsProcessReadyOrPending(execution.PassthroughProcessID) {
|
|
return execution, nil
|
|
}
|
|
m.logger.Info("execution has stale passthrough process ID, relaunching",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("execution_id", execution.ID),
|
|
zap.String("stale_process_id", execution.PassthroughProcessID))
|
|
}
|
|
return m.resumeExistingExecution(ctx, sessionID, execution)
|
|
}
|
|
|
|
// No execution exists - need to create one from session info
|
|
return m.createExecutionFromSessionInfo(ctx, sessionID)
|
|
}
|
|
|
|
// resumeExistingExecution starts the passthrough process for an existing execution
|
|
// that has no running process (e.g., after backend restart).
|
|
func (m *Manager) resumeExistingExecution(ctx context.Context, sessionID string, execution *AgentExecution) (*AgentExecution, error) {
|
|
m.logger.Info("execution exists but passthrough process not running, starting",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("execution_id", execution.ID))
|
|
|
|
if err := m.ResumePassthroughSession(ctx, sessionID); err != nil {
|
|
return nil, fmt.Errorf("resume passthrough session %s: %w", sessionID, err)
|
|
}
|
|
|
|
// Get updated execution with process ID
|
|
execution, exists := m.executionStore.GetBySessionID(sessionID)
|
|
if !exists {
|
|
return nil, fmt.Errorf("execution disappeared after resuming passthrough session %s", sessionID)
|
|
}
|
|
return execution, nil
|
|
}
|
|
|
|
// createExecutionFromSessionInfo creates a new execution for a passthrough session
|
|
// when no execution exists (e.g., backend restarted and execution store was cleared).
|
|
func (m *Manager) createExecutionFromSessionInfo(ctx context.Context, sessionID string) (*AgentExecution, error) {
|
|
if m.workspaceInfoProvider == nil {
|
|
return nil, fmt.Errorf("cannot restore session %s: workspace info provider not configured", sessionID)
|
|
}
|
|
|
|
// Get workspace info from the provider (looks up session to get taskID, workspace path, etc.)
|
|
info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, "", sessionID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get workspace info for session %s: %w", sessionID, err)
|
|
}
|
|
|
|
if info.WorkspacePath == "" {
|
|
return nil, fmt.Errorf("%w: session %s has no workspace path configured", ErrSessionWorkspaceNotReady, sessionID)
|
|
}
|
|
|
|
if info.TaskID == "" {
|
|
return nil, fmt.Errorf("session %s has no associated task ID", sessionID)
|
|
}
|
|
|
|
// Verify this session should use passthrough mode
|
|
if err := m.verifyPassthroughEnabled(ctx, sessionID, info.AgentProfileID); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If agent ID not in workspace info (snapshot missing/empty), resolve from profile
|
|
if info.AgentID == "" && info.AgentProfileID != "" && m.profileResolver != nil {
|
|
profileInfo, err := m.profileResolver.ResolveProfile(ctx, info.AgentProfileID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("resolve agent for session %s: %w", sessionID, err)
|
|
}
|
|
info.AgentID = profileInfo.AgentName
|
|
}
|
|
|
|
// Create the execution
|
|
m.logger.Info("creating execution for passthrough session",
|
|
zap.String("task_id", info.TaskID),
|
|
zap.String("session_id", sessionID),
|
|
zap.String("workspace_path", info.WorkspacePath))
|
|
|
|
execution, err := m.createExecution(ctx, info.TaskID, info)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("create execution for session %s: %w", sessionID, err)
|
|
}
|
|
|
|
// Start the passthrough process using resume command (recovery after restart)
|
|
m.logger.Info("starting passthrough process for session",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("execution_id", execution.ID))
|
|
|
|
if err := m.ResumePassthroughSession(ctx, sessionID); err != nil {
|
|
return nil, fmt.Errorf("start passthrough process for session %s: %w", sessionID, err)
|
|
}
|
|
|
|
// Get updated execution with process ID
|
|
execution, exists := m.executionStore.GetBySessionID(sessionID)
|
|
if !exists {
|
|
return nil, fmt.Errorf("execution disappeared after starting passthrough session %s", sessionID)
|
|
}
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
// verifyPassthroughEnabled checks if the session's profile has CLI passthrough enabled.
|
|
func (m *Manager) verifyPassthroughEnabled(ctx context.Context, sessionID, profileID string) error {
|
|
if m.profileResolver == nil || profileID == "" {
|
|
return fmt.Errorf("session %s has no profile configured for passthrough mode", sessionID)
|
|
}
|
|
|
|
profileInfo, err := m.profileResolver.ResolveProfile(ctx, profileID)
|
|
if err != nil {
|
|
m.logger.Warn("failed to resolve profile for passthrough check",
|
|
zap.String("session_id", sessionID),
|
|
zap.String("profile_id", profileID),
|
|
zap.Error(err))
|
|
return fmt.Errorf("session %s: failed to resolve profile %s: %w", sessionID, profileID, err)
|
|
}
|
|
|
|
if profileInfo == nil || !profileInfo.CLIPassthrough {
|
|
return fmt.Errorf("session %s is not configured for CLI passthrough mode", sessionID)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// createExecution creates an agentctl execution.
|
|
// The agent subprocess is NOT started - call ConfigureAgent + Start explicitly.
|
|
func (m *Manager) createExecution(ctx context.Context, taskID string, info *WorkspaceInfo) (*AgentExecution, error) {
|
|
// Select runtime based on executor type; falls back to standalone if empty/unavailable
|
|
rt, err := m.getExecutorBackend(info.ExecutorType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("no runtime configured: %w", err)
|
|
}
|
|
|
|
if info.AgentID == "" {
|
|
return nil, fmt.Errorf("agent ID is required in WorkspaceInfo")
|
|
}
|
|
|
|
executionID := uuid.New().String()
|
|
|
|
agentConfig, ok := m.registry.Get(info.AgentID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("agent type %q not found in registry", info.AgentID)
|
|
}
|
|
|
|
// Forward AgentProfile.EnvVars to the runtime instance. The Launch /
|
|
// ResumeSession paths merge these into req.Env via buildEnvForExecution
|
|
// before calling LaunchAgent; the lazy workspace-only path (any
|
|
// GetOrEnsureExecution* caller after backend restart) lands here directly,
|
|
// so without this merge the runtime instance gets spawned with empty env
|
|
// and CLAUDE_CONFIG_DIR (and any other workspace profile var) is lost.
|
|
// The agent subprocess inherits the instance env via agentctl, and ACP
|
|
// session/load then looks under the wrong SDK root → -32002 Resource not
|
|
// found.
|
|
env := map[string]string{}
|
|
m.mergeAgentProfileEnv(ctx, info.AgentProfileID, env)
|
|
if len(env) == 0 {
|
|
env = nil
|
|
}
|
|
|
|
req := &ExecutorCreateRequest{
|
|
InstanceID: executionID,
|
|
TaskID: taskID,
|
|
SessionID: info.SessionID,
|
|
TaskEnvironmentID: info.TaskEnvironmentID,
|
|
AgentProfileID: info.AgentProfileID,
|
|
WorkspacePath: info.WorkspacePath,
|
|
Protocol: string(agentConfig.Runtime().Protocol),
|
|
Env: env,
|
|
AgentConfig: agentConfig,
|
|
Metadata: info.Metadata,
|
|
PreviousExecutionID: info.AgentExecutionID,
|
|
AuthToken: m.revealRuntimeSecret(ctx, info.Metadata, MetadataKeyAuthTokenSecret),
|
|
BootstrapNonce: m.revealRuntimeSecret(ctx, info.Metadata, MetadataKeyBootstrapNonceSecret),
|
|
}
|
|
|
|
runtimeInstance, err := rt.CreateInstance(ctx, req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create execution: %w", err)
|
|
}
|
|
|
|
execution := runtimeInstance.ToAgentExecution(req)
|
|
execution.RuntimeName = rt.Name()
|
|
|
|
// Cache the resolved profile env on the execution so a subsequent
|
|
// configureAndStartAgent (when this workspace-only execution is promoted)
|
|
// reuses it via mergeAgentProfileEnvForExecution instead of doing another
|
|
// secret-store round-trip. Mirrors the Launch path (manager_launch.go).
|
|
if env != nil {
|
|
m.cacheResolvedProfileEnv(execution, env)
|
|
}
|
|
|
|
// Set the ACP session ID for session resumption
|
|
if info.ACPSessionID != "" {
|
|
execution.ACPSessionID = info.ACPSessionID
|
|
}
|
|
|
|
// Create trace span for workspace-only execution
|
|
_, sessionSpan := tracing.TraceSessionStart(
|
|
context.Background(), taskID, info.SessionID, executionID,
|
|
)
|
|
execution.SetSessionSpan(sessionSpan)
|
|
if execution.agentctl != nil {
|
|
execution.agentctl.SetTraceContext(execution.SessionTraceContext())
|
|
}
|
|
|
|
if addErr := m.executionStore.Add(execution); addErr != nil {
|
|
// Lost a race: another path created an execution for this session
|
|
// between our check and our Add. Roll back the runtime instance we
|
|
// just spawned (otherwise its subprocess is orphaned) and return the
|
|
// winner so the caller observes a single execution per session.
|
|
if errors.Is(addErr, ErrExecutionAlreadyExistsForSession) {
|
|
m.rollbackRacedExecution(ctx, rt, runtimeInstance, execution)
|
|
if existing, ok := m.executionStore.GetBySessionID(info.SessionID); ok {
|
|
return existing, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("failed to register execution: %w", addErr)
|
|
}
|
|
|
|
// Persist executors_running row in lockstep with the in-memory Add so the
|
|
// DB never holds an execution_id the store doesn't know about. This is the
|
|
// structural fix for the divergence bug — pre-refactor, the orchestrator
|
|
// wrote the row later via a full-row UPDATE that could race with the store.
|
|
m.persistExecutorRunning(ctx, execution)
|
|
|
|
// Persist agentctl auth token only after the execution is tracked, so a
|
|
// race-lost rollback never leaves an orphaned secret in the store.
|
|
m.persistRuntimeSecrets(ctx, runtimeInstance, execution)
|
|
go m.pollOneRemoteStatus(context.Background(), execution)
|
|
|
|
// Publish Starting BEFORE spawning waitForAgentctlReady so subscribers
|
|
// always observe Starting → Ready/Error in order. Doing it after the go
|
|
// call would race: if Health succeeds before this line runs, Ready could
|
|
// be published first and the frontend gate would briefly flicker.
|
|
m.eventPublisher.PublishAgentctlEvent(ctx, events.AgentctlStarting, execution, "")
|
|
go m.waitForAgentctlReady(execution)
|
|
|
|
m.logger.Info("execution created",
|
|
zap.String("execution_id", executionID),
|
|
zap.String("task_id", taskID),
|
|
zap.String("workspace_path", info.WorkspacePath),
|
|
zap.Stringer("runtime", execution.RuntimeName))
|
|
|
|
return execution, nil
|
|
}
|
|
|
|
// rollbackRacedExecution tears down an execution that lost a session-conflict
|
|
// race in the store. Without this the runtime instance (agentctl + agent
|
|
// subprocess if any) keeps running with no tracking entry, and no cleanup path
|
|
// will ever find it.
|
|
func (m *Manager) rollbackRacedExecution(ctx context.Context, rt ExecutorBackend, runtimeInstance *ExecutorInstance, execution *AgentExecution) {
|
|
m.logger.Warn("rolling back duplicate execution after session-conflict race",
|
|
zap.String("execution_id", execution.ID),
|
|
zap.String("session_id", execution.SessionID))
|
|
if rt != nil && runtimeInstance != nil {
|
|
if stopErr := rt.StopInstance(ctx, runtimeInstance, false); stopErr != nil {
|
|
m.logger.Warn("failed to stop raced runtime instance during rollback",
|
|
zap.String("execution_id", execution.ID),
|
|
zap.Error(stopErr))
|
|
}
|
|
}
|
|
if execution.agentctl != nil {
|
|
execution.agentctl.Close()
|
|
}
|
|
execution.EndSessionSpan()
|
|
}
|
|
|
|
const (
|
|
// MetadataKeyAuthTokenSecret is the metadata key for the encrypted agentctl auth token secret ID.
|
|
MetadataKeyAuthTokenSecret = "env_secret_id_AGENTCTL_AUTH_TOKEN"
|
|
// MetadataKeyBootstrapNonceSecret stores the encrypted Docker bootstrap nonce.
|
|
// It lets the backend re-handshake after a container restart starts a new
|
|
// agentctl process with a fresh auth token.
|
|
MetadataKeyBootstrapNonceSecret = "env_secret_id_AGENTCTL_BOOTSTRAP_NONCE"
|
|
)
|
|
|
|
func (m *Manager) persistRuntimeSecrets(ctx context.Context, instance *ExecutorInstance, execution *AgentExecution) {
|
|
m.persistAuthToken(ctx, instance, execution)
|
|
m.persistBootstrapNonce(ctx, instance, execution)
|
|
}
|
|
|
|
// persistAuthToken stores the agentctl handshake auth token in SecretStore
|
|
// and saves the secret ID in the execution's metadata for recovery after restart.
|
|
func (m *Manager) persistAuthToken(ctx context.Context, instance *ExecutorInstance, execution *AgentExecution) {
|
|
m.persistRuntimeSecret(ctx, instance, execution, MetadataKeyAuthTokenSecret, "agentctl-auth", instance.AuthToken)
|
|
}
|
|
|
|
func (m *Manager) persistBootstrapNonce(ctx context.Context, instance *ExecutorInstance, execution *AgentExecution) {
|
|
m.persistRuntimeSecret(ctx, instance, execution, MetadataKeyBootstrapNonceSecret, "agentctl-bootstrap", instance.BootstrapNonce)
|
|
}
|
|
|
|
func (m *Manager) persistRuntimeSecret(
|
|
ctx context.Context,
|
|
instance *ExecutorInstance,
|
|
execution *AgentExecution,
|
|
metadataKey string,
|
|
secretNamePrefix string,
|
|
value string,
|
|
) {
|
|
if value == "" || m.secretStore == nil {
|
|
return
|
|
}
|
|
|
|
secret := &secrets.SecretWithValue{
|
|
Secret: secrets.Secret{
|
|
Name: fmt.Sprintf("%s-%s", secretNamePrefix, truncateID(instance.InstanceID, 12)),
|
|
},
|
|
Value: value,
|
|
}
|
|
if err := m.secretStore.Create(ctx, secret); err != nil {
|
|
m.logger.Error("failed to persist runtime secret",
|
|
zap.String("instance_id", instance.InstanceID),
|
|
zap.String("metadata_key", metadataKey),
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
|
|
if execution.Metadata == nil {
|
|
execution.Metadata = make(map[string]interface{})
|
|
}
|
|
execution.Metadata[metadataKey] = secret.ID
|
|
|
|
m.logger.Debug("persisted runtime secret in secret store",
|
|
zap.String("instance_id", instance.InstanceID),
|
|
zap.String("metadata_key", metadataKey),
|
|
zap.String("secret_id", secret.ID))
|
|
}
|
|
|
|
func (m *Manager) revealRuntimeSecret(ctx context.Context, metadata map[string]interface{}, metadataKey string) string {
|
|
if m.secretStore == nil {
|
|
return ""
|
|
}
|
|
secretID := getMetadataString(metadata, metadataKey)
|
|
if secretID == "" {
|
|
return ""
|
|
}
|
|
value, err := m.secretStore.Reveal(ctx, secretID)
|
|
if err != nil {
|
|
m.logger.Warn("failed to reveal runtime secret",
|
|
zap.String("metadata_key", metadataKey),
|
|
zap.String("secret_id", secretID),
|
|
zap.Error(err))
|
|
return ""
|
|
}
|
|
return value
|
|
}
|
|
|
|
// truncateID safely truncates an ID string to maxLen characters.
|
|
func truncateID(id string, maxLen int) string {
|
|
if len(id) <= maxLen {
|
|
return id
|
|
}
|
|
return id[:maxLen]
|
|
}
|